@hue-run/sdk 0.1.5 → 0.2.0

package/dist/evals/simulation.d.ts

@@ -0,0 +1,184 @@
+import type { HueClient } from "../client.js";
+import { type EnvironmentClient } from "../environment/client.js";
+import { type EnvironmentTool } from "../environment/tools.js";
+import type { EnvironmentDefinition, EnvironmentIdentity } from "../environment/types.js";
+import { type EvaluationClient } from "./client.js";
+import { type ActualAgentManifestInputV2, type AttemptConnectionBundleV2, type RequestedAttemptProviderV2 } from "./attempt.js";
+import { type RunnerReport } from "./runner.js";
+import type { ExperimentCase, Identity, JsonValue, LocalScorer, ScorerDefinition, SimulationMcpCapability } from "./types.js";
+/** Repository-authored scorer identity and its public definition or local binding. */
+export interface RepositorySimulationScorer extends Identity {
+    /** Scorer definition to publish, or a local callback bound to its source digest. */
+    scorer: ScorerDefinition | LocalScorer;
+}
+/** One repository-authored scenario case. */
+export interface RepositorySimulationCase {
+    /** Stable key unique within this scenario dataset. */
+    externalKey: string;
+    /** JSON inputs passed to the local target callback. */
+    inputs: JsonValue;
+    /** Optional reference output for scorers. */
+    expected?: JsonValue;
+    /** Optional caller-owned case metadata. */
+    metadata?: Record<string, JsonValue>;
+}
+/** Immutable app-authored experiment reference or repository-authored scenario definition. */
+export type SimulationScenario = {
+    /** Select an existing app-authored immutable experiment template. */
+    kind: "experiment";
+    /** Experiment to clone into a fresh attempt. */
+    experimentId: string;
+} | {
+    /** Publish and resolve the repository-authored definition. */
+    kind: "repository";
+    /** Experiment display name. */
+    name: string;
+    /** Stable dataset slug used for immutable version resolution. */
+    slug: string;
+    /** Optional experiment/dataset description. */
+    description?: string;
+    /** Environment identity and authored world definition. */
+    environment: EnvironmentIdentity & {
+        /** Definition normalized and published as an immutable version. */
+        definition: EnvironmentDefinition;
+    };
+    /** Cases published into one frozen dataset version. */
+    cases: RepositorySimulationCase[];
+    /** Scorers published and pinned by immutable version. */
+    scorers: RepositorySimulationScorer[];
+    /** JSON configuration passed to every target callback. */
+    config?: JsonValue;
+};
+/** Progress emitted without target output, credentials or other sensitive content. */
+export type SimulationProgress = {
+    /** Emitted as soon as the inspectable experiment exists. */
+    type: "run_created";
+    /** Fresh experiment identity. */
+    experimentId: string;
+    /** Browser URL for inspecting progress and evidence. */
+    runUrl: string;
+} | {
+    /** World/target lifecycle event. */
+    type: "world_created" | "target_started" | "world_sealed";
+    /** Fresh experiment identity. */
+    experimentId: string;
+    /** Target execution identity. */
+    executionId: string;
+    /** Frozen experiment-case identity. */
+    caseId: string;
+    /** Isolated simulated-world identity. */
+    environmentRunId: string;
+} | {
+    /** Strict provider-profile preflight decision. */
+    type: "attempt_prepared";
+    /** Fresh experiment identity. */
+    experimentId: string;
+    /** Target execution identity. */
+    executionId: string;
+    /** Frozen experiment-case identity. */
+    caseId: string;
+    /** Isolated simulated-world identity. */
+    environmentRunId: string;
+    /** Stable attempt binding identity. */
+    bindingId: string;
+    /** Whether strict parity was established. */
+    status: "ready" | "environment_incomplete";
+    /** Stable nonsecret finding codes. */
+    findingCodes: string[];
+    /** Credential-free execution manifest digest for ready attempts. */
+    executionManifestDigest?: AttemptConnectionBundleV2["parity"]["executionManifestDigest"];
+};
+/** Context supplied to the existing local agent callback for one case attempt. */
+export interface SimulationTargetContext {
+    /** Frozen experiment configuration. */
+    config: JsonValue;
+    /** Frozen case including its immutable environment-version pin. */
+    item: ExperimentCase;
+    /** Target execution identity. */
+    executionId: string;
+    /** Stable world identity for adapter control operations such as coverage reporting. */
+    environmentRunId: string;
+    /** Framework-neutral local callables backed by this attempt's isolated world. */
+    tools: Record<string, EnvironmentTool>;
+    /** Short-lived capability for providers that execute MCP remotely. */
+    mcp: SimulationMcpCapability;
+    /** Credential-bearing provider connections for this callback only. Hue never
+     * checkpoints, logs or adds this response to parity digests. */
+    connectionBundle?: AttemptConnectionBundleV2;
+    /** Cancellation is cooperative: pass this signal into the real agent/provider call. */
+    signal?: AbortSignal;
+}
+/** Options for {@link runSimulation}. */
+export interface RunSimulationOptions {
+    /** Evaluation client for the target Hue project. */
+    client: EvaluationClient;
+    /** Environment client for the same Hue origin and project. */
+    environmentClient: EnvironmentClient;
+    /** Hue telemetry client used for target and tool spans. */
+    hue: HueClient;
+    /** Dedicated private directory for resumable checkpoints. */
+    checkpointDirectory: string;
+    /** App-authored reference or repository-authored scenario. */
+    scenario: SimulationScenario;
+    /** Required privacy decision for saved target/scorer content. */
+    persistResultContent: boolean;
+    /** Required trace receipt policy for every target attempt. */
+    traceEvidence: {
+        /** Wait for acknowledged trace/log export. */
+        mode: "required";
+    } | {
+        /** Explicitly omit stored trace evidence. */
+        mode: "omit";
+        /** Bounded explanation for the omission. */
+        reason: string;
+    };
+    /** Local scorer callbacks bound by their declared source digests. */
+    localScorers?: LocalScorer[];
+    /** Cases in flight, 1–16; defaults to 1. */
+    concurrency?: number;
+    /** JSON Schema worker deadline in milliseconds. */
+    schemaTimeoutMillis?: number;
+    /** Per-world action ceiling, 1–500. */
+    maxSteps?: number;
+    /** Per-world lease in seconds, 1–86400. */
+    ttlSeconds?: number;
+    /** Cooperative caller cancellation signal. */
+    signal?: AbortSignal;
+    /** Optional display name for the fresh experiment. */
+    runName?: string;
+    /** Opt into immutable provider-profile preflight. Provider selection and the MCP
+     * projection are required together. Omitted actual evidence normalizes to explicit
+     * V2 missing evidence; it is never assumed equal to the baseline. */
+    actualAgentManifest?: ActualAgentManifestInputV2 | ((context: {
+        /** Frozen experiment configuration. */
+        config: JsonValue;
+        /** Frozen case selected for this attempt. */
+        item: ExperimentCase;
+        /** Cooperative caller cancellation signal. */
+        signal?: AbortSignal;
+    }) => ActualAgentManifestInputV2 | Promise<ActualAgentManifestInputV2>);
+    /** Exact provider instances and ordered surfaces asserted for strict preflight. */
+    requestedProviders?: RequestedAttemptProviderV2[];
+    /** Requested MCP surface projected to the backwards-compatible `context.mcp`. */
+    mcpSurface?: {
+        /** Provider instance selected from `requestedProviders`. */
+        providerInstanceKey: string;
+        /** Selected MCP surface. */
+        surfaceKey: "google.gmail/mcp" | "slack/mcp";
+    };
+    /** Invokes the existing local agent exactly once for this attempt. */
+    target(inputs: JsonValue, context: SimulationTargetContext): JsonValue | undefined | Promise<JsonValue | undefined>;
+    /** Receives bounded nonsecret lifecycle progress. */
+    onProgress?(event: SimulationProgress): void | Promise<void>;
+}
+/** Completed simulation report with its inspectable experiment identity. */
+export interface SimulationReport extends RunnerReport {
+    /** Fresh experiment created for this invocation. */
+    experimentId: string;
+    /** Browser URL joining task, trace, world evidence and scoring. */
+    runUrl: string;
+}
+/** Run an existing agent callback against one fresh hosted world per case. The helper
+ * owns immutable resolution, execution linkage, finalization, scoring and resumable uploads.
+ */
+export declare function runSimulation(options: RunSimulationOptions): Promise<SimulationReport>;