@hue-run/sdk 0.1.4 → 0.2.0

package/dist/evals/attempt.d.ts

@@ -0,0 +1,454 @@
+import { z } from "zod";
+import type { CoverageGap } from "../environment/types.js";
+import type { SimulationMcpCapability } from "./types.js";
+/** Canonical caller-observed agent configuration after missing evidence is made explicit. */
+export type ActualAgentManifestV2 = {
+    /** Manifest schema discriminator. */
+    schemaVersion: 2;
+    /** Digests for the six strict parity components. */
+    components: {
+        /** Agent implementation identity. */
+        agent: {
+            /** Canonical SHA-256 digest, or `null` when the evidence is missing. */
+            digest: string | null;
+            /** How the caller obtained the digest. */
+            evidence: "observed" | "declared" | "missing";
+        };
+        /** Prompt identity. */
+        prompt: ActualAgentManifestV2["components"]["agent"];
+        /** Model configuration identity. */
+        model: ActualAgentManifestV2["components"]["agent"];
+        /** Effective tool configuration identity. */
+        tools: ActualAgentManifestV2["components"]["agent"];
+        /** Approval-policy identity. */
+        approvals: ActualAgentManifestV2["components"]["agent"];
+        /** Orchestration identity. */
+        orchestration: ActualAgentManifestV2["components"]["agent"];
+    };
+    /** MCP catalogs in effective model-visible order; this array is never sorted. */
+    catalogs: Array<{
+        /** Caller-selected provider instance. */
+        providerInstanceKey: string;
+        /** MCP surface whose catalog was observed or declared. */
+        surfaceKey: "google.gmail/mcp" | "slack/mcp";
+        /** Catalog digest, or `null` when missing. */
+        digest: string | null;
+        /** How the caller obtained the digest. */
+        evidence: "observed" | "declared" | "missing";
+    }>;
+    /** Native helper configurations in their effective order. */
+    helperConfigurations: Array<{
+        /** Caller-selected provider instance. */
+        providerInstanceKey: string;
+        /** Native provider surface whose helper configuration is described. */
+        surfaceKey: "google.gmail/rest" | "slack/web-api";
+        /** Helper-configuration digest, or `null` when missing. */
+        digest: string | null;
+        /** How the caller obtained the digest. */
+        evidence: "observed" | "declared" | "missing";
+    }>;
+};
+/** Input form of {@link ActualAgentManifestV2}; omitted evidence normalizes to missing. */
+export type ActualAgentManifestInputV2 = {
+    /** Optional schema assertion; when present it must be V2. */
+    schemaVersion?: 2;
+    /** Any omitted component is recorded as missing. */
+    components?: Partial<ActualAgentManifestV2["components"]>;
+    /** MCP catalogs in effective model-visible order. */
+    catalogs?: ActualAgentManifestV2["catalogs"];
+    /** Native helper configurations in effective order. */
+    helperConfigurations?: ActualAgentManifestV2["helperConfigurations"];
+} | undefined;
+/** Immutable expected agent evidence pinned by an experiment. */
+export type ExpectedAgentManifestV2 = {
+    /** Manifest schema discriminator. */
+    schemaVersion: 2;
+    /** Required digests and minimum provenance for all strict components. */
+    components: {
+        /** Agent implementation requirement. */
+        agent: {
+            /** Canonical SHA-256 digest pinned by the experiment. */
+            digest: string;
+            /** Weakest provenance the caller may supply. */
+            minimumEvidence: "observed" | "declared";
+        };
+        /** Prompt requirement. */
+        prompt: ExpectedAgentManifestV2["components"]["agent"];
+        /** Model configuration requirement. */
+        model: ExpectedAgentManifestV2["components"]["agent"];
+        /** Effective tool configuration requirement. */
+        tools: ExpectedAgentManifestV2["components"]["agent"];
+        /** Approval-policy requirement. */
+        approvals: ExpectedAgentManifestV2["components"]["agent"];
+        /** Orchestration requirement. */
+        orchestration: ExpectedAgentManifestV2["components"]["agent"];
+    };
+    /** Required MCP catalogs in effective model-visible order. */
+    catalogs: Array<{
+        /** Pinned provider instance. */
+        providerInstanceKey: string;
+        /** Pinned MCP surface. */
+        surfaceKey: "google.gmail/mcp" | "slack/mcp";
+        /** Expected catalog digest. */
+        digest: string;
+        /** Weakest accepted provenance for this catalog. */
+        minimumEvidence: "observed" | "declared";
+    }>;
+    /** Required native helper configurations in effective order. */
+    helperConfigurations: Array<{
+        /** Pinned provider instance. */
+        providerInstanceKey: string;
+        /** Pinned native surface. */
+        surfaceKey: "google.gmail/rest" | "slack/web-api";
+        /** Expected helper-configuration digest. */
+        digest: string;
+        /** Weakest accepted provenance for this helper configuration. */
+        minimumEvidence: "observed" | "declared";
+    }>;
+};
+/** Secret-free registration identity for one selected provider surface. */
+export type SurfaceBindingV2 = {
+    /** Factory-issued registration identity. */
+    surfaceRegistrationId: string;
+    /** Selected MCP surface. */
+    surfaceKey: "google.gmail/mcp" | "slack/mcp";
+    /** Provider protocol revision. */
+    protocolVersion: string;
+    /** Agent-visible contract digest. */
+    contractDigest: string;
+    /** Effective MCP catalog digest. */
+    catalogDigest: string;
+    /** Native helper evidence is inapplicable to MCP. */
+    helperConfigurationDigest: null;
+    /** Runtime registration implementation digest. */
+    runtimeRegistrationDigest: string;
+} | {
+    /** Factory-issued registration identity. */
+    surfaceRegistrationId: string;
+    /** Selected native provider surface. */
+    surfaceKey: "google.gmail/rest" | "slack/web-api";
+    /** Provider protocol revision. */
+    protocolVersion: string;
+    /** Agent-visible contract digest. */
+    contractDigest: string;
+    /** MCP catalog evidence is inapplicable to native surfaces. */
+    catalogDigest: null;
+    /** Effective native helper-configuration digest. */
+    helperConfigurationDigest: string;
+    /** Runtime registration implementation digest. */
+    runtimeRegistrationDigest: string;
+};
+/** One provider instance and its immutable profile/surface pins. */
+export type DependencyProviderV2 = {
+    /** Caller-visible provider instance key. */
+    providerInstanceKey: string;
+    /** Provider family. */
+    providerId: "google.gmail" | "slack";
+    /** Synthetic principal bound to this world. */
+    syntheticPrincipalId: string;
+    /** Sorted scopes granted to the synthetic principal. */
+    scopes: string[];
+    /** Immutable profile identity and coverage pins. */
+    profile: {
+        /** Versioned profile identifier. */
+        profileId: string;
+        /** Canonical profile digest. */
+        profileDigest: string;
+        /** Adapter build digest. */
+        buildDigest: string;
+        /** Covered-workflow digest. */
+        coverageDigest: string;
+        /** Contract digests sorted by surface key. */
+        contractDigests: Array<{
+            /** Surface described by the contract. */
+            surfaceKey: "google.gmail/mcp" | "google.gmail/rest" | "slack/mcp" | "slack/web-api";
+            /** Agent-visible contract digest. */
+            contractDigest: string;
+        }>;
+    };
+    /** Digest of the supported provider workflow slice. */
+    workflowDigest: string;
+    /** Selected, secret-free surface bindings. */
+    surfaces: SurfaceBindingV2[];
+};
+/** Secret-free dependency manifest pinned to one attempt. */
+export type DependencyManifestV2 = {
+    /** Manifest schema discriminator. */
+    schemaVersion: 2;
+    /** Uniquely keyed provider instances sharing the same world. */
+    providers: DependencyProviderV2[];
+};
+/** Immutable V2 parity and provider baseline stored with an experiment version. */
+export type AttemptBaselineV2 = {
+    /** Baseline schema discriminator. */
+    schemaVersion: 2;
+    /** Immutable expected-manifest identity. */
+    expectedAgentManifestId: string;
+    /** Digest of the expected manifest. */
+    expectedAgentManifestDigest: string;
+    /** Full expected agent evidence. */
+    expectedAgentManifest: ExpectedAgentManifestV2;
+    /** Secret-free provider dependencies. */
+    dependencyManifest: DependencyManifestV2;
+};
+/** Exact provider instance and surface selection requested for an attempt. */
+export type RequestedAttemptProviderV2 = {
+    /** Provider instance pinned by the baseline. */
+    providerInstanceKey: string;
+    /** Selected surface keys in effective order. */
+    surfaceKeys: Array<"google.gmail/mcp" | "google.gmail/rest" | "slack/mcp" | "slack/web-api">;
+};
+/** Canonical V2 prepare input after defaulting caller evidence. */
+export type PrepareAttemptInputV2 = {
+    /** Request schema discriminator. */
+    schemaVersion: 2;
+    /** Stable idempotency key for this preparation decision. */
+    idempotencyKey: string;
+    /** Execution identity inserted into the route, not the JSON body. */
+    executionId: string;
+    /** Isolated world bound to the execution. */
+    environmentRunId: string;
+    /** Stale-client assertion for the experiment's expected manifest. */
+    expectedAgentManifestDigest: string;
+    /** Caller-observed agent evidence with missing fields made explicit. */
+    actualManifest: ActualAgentManifestV2;
+    /** Exact provider/surface selection. */
+    requestedProviders: RequestedAttemptProviderV2[];
+};
+/** Caller input to {@link EvaluationClient.prepareAttempt}. */
+export type PrepareAttemptRequestV2 = Omit<PrepareAttemptInputV2, "actualManifest"> & {
+    /** Caller-observed evidence; omission records an all-missing V2 manifest. */
+    actualManifest?: ActualAgentManifestInputV2;
+};
+/** One strict parity, profile, binding or coverage finding. */
+export type PreflightFindingV2 = {
+    /** Stable finding category. */
+    code: "baseline_missing" | "baseline_assertion_mismatch" | "manifest_mismatch" | "evidence_missing" | "evidence_insufficient" | "helper_configuration_mismatch" | "catalog_mismatch" | "provider_selection_mismatch" | "profile_unavailable" | "profile_mismatch" | "surface_unsupported" | "coverage_gap";
+    /** Agent component involved, when applicable. */
+    component: "agent" | "prompt" | "model" | "tools" | "approvals" | "orchestration" | null;
+    /** Provider instance involved, when applicable. */
+    providerInstanceKey: string | null;
+    /** Provider surface involved, when applicable. */
+    surfaceKey: "google.gmail/mcp" | "google.gmail/rest" | "slack/mcp" | "slack/web-api" | null;
+    /** Fixed nonsecret explanation for the finding code. */
+    message: string;
+};
+/** Credential-free V2 preflight decision. */
+export type PreflightReportV2 = {
+    /** Report schema discriminator. */
+    schemaVersion: 2;
+    /** Whether strict preflight succeeded. */
+    status: "ready" | "environment_incomplete";
+    /** Provenance class for the actual manifest. */
+    evidenceSource: "caller_supplied";
+    /** Empty for ready; otherwise the bounded reasons the environment is incomplete. */
+    findings: PreflightFindingV2[];
+};
+/** Stable parity evidence stored without endpoints or credentials. */
+export type ParityEvidenceV2 = {
+    /** Immutable expected-manifest identity. */
+    expectedAgentManifestId: string;
+    /** Expected manifest digest asserted by the request. */
+    expectedAgentManifestDigest: string;
+    /** Canonical digest of the caller-observed manifest. */
+    actualAgentManifestDigest: string;
+    /** Caller-observed manifest used for preflight. */
+    actualManifest: ActualAgentManifestV2;
+    /** Digest of the secret-free dependency manifest. */
+    dependencyManifestDigest: string;
+    /** Digest binding agent evidence, dependencies and attempt identity. */
+    executionManifestDigest: string;
+    /** Provenance class for the actual manifest. */
+    evidenceSource: "caller_supplied";
+};
+/** Stable identities shared by every credential generation of an attempt. */
+export type AttemptIdentityV2 = {
+    /** Prepared binding identity. */
+    bindingId: string;
+    /** Local target execution identity. */
+    executionId: string;
+    /** Isolated simulated-world identity. */
+    environmentRunId: string;
+};
+/** Credential-bearing V2 provider connections delivered only to the callback. */
+export type AttemptConnectionBundleV2 = AttemptIdentityV2 & {
+    /** Bundle schema discriminator. */
+    schemaVersion: 2;
+    /** Expiry shared by every selected credential in this generation. */
+    expiresAt: string;
+    /** Monotonic credential generation; immutable evidence does not change on rotation. */
+    credentialGeneration: number;
+    /** Selected provider instances and their short-lived connection details. */
+    providers: Array<Omit<DependencyProviderV2, "surfaces"> & {
+        /** Selected surfaces with attempt-scoped connection material. */
+        surfaces: Array<SurfaceBindingV2 & {
+            /** HTTPS provider-facade endpoint. */
+            endpoint: string;
+            /** Short-lived attempt bearer; never persist or log it. */
+            bearer: string;
+        }>;
+    }>;
+    /** Credential-free evidence binding this connection to the strict preflight. */
+    parity: ParityEvidenceV2;
+};
+/** Secret-free persisted binding state, discriminated without converting V1 evidence. */
+export type AttemptBindingRead = {
+    /** Stable binding identity. */
+    bindingId: string;
+    /** Bound execution identity. */
+    executionId: string;
+    /** Bound simulated-world identity. */
+    environmentRunId: string;
+    /** Persisted preflight outcome. */
+    outcome: "ready" | "environment_incomplete";
+    /** Last credential generation, or `null` when no live credential was issued. */
+    credentialGeneration: number | null;
+    /** Expected-manifest identity, or `null` for an incomplete legacy baseline. */
+    expectedAgentManifestId: string | null;
+    /** Expected-manifest digest, or `null` for an incomplete legacy baseline. */
+    expectedAgentManifestDigest: string | null;
+    /** Execution-manifest digest, or `null` when strict parity was not established. */
+    executionManifestDigest: string | null;
+    /** Binding creation timestamp. */
+    createdAt: string;
+    /** Revocation timestamp, or `null` while active. */
+    revokedAt: string | null;
+} & ({
+    /** Legacy evidence remains V1 and is never upgraded in place. */
+    schemaVersion: 1;
+    /** Legacy actual manifest without native-helper evidence. */
+    actualManifest: Omit<ActualAgentManifestV2, "schemaVersion" | "catalogs" | "helperConfigurations"> & {
+        /** Legacy manifest schema discriminator. */
+        schemaVersion: 1;
+        /** Legacy catalogs may refer to any V1 surface. */
+        catalogs: Array<{
+            /** Provider instance described by the catalog. */
+            providerInstanceKey: string;
+            /** Legacy provider surface. */
+            surfaceKey: "google.gmail/mcp" | "google.gmail/rest" | "slack/mcp" | "slack/web-api";
+            /** Catalog digest, or `null` when missing. */
+            digest: string | null;
+            /** How the caller obtained the digest. */
+            evidence: "observed" | "declared" | "missing";
+        }>;
+    };
+    /** Legacy secret-free dependency pins, or `null` for incomplete evidence. */
+    dependencyManifest: null | {
+        /** Legacy manifest schema discriminator. */
+        schemaVersion: 1;
+        /** Legacy provider dependencies. */
+        providers: Array<Omit<DependencyProviderV2, "surfaces"> & {
+            /** Legacy surfaces carried only a catalog digest. */
+            surfaces: Array<{
+                /** Factory-issued registration identity. */
+                surfaceRegistrationId: string;
+                /** Legacy selected surface. */
+                surfaceKey: "google.gmail/mcp" | "google.gmail/rest" | "slack/mcp" | "slack/web-api";
+                /** Provider protocol revision. */
+                protocolVersion: string;
+                /** Agent-visible contract digest. */
+                contractDigest: string;
+                /** Legacy catalog digest. */
+                catalogDigest: string;
+            }>;
+        }>;
+    };
+    /** Coupled legacy preflight evidence. */
+    preflightReport: Omit<PreflightReportV2, "schemaVersion" | "findings"> & {
+        /** Legacy report schema discriminator. */
+        schemaVersion: 1;
+        /** Legacy findings do not include native-helper mismatch. */
+        findings: Array<Omit<PreflightFindingV2, "code"> & {
+            /** Legacy finding category. */
+            code: Exclude<PreflightFindingV2["code"], "helper_configuration_mismatch">;
+        }>;
+    };
+} | {
+    /** Current binding evidence schema. */
+    schemaVersion: 2;
+    /** Actual V2 agent evidence. */
+    actualManifest: ActualAgentManifestV2;
+    /** Secret-free V2 dependency pins, or `null` for incomplete evidence. */
+    dependencyManifest: DependencyManifestV2 | null;
+    /** Coupled V2 preflight evidence. */
+    preflightReport: PreflightReportV2;
+});
+/** Successful V2 preparation result with fresh callback-only credentials. */
+export interface PrepareAttemptReadyV2 {
+    /** Ready discriminator. */
+    status: "ready";
+    /** Finding-free preflight report. */
+    preflightReport: PreflightReportV2 & {
+        /** Ready discriminator narrowed from the report union. */
+        status: "ready";
+    };
+    /** Fresh credential-bearing connection bundle. */
+    bundle: AttemptConnectionBundleV2;
+}
+/** Inconclusive V2 preparation result; no target or scorer may run. */
+export interface PrepareAttemptIncompleteV2 {
+    /** Incomplete discriminator. */
+    status: "environment_incomplete";
+    /** Stable binding identity recorded with the gap. */
+    bindingId: string;
+    /** Preflight findings that prevented strict parity. */
+    preflightReport: PreflightReportV2 & {
+        /** Incomplete discriminator narrowed from the report union. */
+        status: "environment_incomplete";
+    };
+    /** Durable first coverage/preflight gap on the world. */
+    gap: CoverageGap;
+}
+/** Discriminated result of preparing one V2 attempt. */
+export type PrepareAttemptResultV2 = PrepareAttemptReadyV2 | PrepareAttemptIncompleteV2;
+/** Successful credential rotation; immutable binding evidence is unchanged. */
+export type RefreshAttemptResultV2 = PrepareAttemptReadyV2;
+/** Confirmation that one attempt binding was revoked. */
+export interface RevokeAttemptResult {
+    /** Revoked binding identity. */
+    bindingId: string;
+    /** Server timestamp of revocation. */
+    revokedAt: string;
+}
+/** Runtime validator for the caller-observed V2 agent manifest supplied before target execution. */
+export declare const actualAgentManifestV2: z.ZodType<ActualAgentManifestV2, ActualAgentManifestInputV2>;
+/** Runtime validator for the immutable expected V2 agent manifest. */
+export declare const expectedAgentManifestV2: z.ZodType<ExpectedAgentManifestV2>;
+/** Runtime validator for one secret-free V2 surface binding. */
+export declare const surfaceBindingV2: z.ZodType<SurfaceBindingV2>;
+/** Runtime validator for one V2 provider dependency. */
+export declare const dependencyProviderV2: z.ZodType<DependencyProviderV2>;
+/** Runtime validator for the secret-free V2 dependency manifest. */
+export declare const dependencyManifestV2: z.ZodType<DependencyManifestV2>;
+/** Computes the canonical digest shared by actual and expected V2 agent manifests. */
+export declare function agentManifestDigestV2(manifest: ActualAgentManifestV2 | ExpectedAgentManifestV2): string;
+/** Runtime validator for an immutable V2 experiment baseline. */
+export declare const attemptBaselineV2: z.ZodType<AttemptBaselineV2>;
+/** Internal parser reused by `runSimulation` before an execution exists. */
+export declare const requestedAttemptProvidersV2: z.ZodType<RequestedAttemptProviderV2[], RequestedAttemptProviderV2[]>;
+/** Runtime validator for the full V2 prepare input, including its route identity. */
+export declare const prepareAttemptInputV2: z.ZodType<PrepareAttemptInputV2, PrepareAttemptRequestV2>;
+/** Runtime validator for one V2 preflight finding. */
+export declare const preflightFindingV2: z.ZodType<PreflightFindingV2>;
+/** Runtime validator for the V2 preflight report. */
+export declare const preflightReportV2: z.ZodType<PreflightReportV2>;
+/** Runtime validator for stable, credential-free V2 parity evidence. */
+export declare const parityEvidenceV2: z.ZodType<ParityEvidenceV2>;
+/** Runtime validator for the stable V2 attempt identity. */
+export declare const attemptIdentityV2: z.ZodType<AttemptIdentityV2>;
+/** Binds actual agent evidence, secret-free dependencies and stable attempt identities. */
+export declare function executionManifestDigestV2(actualManifest: ActualAgentManifestV2, dependencyManifest: DependencyManifestV2, identity: AttemptIdentityV2): string;
+/** Runtime validator for a credential-bearing V2 connection bundle. */
+export declare const attemptConnectionBundleV2: z.ZodType<AttemptConnectionBundleV2>;
+/** Removes endpoints and bearers from a validated bundle, retaining its dependency identity. */
+export declare function secretFreeBindingV2(bundle: AttemptConnectionBundleV2): DependencyManifestV2;
+/** Legacy context.mcp is a credential-bearing projection, never another source of binding truth. */
+export declare function projectMcpConnectionV2(bundle: AttemptConnectionBundleV2, instanceKey: string): SimulationMcpCapability | null;
+/** Runtime validator for a coupled, secret-free V1 or V2 binding read. */
+export declare const attemptBindingRead: z.ZodType<AttemptBindingRead>;
+export declare function validateAttemptConnectionBundleV2(value: unknown, options?: {
+    requireFresh?: boolean;
+}): AttemptConnectionBundleV2;
+export declare function parsePrepareAttemptResultV2(value: unknown, expected: PrepareAttemptInputV2): PrepareAttemptResultV2;
+export declare function parseRefreshedAttemptResultV2(value: unknown, previous: AttemptConnectionBundleV2): RefreshAttemptResultV2;
+export declare function parseRevocationResult(value: unknown, expectedBindingId: string): RevokeAttemptResult;