@hue-run/sdk 0.1.5 → 0.2.1

package/dist/evals/simulation.js

@@ -0,0 +1,603 @@
+import { randomUUID } from "node:crypto";
+import { join } from "node:path";
+import { HueEnvironmentError } from "../environment/client.js";
+import { bindEnvironmentTools } from "../environment/tools.js";
+import { HueApiError } from "./client.js";
+import { CheckpointStore } from "./checkpoint.js";
+import { MAX_ENVIRONMENT_STEPS } from "./environment-evidence.js";
+import { aggregateBounds, digest, json } from "./json.js";
+import { normalizeScorerDefinitionForPublication } from "./scorer-publication.js";
+import { actualAgentManifestV2, attemptBaselineV2, projectMcpConnectionV2, requestedAttemptProvidersV2, validateAttemptConnectionBundleV2, } from "./attempt.js";
+import { runExperiment, TargetCancelledError, TargetOutcomeUncertainError, } from "./runner.js";
+const scorerDefinition = (entry) => "definition" in entry.scorer ? entry.scorer.definition : entry.scorer;
+function requestedAttempt(options) {
+    const requested = options.requestedProviders !== undefined;
+    const selected = options.mcpSurface !== undefined;
+    if (!requested && !selected) {
+        if (options.actualAgentManifest !== undefined)
+            throw new TypeError("actualAgentManifest requires requestedProviders and mcpSurface");
+        return undefined;
+    }
+    if (!requested || !selected)
+        throw new TypeError("requestedProviders and mcpSurface must be supplied together");
+    const requestedProviders = requestedAttemptProvidersV2.parse(options.requestedProviders);
+    const mcpSurface = options.mcpSurface;
+    const provider = requestedProviders.find((candidate) => candidate.providerInstanceKey === mcpSurface.providerInstanceKey);
+    if (!provider?.surfaceKeys.includes(mcpSurface.surfaceKey))
+        throw new TypeError("mcpSurface must identify an exactly requested MCP surface");
+    const actualAgentManifest = typeof options.actualAgentManifest === "function"
+        ? options.actualAgentManifest
+        : actualAgentManifestV2.parse(options.actualAgentManifest);
+    return {
+        actualAgentManifest,
+        requestedProviders,
+        mcpSurface: { ...mcpSurface },
+    };
+}
+function expectedManifestDigest(config) {
+    if (!config || typeof config !== "object" || Array.isArray(config))
+        throw new TypeError("Provider-profile simulations require an immutable V2 attempt baseline");
+    const source = config;
+    const baseline = attemptBaselineV2.safeParse(source.attemptBaselineV2);
+    if (!baseline.success) {
+        if (source.attemptBaselineV2 === undefined && source.attemptBaselineV1 !== undefined)
+            throw new TypeError("Legacy V1 attempts require a fresh experiment with a V2 baseline");
+        if (source.attemptBaselineV2 !== undefined)
+            throw new TypeError("The immutable V2 attempt baseline is invalid");
+        throw new TypeError("Provider-profile simulations require an immutable V2 attempt baseline");
+    }
+    return baseline.data.expectedAgentManifestDigest;
+}
+function normalizedEnvironmentDefinition(definition) {
+    return json({
+        ...definition,
+        determinism: {
+            clock: {
+                startNs: definition.determinism?.clock?.startNs ?? "0",
+                stepAdvanceNs: definition.determinism?.clock?.stepAdvanceNs ?? "1000000",
+            },
+        },
+        actions: definition.actions.map((action) => ({
+            ...action,
+            params: (action.params ?? []).map((parameter) => ({
+                ...parameter,
+                required: parameter.required ?? true,
+            })),
+            semantics: {
+                ...action.semantics,
+                config: {
+                    ...action.semantics.config,
+                    guards: action.semantics.config.guards ?? [],
+                    notFoundError: action.semantics.config.notFoundError ?? "not_found",
+                },
+            },
+            observation: action.observation ?? { projection: "identity" },
+        })),
+        provenance: definition.provenance ?? { kind: "handwritten" },
+        metadata: definition.metadata ?? {},
+    }, aggregateBounds(240_000));
+}
+function scenarioIdentity(scenario) {
+    if (scenario.kind === "experiment")
+        return scenario;
+    return json({
+        kind: scenario.kind,
+        name: scenario.name,
+        slug: scenario.slug,
+        description: scenario.description ?? "",
+        environment: {
+            ...scenario.environment,
+            definition: normalizedEnvironmentDefinition(scenario.environment.definition),
+        },
+        cases: scenario.cases,
+        scorers: scenario.scorers.map(({ scorer, ...identity }) => ({
+            ...identity,
+            definition: normalizeScorerDefinitionForPublication("definition" in scorer ? scorer.definition : scorer),
+        })),
+        config: scenario.config ?? {},
+    }, aggregateBounds(8 * 1024 * 1024));
+}
+async function findBySlug(page, slug) {
+    let after;
+    for (;;) {
+        const result = await page(after);
+        const found = result.items.find((item) => item.slug === slug);
+        if (found)
+            return found;
+        if (!result.nextCursor)
+            return undefined;
+        after = result.nextCursor;
+    }
+}
+function canReconcileWrite(error) {
+    if (!(error instanceof HueApiError || error instanceof HueEnvironmentError))
+        return false;
+    return error.status === undefined || error.status === 409 || error.status >= 500;
+}
+/** Re-read only writes whose acknowledgement can be ambiguous or whose 409 can be a
+ * concurrent matching publication. Local encoding and deterministic 4xx failures are
+ * caller errors and retain their original type/status.
+ */
+async function reconcileWrite(error, read, unavailableMessage) {
+    if (!canReconcileWrite(error))
+        throw error;
+    let recovered;
+    try {
+        recovered = await read();
+    }
+    catch (readError) {
+        throw new Error(unavailableMessage, {
+            cause: new AggregateError([error, readError], "Write and reconciliation both failed"),
+        });
+    }
+    if (recovered !== undefined)
+        return recovered;
+    // A received conflict is deterministic when no matching concurrent write exists.
+    if (error.status === 409)
+        throw error;
+    throw new Error(unavailableMessage, { cause: error });
+}
+async function resolveEnvironment(client, source) {
+    let identity = await findBySlug((after) => client.listEnvironments({ after, limit: 100 }), source.slug);
+    if (!identity) {
+        try {
+            identity = await client.createEnvironment({
+                name: source.name,
+                slug: source.slug,
+                ...(source.description === undefined ? {} : { description: source.description }),
+            });
+        }
+        catch (error) {
+            identity = await reconcileWrite(error, () => findBySlug((after) => client.listEnvironments({ after, limit: 100 }), source.slug), "Environment creation acknowledgement is unavailable");
+        }
+    }
+    if (identity.archivedAt)
+        throw new Error("Repository scenario environment is archived");
+    const definitionDigest = digest(normalizedEnvironmentDefinition(source.definition));
+    const current = await client.getEnvironment(identity.id);
+    const existing = current.versions.find((version) => version.contentDigest === definitionDigest);
+    if (existing)
+        return existing.id;
+    try {
+        return (await client.publishVersion(identity.id, source.definition)).id;
+    }
+    catch (error) {
+        return (await reconcileWrite(error, async () => (await client.getEnvironment(identity.id)).versions.find((version) => version.contentDigest === definitionDigest), "Environment publication acknowledgement is unavailable")).id;
+    }
+}
+async function resolveScorers(client, sources) {
+    if (!sources.length)
+        throw new TypeError("Repository scenarios require at least one scorer");
+    const versionIds = [];
+    const bindings = [];
+    for (const source of sources) {
+        const supplied = scorerDefinition(source);
+        const definition = normalizeScorerDefinitionForPublication(supplied);
+        let identity = await findBySlug((after) => client.listScorers({ after, limit: 100, includeArchived: true }), source.slug);
+        if (!identity) {
+            try {
+                identity = await client.createScorer({
+                    name: source.name,
+                    slug: source.slug,
+                    ...(source.description === undefined ? {} : { description: source.description }),
+                });
+            }
+            catch (error) {
+                identity = await reconcileWrite(error, () => findBySlug((after) => client.listScorers({ after, limit: 100, includeArchived: true }), source.slug), "Scorer creation acknowledgement is unavailable");
+            }
+        }
+        if (identity.archivedAt)
+            throw new Error("Repository scenario scorer is archived");
+        const full = await client.getScorer(identity.id);
+        const definitionDigest = digest(definition);
+        let version = full.versions?.find((item) => item.contentDigest === definitionDigest);
+        if (!version) {
+            try {
+                version = await client.publishScorerVersion(identity.id, definition);
+            }
+            catch (error) {
+                version = await reconcileWrite(error, async () => (await client.getScorer(identity.id)).versions?.find((item) => item.contentDigest === definitionDigest), "Scorer publication acknowledgement is unavailable");
+            }
+        }
+        versionIds.push(version.id);
+        if ("definition" in source.scorer)
+            bindings.push(source.scorer);
+    }
+    return { versionIds, bindings };
+}
+const normalizedCase = (value, environmentVersionId) => ({
+    externalKey: value.externalKey,
+    inputs: json(value.inputs),
+    ...(Object.hasOwn(value, "expected") ? { expected: json(value.expected) } : {}),
+    metadata: json(value.metadata ?? {}),
+    environmentVersionId,
+});
+const caseDigestValue = (value) => ({
+    externalKey: value.externalKey,
+    inputs: value.inputs,
+    expected: Object.hasOwn(value, "expected") ? value.expected : null,
+    hasExpected: Object.hasOwn(value, "expected"),
+    metadata: value.metadata ?? {},
+    sourceTraceId: value.sourceTraceId ?? null,
+    sourceTraceRevision: value.sourceTraceRevision ?? null,
+    ...(value.artifactManifestId ? { artifactManifestId: value.artifactManifestId } : {}),
+    ...(value.environmentVersionId ? { environmentVersionId: value.environmentVersionId } : {}),
+});
+async function resolveDataset(client, scenario, environmentVersionId) {
+    if (!scenario.cases.length)
+        throw new TypeError("Repository scenarios require at least one case");
+    const cases = scenario.cases.map((item) => normalizedCase(item, environmentVersionId));
+    const keys = new Set(cases.map((item) => item.externalKey));
+    if (keys.size !== cases.length)
+        throw new TypeError("Repository scenario case keys must be unique");
+    const ordered = [...cases].sort((a, b) => Buffer.compare(Buffer.from(a.externalKey), Buffer.from(b.externalKey)));
+    const wantedDigest = digest(ordered.map(caseDigestValue));
+    let identity = await findBySlug((after) => client.listDatasets({ after, limit: 100, includeArchived: true }), scenario.slug);
+    if (!identity) {
+        try {
+            identity = await client.createDataset({
+                name: scenario.name,
+                slug: scenario.slug,
+                ...(scenario.description === undefined ? {} : { description: scenario.description }),
+            });
+        }
+        catch (error) {
+            identity = await reconcileWrite(error, () => findBySlug((after) => client.listDatasets({ after, limit: 100, includeArchived: true }), scenario.slug), "Dataset creation acknowledgement is unavailable");
+        }
+    }
+    if (identity.archivedAt)
+        throw new Error("Repository scenario dataset is archived");
+    let dataset = await client.getDataset(identity.id);
+    const frozen = dataset.versions.find((version) => version.contentDigest === wantedDigest);
+    if (frozen)
+        return frozen.id;
+    let draft = dataset.versions.find((version) => !version.frozenAt);
+    if (!draft) {
+        try {
+            draft = await client.createDatasetVersion(identity.id);
+        }
+        catch (error) {
+            draft = await reconcileWrite(error, async () => {
+                dataset = await client.getDataset(identity.id);
+                return dataset.versions.find((version) => !version.frozenAt);
+            }, "Dataset draft creation acknowledgement is unavailable");
+        }
+    }
+    const current = await allCases(client, draft.id);
+    const desired = new Map(cases.map((item) => [item.externalKey, item]));
+    for (const item of current) {
+        const expected = desired.get(item.externalKey);
+        if (!expected || digest(caseDigestValue(item)) !== digest(caseDigestValue(expected)))
+            throw new Error("Repository scenario conflicts with an existing mutable dataset draft");
+        desired.delete(item.externalKey);
+    }
+    for (const item of cases) {
+        if (!desired.has(item.externalKey))
+            continue;
+        const draftId = draft.id;
+        const expectedRevision = draft.revision;
+        try {
+            const added = await client.addCase(draftId, {
+                ...item,
+                expectedRevision,
+            });
+            draft = added.version;
+        }
+        catch (error) {
+            await reconcileWrite(error, async () => {
+                const recovered = (await allCases(client, draftId)).find((value) => value.externalKey === item.externalKey);
+                return recovered && digest(caseDigestValue(recovered)) === digest(caseDigestValue(item))
+                    ? recovered
+                    : undefined;
+            }, "Dataset case write acknowledgement is unavailable");
+            draft = await client.getDatasetVersion(draftId);
+        }
+    }
+    try {
+        return (await client.freezeDatasetVersion(draft.id, draft.revision)).id;
+    }
+    catch (error) {
+        return (await reconcileWrite(error, async () => {
+            const recovered = await client.getDatasetVersion(draft.id);
+            return recovered.contentDigest === wantedDigest ? recovered : undefined;
+        }, "Dataset freeze acknowledgement is unavailable")).id;
+    }
+}
+async function allCases(client, versionId) {
+    const items = [];
+    let after;
+    for (;;) {
+        const page = await client.listCases(versionId, { after, limit: 100 });
+        items.push(...page.items);
+        if (!page.nextCursor)
+            return items;
+        after = page.nextCursor;
+    }
+}
+async function resolveExperiment(options, idempotencyKey) {
+    if (options.scenario.kind === "experiment") {
+        const source = await options.client.getExperiment(options.scenario.experimentId);
+        const created = await options.client.createExperiment({
+            idempotencyKey,
+            name: options.runName ?? source.name,
+            datasetVersionId: source.datasetVersionId,
+            scorerVersionIds: source.evaluation.scorerVersions.map((item) => item.id),
+            config: source.config,
+        });
+        return { experimentId: created.id, bindings: options.localScorers ?? [] };
+    }
+    const environmentVersionId = await resolveEnvironment(options.environmentClient, options.scenario.environment);
+    const datasetVersionId = await resolveDataset(options.client, options.scenario, environmentVersionId);
+    const scorers = await resolveScorers(options.client, options.scenario.scorers);
+    const created = await options.client.createExperiment({
+        idempotencyKey,
+        name: options.runName ?? options.scenario.name,
+        datasetVersionId,
+        scorerVersionIds: scorers.versionIds,
+        config: options.scenario.config ?? {},
+    });
+    return {
+        experimentId: created.id,
+        bindings: [...scorers.bindings, ...(options.localScorers ?? [])],
+    };
+}
+async function seal(client, runId, executionId, status) {
+    try {
+        await client.finishRun(runId, {
+            idempotencyKey: `execution:${executionId}:${status}`,
+            status,
+        });
+    }
+    catch (error) {
+        const recovered = await client.getRun(runId).catch(() => undefined);
+        if (recovered?.status !== status)
+            throw error;
+    }
+}
+/** Run an existing agent callback against one fresh hosted world per case. The helper
+ * owns immutable resolution, execution linkage, finalization, scoring and resumable uploads.
+ */
+export async function runSimulation(options) {
+    const requestedConfiguration = requestedAttempt(options);
+    if (options.maxSteps !== undefined &&
+        (!Number.isInteger(options.maxSteps) ||
+            options.maxSteps < 1 ||
+            options.maxSteps > MAX_ENVIRONMENT_STEPS))
+        throw new RangeError(`maxSteps must be 1–${MAX_ENVIRONMENT_STEPS}`);
+    if (options.ttlSeconds !== undefined &&
+        (!Number.isInteger(options.ttlSeconds) || options.ttlSeconds < 1 || options.ttlSeconds > 86_400))
+        throw new RangeError("ttlSeconds must be 1–86400");
+    if (options.environmentClient.baseUrl !== options.client.baseUrl)
+        throw new Error("Environments and evaluations must use the same Hue origin");
+    const project = await options.client.checkConnection();
+    const store = await CheckpointStore.acquire(options.checkpointDirectory, {
+        kind: "simulation",
+        projectId: project.id,
+        baseUrl: options.client.baseUrl,
+    });
+    try {
+        const scenarioDigest = digest(scenarioIdentity(options.scenario));
+        let attempt = await store.read("active-attempt");
+        if (attempt && attempt.stage !== "completed" && attempt.scenarioDigest !== scenarioDigest)
+            throw new Error("Recover the unfinished simulation before running a changed scenario");
+        if (!attempt || attempt.stage === "completed") {
+            attempt = {
+                scenarioDigest,
+                idempotencyKey: randomUUID(),
+                stage: "preparing",
+            };
+            await store.write("active-attempt", attempt);
+        }
+        let bindings = options.localScorers ?? [];
+        if (!attempt.experimentId) {
+            const resolved = await resolveExperiment(options, attempt.idempotencyKey);
+            attempt.experimentId = resolved.experimentId;
+            bindings = resolved.bindings;
+            attempt.stage = "running";
+            await store.write("active-attempt", attempt);
+        }
+        else if (options.scenario.kind === "repository") {
+            bindings = [
+                ...options.scenario.scorers
+                    .filter((item) => "definition" in item.scorer)
+                    .map((item) => item.scorer),
+                ...(options.localScorers ?? []),
+            ];
+        }
+        const experimentId = attempt.experimentId;
+        const runUrl = new URL(`/experiments/${experimentId}`, options.client.baseUrl).toString();
+        await options.onProgress?.({ type: "run_created", experimentId, runUrl });
+        const requested = requestedConfiguration
+            ? {
+                ...requestedConfiguration,
+                expectedAgentManifestDigest: expectedManifestDigest((await options.client.getExperiment(experimentId)).config),
+            }
+            : undefined;
+        const report = await runExperiment({
+            client: options.client,
+            hue: options.hue,
+            experimentId,
+            checkpointDirectory: join(store.directory, `experiment-${experimentId}`),
+            persistResultContent: options.persistResultContent,
+            traceEvidence: options.traceEvidence,
+            environmentEvidence: "required",
+            scorers: bindings,
+            concurrency: options.concurrency,
+            schemaTimeoutMillis: options.schemaTimeoutMillis,
+            target: async (inputs, context) => {
+                const environmentVersionId = context.item.environmentVersionId;
+                if (!environmentVersionId)
+                    throw new Error("The simulation case has no pinned environment version");
+                const run = await options.environmentClient.createRun({
+                    idempotencyKey: `execution:${context.executionId}`,
+                    environmentVersionId,
+                    executionId: context.executionId,
+                    maxSteps: options.maxSteps,
+                    ttlSeconds: options.ttlSeconds,
+                });
+                const progress = (type) => options.onProgress?.({
+                    type,
+                    experimentId,
+                    executionId: context.executionId,
+                    caseId: context.item.id,
+                    environmentRunId: run.id,
+                });
+                let finalized = false;
+                try {
+                    await progress("world_created");
+                    if (options.signal?.aborted)
+                        throw new TargetCancelledError();
+                    const tools = bindEnvironmentTools({
+                        hue: options.hue,
+                        client: options.environmentClient,
+                        run,
+                        parentContext: context.span.context,
+                    });
+                    let connectionBundle;
+                    let mcp;
+                    if (requested) {
+                        const actualManifest = actualAgentManifestV2.parse(typeof requested.actualAgentManifest === "function"
+                            ? await requested.actualAgentManifest({
+                                config: context.config,
+                                item: structuredClone(context.item),
+                                signal: options.signal,
+                            })
+                            : requested.actualAgentManifest);
+                        let prepared;
+                        try {
+                            prepared = await options.client.prepareAttempt({
+                                schemaVersion: 2,
+                                idempotencyKey: randomUUID(),
+                                executionId: context.executionId,
+                                environmentRunId: run.id,
+                                expectedAgentManifestDigest: requested.expectedAgentManifestDigest,
+                                actualManifest,
+                                requestedProviders: requested.requestedProviders,
+                            });
+                        }
+                        catch (error) {
+                            // A transport failure or malformed credential-bearing response may
+                            // follow a committed decision. Preserve the running checkpoint and
+                            // never reacquire credentials or replay the target on resume.
+                            throw new TargetOutcomeUncertainError(context.executionId, { cause: error });
+                        }
+                        await options.onProgress?.({
+                            type: "attempt_prepared",
+                            experimentId,
+                            executionId: context.executionId,
+                            caseId: context.item.id,
+                            environmentRunId: run.id,
+                            bindingId: prepared.status === "ready" ? prepared.bundle.bindingId : prepared.bindingId,
+                            status: prepared.status,
+                            findingCodes: prepared.preflightReport.findings.map((finding) => finding.code),
+                            ...(prepared.status === "ready"
+                                ? {
+                                    executionManifestDigest: prepared.bundle.parity.executionManifestDigest,
+                                }
+                                : {}),
+                        });
+                        if (prepared.status === "environment_incomplete") {
+                            try {
+                                await seal(options.environmentClient, run.id, context.executionId, "completed");
+                            }
+                            catch (error) {
+                                throw new TargetOutcomeUncertainError(context.executionId, { cause: error });
+                            }
+                            finalized = true;
+                            await Promise.resolve(progress("world_sealed")).catch(() => undefined);
+                            return undefined;
+                        }
+                        connectionBundle = validateAttemptConnectionBundleV2(prepared.bundle, {
+                            requireFresh: true,
+                        });
+                        const projected = projectMcpConnectionV2(connectionBundle, requested.mcpSurface.providerInstanceKey);
+                        if (!projected)
+                            throw new TypeError("The prepared attempt has no selected MCP surface");
+                        mcp = projected;
+                    }
+                    else {
+                        mcp = await options.client.createSimulationMcpCapability({
+                            runId: run.id,
+                            executionId: context.executionId,
+                        });
+                    }
+                    if (options.signal?.aborted)
+                        throw new TargetCancelledError();
+                    await progress("target_started");
+                    const output = await options.target(inputs, {
+                        config: context.config,
+                        item: context.item,
+                        executionId: context.executionId,
+                        environmentRunId: run.id,
+                        tools,
+                        mcp,
+                        ...(connectionBundle ? { connectionBundle } : {}),
+                        signal: options.signal,
+                    });
+                    try {
+                        await seal(options.environmentClient, run.id, context.executionId, "completed");
+                    }
+                    catch (error) {
+                        throw new TargetOutcomeUncertainError(context.executionId, { cause: error });
+                    }
+                    finalized = true;
+                    await Promise.resolve(progress("world_sealed")).catch(() => undefined);
+                    return output;
+                }
+                catch (error) {
+                    if (error instanceof TargetOutcomeUncertainError || finalized)
+                        throw error;
+                    let environmentIncomplete;
+                    try {
+                        environmentIncomplete =
+                            (await options.environmentClient.getRun(run.id)).validity ===
+                                "environment_incomplete";
+                    }
+                    catch (inspectionError) {
+                        // A target error can be the adapter surfacing a coverage gap. If the
+                        // authoritative run cannot be read, do not guess that it was an agent
+                        // failure or replay the target on resume.
+                        throw new TargetOutcomeUncertainError(context.executionId, {
+                            cause: new AggregateError([error, inspectionError]),
+                        });
+                    }
+                    // A durable coverage gap invalidates parity independently of caller timing;
+                    // do not let a racing local abort hide it as an ordinary cancellation.
+                    if (environmentIncomplete) {
+                        try {
+                            await seal(options.environmentClient, run.id, context.executionId, "completed");
+                        }
+                        catch (finalizationError) {
+                            throw new TargetOutcomeUncertainError(context.executionId, {
+                                cause: new AggregateError([error, finalizationError]),
+                            });
+                        }
+                        finalized = true;
+                        await Promise.resolve(progress("world_sealed")).catch(() => undefined);
+                        return undefined;
+                    }
+                    try {
+                        await seal(options.environmentClient, run.id, context.executionId, "abandoned");
+                    }
+                    catch (finalizationError) {
+                        throw new TargetOutcomeUncertainError(context.executionId, {
+                            cause: new AggregateError([error, finalizationError]),
+                        });
+                    }
+                    await Promise.resolve(progress("world_sealed")).catch(() => undefined);
+                    if (options.signal?.aborted && !(error instanceof TargetCancelledError))
+                        throw new TargetCancelledError();
+                    throw error;
+                }
+            },
+        });
+        const complete = { ...report, experimentId, runUrl };
+        attempt.stage = "completed";
+        attempt.report = complete;
+        await store.write("active-attempt", attempt);
+        return complete;
+    }
+    finally {
+        await store.release();
+    }
+}