@hue-run/sdk 0.3.2 → 0.4.2

package/dist/evals/simulation.js

@@ -46,24 +46,24 @@ function normalizedEnvironmentDefinition(definition) {
         metadata: definition.metadata ?? {},
     }, aggregateBounds(240_000));
 }
-function scenarioIdentity(scenario) {
-    if (scenario.kind === "experiment")
-        return scenario;
+function definitionIdentity(definition) {
+    if (definition.kind === "experiment")
+        return definition;
     return json({
-        kind: scenario.kind,
-        name: scenario.name,
-        slug: scenario.slug,
-        description: scenario.description ?? "",
+        kind: definition.kind,
+        name: definition.name,
+        slug: definition.slug,
+        description: definition.description ?? "",
         environment: {
-            ...scenario.environment,
-            definition: normalizedEnvironmentDefinition(scenario.environment.definition),
+            ...definition.environment,
+            definition: normalizedEnvironmentDefinition(definition.environment.definition),
         },
-        cases: scenario.cases,
-        scorers: scenario.scorers.map(({ scorer, ...identity }) => ({
+        cases: definition.cases,
+        scorers: definition.scorers.map(({ scorer, ...identity }) => ({
             ...identity,
             definition: normalizeScorerDefinitionForPublication("definition" in scorer ? scorer.definition : scorer),
         })),
-        config: scenario.config ?? {},
+        config: definition.config ?? {},
     }, aggregateBounds(8 * 1024 * 1024));
 }
 async function findBySlug(page, slug) {
@@ -283,9 +283,9 @@ async function allCases(client, versionId) {
         after = page.nextCursor;
     }
 }
-async function resolveExperiment(options, idempotencyKey) {
-    if (options.scenario.kind === "experiment") {
-        const source = await options.client.getExperiment(options.scenario.experimentId);
+async function resolveExperiment(options, definition, idempotencyKey) {
+    if (definition.kind === "experiment") {
+        const source = await options.client.getExperiment(definition.experimentId);
         const created = await options.client.createExperiment({
             idempotencyKey,
             name: options.runName ?? source.name,
@@ -295,25 +295,35 @@ async function resolveExperiment(options, idempotencyKey) {
         });
         return { experimentId: created.id, bindings: options.localScorers ?? [] };
     }
-    const environmentVersionId = await resolveEnvironment(options.environmentClient, options.scenario.environment);
-    const datasetVersionId = await resolveDataset(options.client, options.scenario, environmentVersionId);
-    const scorers = await resolveScorers(options.client, options.scenario.scorers);
+    const environmentVersionId = await resolveEnvironment(options.environmentClient, definition.environment);
+    const datasetVersionId = await resolveDataset(options.client, definition, environmentVersionId);
+    const scorers = await resolveScorers(options.client, definition.scorers);
     const created = await options.client.createExperiment({
         idempotencyKey,
-        name: options.runName ?? options.scenario.name,
+        name: options.runName ?? definition.name,
         datasetVersionId,
         scorerVersionIds: scorers.versionIds,
-        config: options.scenario.config ?? {},
+        config: definition.config ?? {},
     });
     return {
         experimentId: created.id,
         bindings: [...scorers.bindings, ...(options.localScorers ?? [])],
     };
 }
+let scenarioDeprecationWarned = false;
 /** Run an existing agent callback against one fresh hosted world per case. The helper
  * owns immutable resolution, execution linkage, finalization, scoring and resumable uploads.
  */
 export async function runSimulation(options) {
+    const definition = options.definition ?? options.scenario;
+    if (!definition)
+        throw new TypeError("runSimulation requires a definition");
+    if (options.definition && options.scenario && options.definition !== options.scenario)
+        throw new TypeError("Pass either definition or scenario, not both");
+    if (!options.definition && !scenarioDeprecationWarned) {
+        scenarioDeprecationWarned = true;
+        process.emitWarning("runSimulation option scenario is deprecated; use definition", "DeprecationWarning");
+    }
     const requestedConfiguration = requestedAttemptV2(options);
     if (options.maxSteps !== undefined &&
         (!Number.isInteger(options.maxSteps) ||
@@ -332,7 +342,7 @@ export async function runSimulation(options) {
         baseUrl: options.client.baseUrl,
     });
     try {
-        const scenarioDigest = digest(scenarioIdentity(options.scenario));
+        const scenarioDigest = digest(definitionIdentity(definition));
         let attempt = await store.read("active-attempt");
         if (attempt && attempt.stage !== "completed" && attempt.scenarioDigest !== scenarioDigest)
             throw new Error("Recover the unfinished simulation before running a changed scenario");
@@ -346,15 +356,15 @@ export async function runSimulation(options) {
         }
         let bindings = options.localScorers ?? [];
         if (!attempt.experimentId) {
-            const resolved = await resolveExperiment(options, attempt.idempotencyKey);
+            const resolved = await resolveExperiment(options, definition, attempt.idempotencyKey);
             attempt.experimentId = resolved.experimentId;
             bindings = resolved.bindings;
             attempt.stage = "running";
             await store.write("active-attempt", attempt);
         }
-        else if (options.scenario.kind === "repository") {
+        else if (definition.kind === "repository") {
             bindings = [
-                ...options.scenario.scorers
+                ...definition.scorers
                     .filter((item) => "definition" in item.scorer)
                     .map((item) => item.scorer),
                 ...(options.localScorers ?? []),

package/dist/evals.d.ts

@@ -3,7 +3,7 @@ export type { EvaluationClientOptions } from "./evals/client.js";
 export { runExperiment, rescore, UncertainExecutionError, OutcomeSerializationError, TargetCancelledError, TargetOutcomeUncertainError, } from "./evals/runner.js";
 export type { RunExperimentOptions, RescoreOptions, RunnerReport } from "./evals/runner.js";
 export { runSimulation } from "./evals/simulation.js";
-export type { RepositorySimulationCase, RepositorySimulationScorer, RunSimulationOptions, SimulationProgress, SimulationReport, SimulationScenario, SimulationTargetContext, } from "./evals/simulation.js";
+export type { RepositorySimulationCase, RepositorySimulationScorer, RunSimulationOptions, SimulationDefinition, SimulationProgress, SimulationReport, SimulationScenario, SimulationTargetContext, } from "./evals/simulation.js";
 export { actualAgentManifestV2, agentManifestDigestV2, attemptBaselineV2, attemptBindingRead, attemptConnectionBundleV2, attemptIdentityV2, dependencyManifestV2, dependencyProviderV2, expectedAgentManifestV2, executionManifestDigestV2, parityEvidenceV2, preflightFindingV2, preflightReportV2, prepareAttemptInputV2, projectMcpConnectionV2, secretFreeBindingV2, surfaceBindingV2, } from "./evals/attempt.js";
 export type { ActualAgentManifestInputV2, ActualAgentManifestV2, AttemptBaselineV2, AttemptBindingRead, AttemptConnectionBundleV2, AttemptIdentityV2, DependencyManifestV2, DependencyProviderV2, ExpectedAgentManifestV2, ParityEvidenceV2, PreflightFindingV2, PreflightReportV2, PrepareAttemptIncompleteV2, PrepareAttemptInputV2, PrepareAttemptReadyV2, PrepareAttemptRequestV2, PrepareAttemptResultV2, RefreshAttemptResultV2, RequestedAttemptProviderV2, RevokeAttemptResult, SurfaceBindingV2, } from "./evals/attempt.js";
 export { builtins, defineLocalScorer, scoreLocally } from "./evals/scorers.js";

package/dist/receipt.js

@@ -31,7 +31,7 @@ function validId(value, length) {
 function record(value) {
     return value !== null && typeof value === "object" && !Array.isArray(value);
 }
-function parseReceipt(value, traceId, expected, origin) {
+function parseReceipt(value, traceId, expected, origin, setup = false) {
     if (!record(value) ||
         value.traceId !== traceId ||
         !Number.isSafeInteger(value.spanCount) ||
@@ -52,6 +52,24 @@ function parseReceipt(value, traceId, expected, origin) {
     }
     if (traceUrl.origin !== origin || traceUrl.username || traceUrl.password)
         invalidResponse();
+    if (setup) {
+        const uuid = /^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/u;
+        if (traceUrl.href !== value.traceUrl ||
+            traceUrl.hash ||
+            !traceUrl.pathname.startsWith("/traces/") ||
+            !uuid.test(traceUrl.pathname.slice(8)))
+            invalidResponse();
+        const queryNames = [];
+        traceUrl.searchParams.forEach((value, name) => {
+            if (!["projectId", "organizationId"].includes(name) ||
+                queryNames.includes(name) ||
+                !uuid.test(value))
+                invalidResponse();
+            queryNames.push(name);
+        });
+        if (queryNames.length !== 2)
+            invalidResponse();
+    }
     const matched = value.matchedSpanIds, missing = value.missingSpanIds;
     if (!Array.isArray(matched) ||
         !Array.isArray(missing) ||
@@ -128,6 +146,13 @@ async function pause(milliseconds, signal) {
 }
 /** Observe persisted evidence after the application and its exporter have finished. */
 export async function verifyTrace(connection, traceId, options = {}) {
+    return verifyTraceAtPath(connection, traceId, options, false, fetch);
+}
+/** @internal Dedicated receipt path for setup credentials; ordinary clients stay unchanged. */
+export async function verifySetupTrace(connection, traceId, options, fetcher, signal) {
+    return verifyTraceAtPath(connection, traceId, options, true, fetcher, signal);
+}
+async function verifyTraceAtPath(connection, traceId, options, setup, fetcher, signal) {
     if (!validId(traceId, 32))
         throw new TypeError("traceId must be a nonzero lowercase 32-character OpenTelemetry trace ID");
     if (options === null || typeof options !== "object" || Array.isArray(options))
@@ -148,27 +173,28 @@ export async function verifyTrace(connection, traceId, options = {}) {
         throw new TypeError("timeoutMillis must be greater than zero and at most 60000");
     // Snapshot caller arrays so concurrent mutation cannot alter the verification criteria.
     const expectedIds = [...expected], requiredFields = [...required];
-    const url = new URL(`/api/v1/traces/${traceId}/receipt`, connection.baseUrl);
+    const url = new URL(`/api/v1/${setup ? "setup/" : ""}traces/${traceId}/receipt`, connection.baseUrl);
     for (const id of expectedIds)
         url.searchParams.append("expectedSpanId", id);
     const controller = new AbortController();
     const deadline = performance.now() + timeout;
     const timer = setTimeout(() => controller.abort(), timeout);
+    const requestSignal = signal ? AbortSignal.any([signal, controller.signal]) : controller.signal;
     let receipt = null;
     let delay = 250;
     try {
-        while (!controller.signal.aborted && performance.now() < deadline) {
-            const response = await fetch(url, {
+        while (!requestSignal.aborted && performance.now() < deadline) {
+            const response = await fetcher(url, {
                 headers: { Authorization: `Bearer ${connection.apiKey}`, Accept: "application/json" },
                 redirect: "manual",
                 credentials: "omit",
                 cache: "no-store",
-                signal: controller.signal,
+                signal: requestSignal,
             });
             let retryAfter = 0;
             if (response.status === 200) {
-                receipt = parseReceipt(await readJson(response), traceId, expectedIds, url.origin);
-                if (!controller.signal.aborted &&
+                receipt = parseReceipt(await readJson(response), traceId, expectedIds, url.origin, setup);
+                if (!requestSignal.aborted &&
                     performance.now() < deadline &&
                     receipt.missingSpanIds.length === 0 &&
                     requiredFields.every((field) => receipt.fields[field]))
@@ -195,7 +221,7 @@ export async function verifyTrace(connection, traceId, options = {}) {
             if (remaining <= 0)
                 break;
             const wait = Math.max(delay, retryAfter);
-            await pause(Math.min(wait, remaining), controller.signal);
+            await pause(Math.min(wait, remaining), requestSignal);
             // A truncated backoff exhausts this call even if a timer wakes just early.
             if (wait >= remaining)
                 break;
@@ -203,6 +229,8 @@ export async function verifyTrace(connection, traceId, options = {}) {
         }
     }
     catch (error) {
+        if (signal?.aborted)
+            throw new Error("Setup interrupted");
         if (!controller.signal.aborted && performance.now() < deadline) {
             if (error instanceof HueTraceVerificationError)
                 throw error;

package/dist/setup/application.d.ts

@@ -0,0 +1,74 @@
+import type { SetupFileChange } from "./configure.js";
+import { type FileSetupInstallationStore, type SetupInstallationRecord, type SetupStoredApplicationEvidence } from "./installation.js";
+import type { SetupProjectDetection } from "./types.js";
+/** Closed automatic application matrix; other projects require an explicit agent-owned integration. */
+export type SetupApplicationPlan = {
+    /** JavaScript or TypeScript application using the shared TypeScript SDK. */
+    language: "typescript";
+    /** Package manager selected from the project's manifest and lockfile. */
+    manager: "bun" | "npm";
+    /** Recognized HTTP framework. */
+    framework: "express";
+    /** Runtime named by the recognized start script; defaults to Node when omitted. */
+    runtime?: "node" | "bun";
+    /** Whether the start script explicitly enables Node source maps. */
+    sourceMaps?: boolean;
+    /** Validated project-relative application entrypoint. */
+    entrypoint: string;
+    /** One literal loopback HTTP pathname selected for the application request. */
+    requestPath: string;
+    /** SHA-256 of the source inspected during planning. */
+    entryDigest: string;
+} | {
+    /** Python application using the Python SDK. */
+    language: "python";
+    /** Environment manager for the supported Python application. */
+    manager: "uv";
+    /** Recognized HTTP framework. */
+    framework: "flask";
+    /** Supported project-relative Flask entrypoint. */
+    entrypoint: "app.py";
+    /** One literal loopback HTTP pathname selected for the application request. */
+    requestPath: string;
+    /** SHA-256 of the source inspected during planning. */
+    entryDigest: string;
+};
+/** Stable reason an automatic application integration is not safe. */
+export declare class SetupApplicationActionRequired extends Error {
+    /** Stable reason the caller must resolve before automatic integration continues. */
+    readonly code: "ambiguous-project" | "unsupported-manager" | "unsupported-framework" | "ambiguous-entrypoint" | "custom-instrumentation";
+    constructor(
+    /** Stable reason the caller must resolve before automatic integration continues. */
+    code: "ambiguous-project" | "unsupported-manager" | "unsupported-framework" | "ambiguous-entrypoint" | "custom-instrumentation", message: string);
+}
+/** Fixed argv execution boundary used for package managers and application entrypoints. */
+export interface SetupCommand {
+    /** Fixed executable name or runtime path; never evaluated by a shell. */
+    command: string;
+    /** Explicit argument vector supplied to the executable. */
+    args: string[];
+    /** Selected application's working directory. */
+    cwd: string;
+    /** Optional child environment; values are never included in command output. */
+    env?: NodeJS.ProcessEnv;
+    /** Maximum elapsed execution time before termination. */
+    timeoutMillis: number;
+}
+/** Executes a fixed argv without a shell and returns only its status. */
+export type SetupCommandRunner = (command: SetupCommand) => Promise<void>;
+/** Statically recognizes the deliberately narrow automatic matrix without executing project code. */
+export declare function planSetupApplication(project: SetupProjectDetection): Promise<SetupApplicationPlan>;
+/** Default command runner: fixed argv, no shell, bounded output and deadline. */
+export declare function runSetupCommand(input: SetupCommand): Promise<void>;
+/** Installs the exact runtime with the detected owner of the project's manifest and lockfile. */
+export declare function installSetupRuntime(project: SetupProjectDetection, plan: SetupApplicationPlan, runner?: SetupCommandRunner): Promise<boolean>;
+/** Adds only two owned wiring blocks around an existing app object; business logic is untouched. */
+export declare function wireSetupApplication(store: FileSetupInstallationStore, record: SetupInstallationRecord, plan: SetupApplicationPlan): Promise<SetupFileChange | undefined>;
+/** Starts the existing entrypoint without a shell and exercises one existing HTTP GET route. */
+export declare function exerciseSetupApplication(store: FileSetupInstallationStore, record: SetupInstallationRecord, plan: SetupApplicationPlan, signal?: AbortSignal, deadlines?: {
+    readinessMillis: number;
+    requestMillis: number;
+    evidenceMillis: number;
+    /** Internal loopback test seam; the public CLI always chooses a random port. */
+    port?: number;
+}): Promise<SetupStoredApplicationEvidence>;