anpord 0.1.20 → 0.1.21

package/dist/{evals-api-B0lnr_R8.mjs → evals-api-CK3F1KoI.mjs}

@@ -1,6 +1,6 @@
 import { n as EvalJudgment, r as EvalHarness$1, t as EvalJudge } from "./eval-judges-Bee_ABJt.mjs";
-import { t as EvalValidations } from "./eval-validations-CUG2PvR_.mjs";
-import { r as EvalUser } from "./eval-turns-DZTHit19.mjs";
+import { n as EvalValidations } from "./eval-validations-DNO-qwkU.mjs";
+import { i as EvalUser } from "./eval-turns-BC-GMHD-.mjs";
 import { Context, Schema } from "effect";
 import { HttpApiEndpoint, HttpApiGroup, HttpApiMiddleware, HttpApiSchema, HttpApiSecurity, OpenApi } from "@effect/platform";
 //#region ../schema/src/domain/credentials.ts
@@ -199,10 +199,11 @@ const PROFILE_HARNESS_RULE = "The command harness needs a profile with a run com
 //#endregion
 //#region ../schema/src/domain/evals.ts
 const EvalHarness = EvalHarness$1;
-const EvalSandbox = Schema.Literal("daytona", "e2b", "upstash", "modal", "cloudflare", "vercel");
-EvalSandbox.literals;
+const EvalSandbox = Schema.Literal("daytona", "e2b", "upstash", "modal", "cloudflare", "vercel", "local");
+const HOSTED_SANDBOXES = EvalSandbox.literals.filter((sandbox) => sandbox !== "local");
 const EvalTrialStatus = Schema.Literal("queued", "running", "passed", "failed", "void");
 const EvalRunStatus = Schema.Literal("running", "finished", "failed");
+const EvalExecutor = Schema.Literal("client");
 const EvalSource = Schema.Union(Schema.Struct({ kind: Schema.Literal("empty") }), Schema.Struct({
 	kind: Schema.Literal("repo"),
 	ref: Schema.NullOr(Schema.String),
@@ -307,7 +308,7 @@ Schema.Struct({
 	tasks: Schema.Array(EvalTaskRequest).pipe(Schema.minItems(1), Schema.maxItems(20)),
 	trials: Schema.Int.pipe(Schema.between(1, 10))
 });
-const OccurredAtMillis = Schema.NullOr(Schema.Number);
+const OccurredAtMillis$1 = Schema.NullOr(Schema.Number);
 const EvalUsage = Schema.Struct({
 	cacheReadTokens: Schema.Int,
 	cacheWriteTokens: Schema.Int,
@@ -323,18 +324,18 @@ const EvalJournalEntry = Schema.Union(Schema.Struct({
 	_tag: Schema.Literal("command"),
 	command: Schema.String,
 	exitCode: Schema.NullOr(Schema.Int),
-	finishedAtMillis: OccurredAtMillis,
+	finishedAtMillis: OccurredAtMillis$1,
 	output: Schema.String,
 	outputTruncated: Schema.optional(Schema.Boolean),
-	startedAtMillis: OccurredAtMillis
+	startedAtMillis: OccurredAtMillis$1
 }), Schema.Struct({
 	_tag: Schema.Literal("message"),
-	finishedAtMillis: OccurredAtMillis,
+	finishedAtMillis: OccurredAtMillis$1,
 	text: Schema.String,
 	usage: Schema.optional(Schema.NullOr(EvalUsage))
 }), Schema.Struct({
 	_tag: Schema.Literal("toolCall"),
-	finishedAtMillis: OccurredAtMillis,
+	finishedAtMillis: OccurredAtMillis$1,
 	input: Schema.optional(Schema.String),
 	name: Schema.String,
 	output: Schema.optional(Schema.String),
@@ -342,11 +343,11 @@ const EvalJournalEntry = Schema.Union(Schema.Struct({
 	outputTruncated: Schema.optional(Schema.Boolean),
 	inputTruncated: Schema.optional(Schema.Boolean),
 	errorTruncated: Schema.optional(Schema.Boolean),
-	startedAtMillis: Schema.optional(OccurredAtMillis),
+	startedAtMillis: Schema.optional(OccurredAtMillis$1),
 	status: Schema.NullOr(Schema.String)
 }), Schema.Struct({
 	_tag: Schema.Literal("fileChange"),
-	finishedAtMillis: OccurredAtMillis,
+	finishedAtMillis: OccurredAtMillis$1,
 	paths: Schema.Array(Schema.String)
 })).annotations({
 	description: "A normalized event recorded from the harness trajectory.",
@@ -493,6 +494,7 @@ const EvalRun = Schema.Struct({
 	cases: Schema.Array(Schema.String),
 	cells: Schema.Array(EvalCell),
 	costs: Schema.NullOr(EvalCosts),
+	executedBy: Schema.optionalWith(Schema.NullOr(EvalExecutor), { default: () => null }),
 	failure: Schema.NullOr(Schema.String),
 	finishedAt: Schema.NullOr(EvalTimestamp),
 	id: Schema.String,
@@ -547,6 +549,14 @@ const EvalCellHistoryEntry = Schema.Struct({
 	description: "A previous scored result for the same cell identity.",
 	identifier: "EvalCellHistoryEntry"
 });
+const RunSubscription = Schema.Struct({
+	expiresAtMillis: Schema.Number,
+	tag: Schema.String,
+	token: Schema.String
+}).annotations({
+	description: "A scoped, read-only token for watching one run in real time.",
+	identifier: "RunSubscription"
+});
 //#endregion
 //#region ../schema/src/domain/eval-playground.ts
 const PlaygroundCaseView = Schema.Struct({
@@ -628,6 +638,75 @@ const ModelCatalogue = Schema.Struct({
 	identifier: "ModelCatalogue"
 });
 //#endregion
+//#region ../schema/src/domain/harness-event.ts
+const OccurredAtMillis = Schema.optional(Schema.Number);
+const HarnessUsage = Schema.Struct({
+	cacheReadTokens: Schema.Int,
+	cacheWriteTokens: Schema.Int,
+	costUsd: Schema.optional(Schema.Number),
+	inputTokens: Schema.Int,
+	outputTokens: Schema.Int,
+	totalTokens: Schema.Int
+});
+const HarnessEvent = Schema.Union(Schema.Struct({
+	_tag: Schema.Literal("Started"),
+	at: OccurredAtMillis,
+	model: Schema.String,
+	sessionId: Schema.String
+}), Schema.Struct({
+	_tag: Schema.Literal("Message"),
+	at: OccurredAtMillis,
+	role: Schema.Literal("assistant", "user"),
+	text: Schema.String,
+	usage: Schema.optional(HarnessUsage)
+}), Schema.Struct({
+	_tag: Schema.Literal("Command"),
+	at: OccurredAtMillis,
+	command: Schema.String,
+	exitCode: Schema.NullOr(Schema.Int),
+	output: Schema.String,
+	startedAt: Schema.optional(Schema.Number)
+}), Schema.Struct({
+	_tag: Schema.Literal("FileChange"),
+	at: OccurredAtMillis,
+	paths: Schema.Array(Schema.String)
+}), Schema.Struct({
+	_tag: Schema.Literal("ToolCall"),
+	at: OccurredAtMillis,
+	callId: Schema.NullOr(Schema.String),
+	input: Schema.String,
+	name: Schema.String,
+	output: Schema.optional(Schema.String),
+	error: Schema.optional(Schema.String),
+	startedAt: Schema.optional(Schema.Number),
+	status: Schema.NullOr(Schema.String)
+}), Schema.Struct({
+	_tag: Schema.Literal("Finished"),
+	at: OccurredAtMillis,
+	reason: Schema.String
+}));
+//#endregion
+//#region ../schema/src/domain/trial.ts
+const TrialStatus = Schema.Literal("queued", "running", "passed", "failed", "void");
+Schema.decodeUnknownOption(TrialStatus);
+const VerifyStepResult = Schema.Struct({
+	command: Schema.String,
+	exitCode: Schema.Int
+});
+const TrialOutcome = Schema.Struct({
+	artifacts: Schema.optional(Schema.Array(EvalArtifactMetadata)),
+	validations: Schema.optional(EvalValidations),
+	judgments: Schema.optional(Schema.Array(EvalJudgment)),
+	commandCount: Schema.Int,
+	exitCode: Schema.Int,
+	modelMs: Schema.Int,
+	passed: Schema.Boolean,
+	sandboxMs: Schema.Int,
+	status: TrialStatus,
+	verifySteps: Schema.Array(VerifyStepResult),
+	voidFields: Schema.Array(Schema.String)
+});
+//#endregion
 //#region ../schema/src/public/evals-api.ts
 const EvalRunRequest = Schema.Struct({ id: Schema.String }).annotations({
 	description: "Select an eval run by id.",
@@ -644,7 +723,7 @@ const EvalModelsRequest = Schema.Struct({
 	description: "Select a harness whose available models should be listed.",
 	identifier: "EvalModelsRequest"
 });
-const PublicEvalSandbox = EvalSandbox.annotations({
+const PublicEvalSandbox = Schema.Literal(...HOSTED_SANDBOXES).annotations({
 	description: "The hosted sandbox a task runs in.",
 	identifier: "PublicEvalSandbox"
 });
@@ -680,6 +759,7 @@ const PublicEvalTask = Schema.Struct({
 const PublicStartEvalRequest = Schema.Struct({
 	trigger: Schema.optional(EvalTrigger),
 	cases: Schema.Array(PublicEvalCase).pipe(Schema.minItems(1), Schema.maxItems(100)),
+	executeLocally: Schema.optional(Schema.Boolean),
 	name: Schema.optional(EvalName),
 	prompt: EvalPrompt,
 	tasks: Schema.Array(PublicEvalTask).pipe(Schema.minItems(1), Schema.maxItems(20)),
@@ -688,6 +768,25 @@ const PublicStartEvalRequest = Schema.Struct({
 	description: `Start a grid with at most 100 total case, task, and trial combinations.`,
 	identifier: "StartEvalRequest"
 });
-var PublicEvalsGroup = class extends HttpApiGroup.make("evals").add(HttpApiEndpoint.post("list", "/evals.list").setPayload(ListEvalsRequest).addSuccess(EvalRunPage).annotate(OpenApi.Summary, "List eval runs").annotate(OpenApi.Description, "Newest first. Pass the `next` cursor from a response to read the page after it; a null `next` means there are no more.")).add(HttpApiEndpoint.post("start", "/evals.start").setPayload(PublicStartEvalRequest).addSuccess(StartedEval).annotate(OpenApi.Summary, "Start an eval run").annotate(OpenApi.Description, `Starts the grid and returns its id while trials continue in the background. ${PROFILE_HARNESS_RULE}`)).add(HttpApiEndpoint.post("artifact", "/evals.artifact").setPayload(EvalArtifactRequest).addSuccess(EvalArtifact)).add(HttpApiEndpoint.post("get", "/evals.get").setPayload(EvalRunRequest).addSuccess(EvalRun).annotate(OpenApi.Summary, "Get an eval run")).add(HttpApiEndpoint.post("cellHistory", "/evals.cellHistory").setPayload(EvalCellRequest).addSuccess(Schema.Array(EvalCellHistoryEntry)).annotate(OpenApi.Summary, "List a cell's history").annotate(OpenApi.Description, "Returns the 20 most recent results.")).add(HttpApiEndpoint.post("rerunCell", "/evals.rerunCell").setPayload(Schema.extend(EvalRunRequest, Schema.extend(EvalCellRequest, RerunCellRequest))).addSuccess(StartedEval).annotate(OpenApi.Summary, "Rerun one cell")).add(HttpApiEndpoint.post("models", "/evals.models").setPayload(EvalModelsRequest).addSuccess(ModelCatalogue).annotate(OpenApi.Summary, "List models available to the harness").annotate(OpenApi.Description, "The command harness has no catalogue of its own, so its list is empty: the model is whatever the profile's run command reads from ANPORD_MODEL.")).addError(BadRequest).addError(Conflict).addError(Forbidden).addError(NotFound).middleware(ApiKeyAuthentication).annotate(OpenApi.Title, "Evals").annotate(OpenApi.Description, "Run cases across harness, model, and sandbox combinations and compare the results with their baselines.") {};
+const ReportedTrial = Schema.Struct({
+	caseName: EvalCaseName,
+	events: Schema.Array(HarnessEvent),
+	ordinal: Schema.Int.pipe(Schema.nonNegative()),
+	outcome: TrialOutcome,
+	sandboxId: Schema.optional(Schema.NullOr(Schema.String)),
+	taskIndex: Schema.Int.pipe(Schema.nonNegative()),
+	usage: Schema.optional(Schema.NullOr(HarnessUsage))
+}).annotations({
+	description: "One trial a client ran and is reporting the result of.",
+	identifier: "ReportedTrial"
+});
+const ReportTrialRequest = Schema.Struct({
+	id: Schema.String,
+	trial: ReportedTrial
+}).annotations({
+	description: "A trial the caller ran, named by the run it belongs to.",
+	identifier: "ReportTrialRequest"
+});
+var PublicEvalsGroup = class extends HttpApiGroup.make("evals").add(HttpApiEndpoint.post("list", "/evals.list").setPayload(ListEvalsRequest).addSuccess(EvalRunPage).annotate(OpenApi.Summary, "List eval runs").annotate(OpenApi.Description, "Newest first. Pass the `next` cursor from a response to read the page after it; a null `next` means there are no more.")).add(HttpApiEndpoint.post("start", "/evals.start").setPayload(PublicStartEvalRequest).addSuccess(StartedEval).annotate(OpenApi.Summary, "Start an eval run").annotate(OpenApi.Description, `Starts the grid and returns its id while trials continue in the background. ${PROFILE_HARNESS_RULE}`)).add(HttpApiEndpoint.post("reportTrial", "/evals.reportTrial").setPayload(ReportTrialRequest).addSuccess(Schema.Void).annotate(OpenApi.Summary, "Report a trial run outside the platform").annotate(OpenApi.Description, "For a run started with executeLocally. The result is recorded as reported: it was produced somewhere the platform cannot inspect, so it is marked and kept out of baselines.")).add(HttpApiEndpoint.post("finishRun", "/evals.finishRun").setPayload(EvalRunRequest).addSuccess(EvalRun).annotate(OpenApi.Summary, "Settle a run whose trials the caller ran").annotate(OpenApi.Description, "Settles the run from the trials reported so far. A run left unsettled is swept like any other abandoned work.")).add(HttpApiEndpoint.post("artifact", "/evals.artifact").setPayload(EvalArtifactRequest).addSuccess(EvalArtifact)).add(HttpApiEndpoint.post("get", "/evals.get").setPayload(EvalRunRequest).addSuccess(EvalRun).annotate(OpenApi.Summary, "Get an eval run")).add(HttpApiEndpoint.post("subscription", "/evals.subscription").setPayload(EvalRunRequest).addSuccess(RunSubscription).annotate(OpenApi.Summary, "Watch an eval run as it moves").annotate(OpenApi.Description, "Returns a read-only token scoped to this run, for following it in real time. Short-lived: request another when it expires.")).add(HttpApiEndpoint.post("cellHistory", "/evals.cellHistory").setPayload(EvalCellRequest).addSuccess(Schema.Array(EvalCellHistoryEntry)).annotate(OpenApi.Summary, "List a cell's history").annotate(OpenApi.Description, "Returns the 20 most recent results.")).add(HttpApiEndpoint.post("rerunCell", "/evals.rerunCell").setPayload(Schema.extend(EvalRunRequest, Schema.extend(EvalCellRequest, RerunCellRequest))).addSuccess(StartedEval).annotate(OpenApi.Summary, "Rerun one cell")).add(HttpApiEndpoint.post("models", "/evals.models").setPayload(EvalModelsRequest).addSuccess(ModelCatalogue).annotate(OpenApi.Summary, "List models available to the harness").annotate(OpenApi.Description, "The command harness has no catalogue of its own, so its list is empty: the model is whatever the profile's run command reads from ANPORD_MODEL.")).addError(BadRequest).addError(Conflict).addError(Forbidden).addError(NotFound).middleware(ApiKeyAuthentication).annotate(OpenApi.Title, "Evals").annotate(OpenApi.Description, "Run cases across harness, model, and sandbox combinations and compare the results with their baselines.") {};
 //#endregion
-export { CredentialConnection as _, EnvName as a, CredentialValues as b, ProfilePath as c, EvalTrigger as d, ApiKeyAuthentication as f, NotFound as g, Forbidden as h, EvalValidator as i, profileFitsHarness as l, Conflict as m, PublicStartEvalRequest as n, HarnessProfile as o, BadRequest as p, EvalRun as r, PROFILE_LIMITS as s, PublicEvalsGroup as t, EvalSourceFiles as u, CredentialIntegration as v, CredentialScope as y };
+export { CredentialValues as S, Forbidden as _, EvalSandbox as a, CredentialIntegration as b, HarnessProfile as c, profileFitsHarness as d, EvalSourceFiles as f, Conflict as g, BadRequest as h, EvalRun as i, PROFILE_LIMITS as l, ApiKeyAuthentication as m, PublicStartEvalRequest as n, EvalValidator as o, EvalTrigger as p, HarnessEvent as r, EnvName as s, PublicEvalsGroup as t, ProfilePath as u, NotFound as v, CredentialScope as x, CredentialConnection as y };

package/dist/{evals-api-DuDr8UdV.cjs → evals-api-CNoLlXL5.cjs}

@@ -1,6 +1,6 @@
 const require_eval_judges = require("./eval-judges-1zF4PVqV.cjs");
-const require_eval_validations = require("./eval-validations-D0Cz6xSg.cjs");
-const require_eval_turns = require("./eval-turns-weR_7JIp.cjs");
+const require_eval_validations = require("./eval-validations-CZm7vVtQ.cjs");
+const require_eval_turns = require("./eval-turns-DcJ8_1qr.cjs");
 let effect = require("effect");
 let _effect_platform = require("@effect/platform");
 //#region ../schema/src/domain/credentials.ts
@@ -199,10 +199,11 @@ const PROFILE_HARNESS_RULE = "The command harness needs a profile with a run com
 //#endregion
 //#region ../schema/src/domain/evals.ts
 const EvalHarness = require_eval_judges.EvalHarness;
-const EvalSandbox = effect.Schema.Literal("daytona", "e2b", "upstash", "modal", "cloudflare", "vercel");
-EvalSandbox.literals;
+const EvalSandbox = effect.Schema.Literal("daytona", "e2b", "upstash", "modal", "cloudflare", "vercel", "local");
+const HOSTED_SANDBOXES = EvalSandbox.literals.filter((sandbox) => sandbox !== "local");
 const EvalTrialStatus = effect.Schema.Literal("queued", "running", "passed", "failed", "void");
 const EvalRunStatus = effect.Schema.Literal("running", "finished", "failed");
+const EvalExecutor = effect.Schema.Literal("client");
 const EvalSource = effect.Schema.Union(effect.Schema.Struct({ kind: effect.Schema.Literal("empty") }), effect.Schema.Struct({
 	kind: effect.Schema.Literal("repo"),
 	ref: effect.Schema.NullOr(effect.Schema.String),
@@ -307,7 +308,7 @@ effect.Schema.Struct({
 	tasks: effect.Schema.Array(EvalTaskRequest).pipe(effect.Schema.minItems(1), effect.Schema.maxItems(20)),
 	trials: effect.Schema.Int.pipe(effect.Schema.between(1, 10))
 });
-const OccurredAtMillis = effect.Schema.NullOr(effect.Schema.Number);
+const OccurredAtMillis$1 = effect.Schema.NullOr(effect.Schema.Number);
 const EvalUsage = effect.Schema.Struct({
 	cacheReadTokens: effect.Schema.Int,
 	cacheWriteTokens: effect.Schema.Int,
@@ -323,18 +324,18 @@ const EvalJournalEntry = effect.Schema.Union(effect.Schema.Struct({
 	_tag: effect.Schema.Literal("command"),
 	command: effect.Schema.String,
 	exitCode: effect.Schema.NullOr(effect.Schema.Int),
-	finishedAtMillis: OccurredAtMillis,
+	finishedAtMillis: OccurredAtMillis$1,
 	output: effect.Schema.String,
 	outputTruncated: effect.Schema.optional(effect.Schema.Boolean),
-	startedAtMillis: OccurredAtMillis
+	startedAtMillis: OccurredAtMillis$1
 }), effect.Schema.Struct({
 	_tag: effect.Schema.Literal("message"),
-	finishedAtMillis: OccurredAtMillis,
+	finishedAtMillis: OccurredAtMillis$1,
 	text: effect.Schema.String,
 	usage: effect.Schema.optional(effect.Schema.NullOr(EvalUsage))
 }), effect.Schema.Struct({
 	_tag: effect.Schema.Literal("toolCall"),
-	finishedAtMillis: OccurredAtMillis,
+	finishedAtMillis: OccurredAtMillis$1,
 	input: effect.Schema.optional(effect.Schema.String),
 	name: effect.Schema.String,
 	output: effect.Schema.optional(effect.Schema.String),
@@ -342,11 +343,11 @@ const EvalJournalEntry = effect.Schema.Union(effect.Schema.Struct({
 	outputTruncated: effect.Schema.optional(effect.Schema.Boolean),
 	inputTruncated: effect.Schema.optional(effect.Schema.Boolean),
 	errorTruncated: effect.Schema.optional(effect.Schema.Boolean),
-	startedAtMillis: effect.Schema.optional(OccurredAtMillis),
+	startedAtMillis: effect.Schema.optional(OccurredAtMillis$1),
 	status: effect.Schema.NullOr(effect.Schema.String)
 }), effect.Schema.Struct({
 	_tag: effect.Schema.Literal("fileChange"),
-	finishedAtMillis: OccurredAtMillis,
+	finishedAtMillis: OccurredAtMillis$1,
 	paths: effect.Schema.Array(effect.Schema.String)
 })).annotations({
 	description: "A normalized event recorded from the harness trajectory.",
@@ -493,6 +494,7 @@ const EvalRun = effect.Schema.Struct({
 	cases: effect.Schema.Array(effect.Schema.String),
 	cells: effect.Schema.Array(EvalCell),
 	costs: effect.Schema.NullOr(EvalCosts),
+	executedBy: effect.Schema.optionalWith(effect.Schema.NullOr(EvalExecutor), { default: () => null }),
 	failure: effect.Schema.NullOr(effect.Schema.String),
 	finishedAt: effect.Schema.NullOr(EvalTimestamp),
 	id: effect.Schema.String,
@@ -547,6 +549,14 @@ const EvalCellHistoryEntry = effect.Schema.Struct({
 	description: "A previous scored result for the same cell identity.",
 	identifier: "EvalCellHistoryEntry"
 });
+const RunSubscription = effect.Schema.Struct({
+	expiresAtMillis: effect.Schema.Number,
+	tag: effect.Schema.String,
+	token: effect.Schema.String
+}).annotations({
+	description: "A scoped, read-only token for watching one run in real time.",
+	identifier: "RunSubscription"
+});
 //#endregion
 //#region ../schema/src/domain/eval-playground.ts
 const PlaygroundCaseView = effect.Schema.Struct({
@@ -628,6 +638,75 @@ const ModelCatalogue = effect.Schema.Struct({
 	identifier: "ModelCatalogue"
 });
 //#endregion
+//#region ../schema/src/domain/harness-event.ts
+const OccurredAtMillis = effect.Schema.optional(effect.Schema.Number);
+const HarnessUsage = effect.Schema.Struct({
+	cacheReadTokens: effect.Schema.Int,
+	cacheWriteTokens: effect.Schema.Int,
+	costUsd: effect.Schema.optional(effect.Schema.Number),
+	inputTokens: effect.Schema.Int,
+	outputTokens: effect.Schema.Int,
+	totalTokens: effect.Schema.Int
+});
+const HarnessEvent = effect.Schema.Union(effect.Schema.Struct({
+	_tag: effect.Schema.Literal("Started"),
+	at: OccurredAtMillis,
+	model: effect.Schema.String,
+	sessionId: effect.Schema.String
+}), effect.Schema.Struct({
+	_tag: effect.Schema.Literal("Message"),
+	at: OccurredAtMillis,
+	role: effect.Schema.Literal("assistant", "user"),
+	text: effect.Schema.String,
+	usage: effect.Schema.optional(HarnessUsage)
+}), effect.Schema.Struct({
+	_tag: effect.Schema.Literal("Command"),
+	at: OccurredAtMillis,
+	command: effect.Schema.String,
+	exitCode: effect.Schema.NullOr(effect.Schema.Int),
+	output: effect.Schema.String,
+	startedAt: effect.Schema.optional(effect.Schema.Number)
+}), effect.Schema.Struct({
+	_tag: effect.Schema.Literal("FileChange"),
+	at: OccurredAtMillis,
+	paths: effect.Schema.Array(effect.Schema.String)
+}), effect.Schema.Struct({
+	_tag: effect.Schema.Literal("ToolCall"),
+	at: OccurredAtMillis,
+	callId: effect.Schema.NullOr(effect.Schema.String),
+	input: effect.Schema.String,
+	name: effect.Schema.String,
+	output: effect.Schema.optional(effect.Schema.String),
+	error: effect.Schema.optional(effect.Schema.String),
+	startedAt: effect.Schema.optional(effect.Schema.Number),
+	status: effect.Schema.NullOr(effect.Schema.String)
+}), effect.Schema.Struct({
+	_tag: effect.Schema.Literal("Finished"),
+	at: OccurredAtMillis,
+	reason: effect.Schema.String
+}));
+//#endregion
+//#region ../schema/src/domain/trial.ts
+const TrialStatus = effect.Schema.Literal("queued", "running", "passed", "failed", "void");
+effect.Schema.decodeUnknownOption(TrialStatus);
+const VerifyStepResult = effect.Schema.Struct({
+	command: effect.Schema.String,
+	exitCode: effect.Schema.Int
+});
+const TrialOutcome = effect.Schema.Struct({
+	artifacts: effect.Schema.optional(effect.Schema.Array(EvalArtifactMetadata)),
+	validations: effect.Schema.optional(require_eval_validations.EvalValidations),
+	judgments: effect.Schema.optional(effect.Schema.Array(require_eval_judges.EvalJudgment)),
+	commandCount: effect.Schema.Int,
+	exitCode: effect.Schema.Int,
+	modelMs: effect.Schema.Int,
+	passed: effect.Schema.Boolean,
+	sandboxMs: effect.Schema.Int,
+	status: TrialStatus,
+	verifySteps: effect.Schema.Array(VerifyStepResult),
+	voidFields: effect.Schema.Array(effect.Schema.String)
+});
+//#endregion
 //#region ../schema/src/public/evals-api.ts
 const EvalRunRequest = effect.Schema.Struct({ id: effect.Schema.String }).annotations({
 	description: "Select an eval run by id.",
@@ -644,7 +723,7 @@ const EvalModelsRequest = effect.Schema.Struct({
 	description: "Select a harness whose available models should be listed.",
 	identifier: "EvalModelsRequest"
 });
-const PublicEvalSandbox = EvalSandbox.annotations({
+const PublicEvalSandbox = effect.Schema.Literal(...HOSTED_SANDBOXES).annotations({
 	description: "The hosted sandbox a task runs in.",
 	identifier: "PublicEvalSandbox"
 });
@@ -680,6 +759,7 @@ const PublicEvalTask = effect.Schema.Struct({
 const PublicStartEvalRequest = effect.Schema.Struct({
 	trigger: effect.Schema.optional(EvalTrigger),
 	cases: effect.Schema.Array(PublicEvalCase).pipe(effect.Schema.minItems(1), effect.Schema.maxItems(100)),
+	executeLocally: effect.Schema.optional(effect.Schema.Boolean),
 	name: effect.Schema.optional(EvalName),
 	prompt: EvalPrompt,
 	tasks: effect.Schema.Array(PublicEvalTask).pipe(effect.Schema.minItems(1), effect.Schema.maxItems(20)),
@@ -688,7 +768,26 @@ const PublicStartEvalRequest = effect.Schema.Struct({
 	description: `Start a grid with at most 100 total case, task, and trial combinations.`,
 	identifier: "StartEvalRequest"
 });
-var PublicEvalsGroup = class extends _effect_platform.HttpApiGroup.make("evals").add(_effect_platform.HttpApiEndpoint.post("list", "/evals.list").setPayload(ListEvalsRequest).addSuccess(EvalRunPage).annotate(_effect_platform.OpenApi.Summary, "List eval runs").annotate(_effect_platform.OpenApi.Description, "Newest first. Pass the `next` cursor from a response to read the page after it; a null `next` means there are no more.")).add(_effect_platform.HttpApiEndpoint.post("start", "/evals.start").setPayload(PublicStartEvalRequest).addSuccess(StartedEval).annotate(_effect_platform.OpenApi.Summary, "Start an eval run").annotate(_effect_platform.OpenApi.Description, `Starts the grid and returns its id while trials continue in the background. ${PROFILE_HARNESS_RULE}`)).add(_effect_platform.HttpApiEndpoint.post("artifact", "/evals.artifact").setPayload(EvalArtifactRequest).addSuccess(EvalArtifact)).add(_effect_platform.HttpApiEndpoint.post("get", "/evals.get").setPayload(EvalRunRequest).addSuccess(EvalRun).annotate(_effect_platform.OpenApi.Summary, "Get an eval run")).add(_effect_platform.HttpApiEndpoint.post("cellHistory", "/evals.cellHistory").setPayload(EvalCellRequest).addSuccess(effect.Schema.Array(EvalCellHistoryEntry)).annotate(_effect_platform.OpenApi.Summary, "List a cell's history").annotate(_effect_platform.OpenApi.Description, "Returns the 20 most recent results.")).add(_effect_platform.HttpApiEndpoint.post("rerunCell", "/evals.rerunCell").setPayload(effect.Schema.extend(EvalRunRequest, effect.Schema.extend(EvalCellRequest, RerunCellRequest))).addSuccess(StartedEval).annotate(_effect_platform.OpenApi.Summary, "Rerun one cell")).add(_effect_platform.HttpApiEndpoint.post("models", "/evals.models").setPayload(EvalModelsRequest).addSuccess(ModelCatalogue).annotate(_effect_platform.OpenApi.Summary, "List models available to the harness").annotate(_effect_platform.OpenApi.Description, "The command harness has no catalogue of its own, so its list is empty: the model is whatever the profile's run command reads from ANPORD_MODEL.")).addError(BadRequest).addError(Conflict).addError(Forbidden).addError(NotFound).middleware(ApiKeyAuthentication).annotate(_effect_platform.OpenApi.Title, "Evals").annotate(_effect_platform.OpenApi.Description, "Run cases across harness, model, and sandbox combinations and compare the results with their baselines.") {};
+const ReportedTrial = effect.Schema.Struct({
+	caseName: EvalCaseName,
+	events: effect.Schema.Array(HarnessEvent),
+	ordinal: effect.Schema.Int.pipe(effect.Schema.nonNegative()),
+	outcome: TrialOutcome,
+	sandboxId: effect.Schema.optional(effect.Schema.NullOr(effect.Schema.String)),
+	taskIndex: effect.Schema.Int.pipe(effect.Schema.nonNegative()),
+	usage: effect.Schema.optional(effect.Schema.NullOr(HarnessUsage))
+}).annotations({
+	description: "One trial a client ran and is reporting the result of.",
+	identifier: "ReportedTrial"
+});
+const ReportTrialRequest = effect.Schema.Struct({
+	id: effect.Schema.String,
+	trial: ReportedTrial
+}).annotations({
+	description: "A trial the caller ran, named by the run it belongs to.",
+	identifier: "ReportTrialRequest"
+});
+var PublicEvalsGroup = class extends _effect_platform.HttpApiGroup.make("evals").add(_effect_platform.HttpApiEndpoint.post("list", "/evals.list").setPayload(ListEvalsRequest).addSuccess(EvalRunPage).annotate(_effect_platform.OpenApi.Summary, "List eval runs").annotate(_effect_platform.OpenApi.Description, "Newest first. Pass the `next` cursor from a response to read the page after it; a null `next` means there are no more.")).add(_effect_platform.HttpApiEndpoint.post("start", "/evals.start").setPayload(PublicStartEvalRequest).addSuccess(StartedEval).annotate(_effect_platform.OpenApi.Summary, "Start an eval run").annotate(_effect_platform.OpenApi.Description, `Starts the grid and returns its id while trials continue in the background. ${PROFILE_HARNESS_RULE}`)).add(_effect_platform.HttpApiEndpoint.post("reportTrial", "/evals.reportTrial").setPayload(ReportTrialRequest).addSuccess(effect.Schema.Void).annotate(_effect_platform.OpenApi.Summary, "Report a trial run outside the platform").annotate(_effect_platform.OpenApi.Description, "For a run started with executeLocally. The result is recorded as reported: it was produced somewhere the platform cannot inspect, so it is marked and kept out of baselines.")).add(_effect_platform.HttpApiEndpoint.post("finishRun", "/evals.finishRun").setPayload(EvalRunRequest).addSuccess(EvalRun).annotate(_effect_platform.OpenApi.Summary, "Settle a run whose trials the caller ran").annotate(_effect_platform.OpenApi.Description, "Settles the run from the trials reported so far. A run left unsettled is swept like any other abandoned work.")).add(_effect_platform.HttpApiEndpoint.post("artifact", "/evals.artifact").setPayload(EvalArtifactRequest).addSuccess(EvalArtifact)).add(_effect_platform.HttpApiEndpoint.post("get", "/evals.get").setPayload(EvalRunRequest).addSuccess(EvalRun).annotate(_effect_platform.OpenApi.Summary, "Get an eval run")).add(_effect_platform.HttpApiEndpoint.post("subscription", "/evals.subscription").setPayload(EvalRunRequest).addSuccess(RunSubscription).annotate(_effect_platform.OpenApi.Summary, "Watch an eval run as it moves").annotate(_effect_platform.OpenApi.Description, "Returns a read-only token scoped to this run, for following it in real time. Short-lived: request another when it expires.")).add(_effect_platform.HttpApiEndpoint.post("cellHistory", "/evals.cellHistory").setPayload(EvalCellRequest).addSuccess(effect.Schema.Array(EvalCellHistoryEntry)).annotate(_effect_platform.OpenApi.Summary, "List a cell's history").annotate(_effect_platform.OpenApi.Description, "Returns the 20 most recent results.")).add(_effect_platform.HttpApiEndpoint.post("rerunCell", "/evals.rerunCell").setPayload(effect.Schema.extend(EvalRunRequest, effect.Schema.extend(EvalCellRequest, RerunCellRequest))).addSuccess(StartedEval).annotate(_effect_platform.OpenApi.Summary, "Rerun one cell")).add(_effect_platform.HttpApiEndpoint.post("models", "/evals.models").setPayload(EvalModelsRequest).addSuccess(ModelCatalogue).annotate(_effect_platform.OpenApi.Summary, "List models available to the harness").annotate(_effect_platform.OpenApi.Description, "The command harness has no catalogue of its own, so its list is empty: the model is whatever the profile's run command reads from ANPORD_MODEL.")).addError(BadRequest).addError(Conflict).addError(Forbidden).addError(NotFound).middleware(ApiKeyAuthentication).annotate(_effect_platform.OpenApi.Title, "Evals").annotate(_effect_platform.OpenApi.Description, "Run cases across harness, model, and sandbox combinations and compare the results with their baselines.") {};
 //#endregion
 Object.defineProperty(exports, "ApiKeyAuthentication", {
 	enumerable: true,
@@ -744,6 +843,12 @@ Object.defineProperty(exports, "EvalRun", {
 		return EvalRun;
 	}
 });
+Object.defineProperty(exports, "EvalSandbox", {
+	enumerable: true,
+	get: function() {
+		return EvalSandbox;
+	}
+});
 Object.defineProperty(exports, "EvalSourceFiles", {
 	enumerable: true,
 	get: function() {
@@ -768,6 +873,12 @@ Object.defineProperty(exports, "Forbidden", {
 		return Forbidden;
 	}
 });
+Object.defineProperty(exports, "HarnessEvent", {
+	enumerable: true,
+	get: function() {
+		return HarnessEvent;
+	}
+});
 Object.defineProperty(exports, "HarnessProfile", {
 	enumerable: true,
 	get: function() {

package/dist/index.cjs

@@ -1,7 +1,7 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_client = require("./client-D4SBJEsG.cjs");
+const require_client = require("./client-BZDftC1x.cjs");
 const require_errors = require("./errors-BX1wry8K.cjs");
-const require_compiler = require("./compiler-DSN62KNe.cjs");
+const require_compiler = require("./compiler-Cy85vNbp.cjs");
 const require_source = require("./source-aW5nSo2h.cjs");
 let effect = require("effect");
 let _effect_platform = require("@effect/platform");

package/dist/index.d.cts

@@ -1,6 +1,6 @@
-import { a as layer, i as DEFAULT_BASE_URL, n as AnpordClient, o as make, r as ClientOptions, t as AnpordApi } from "./client-BD2VyKja.cjs";
-import { a as EvalTaskDefinition, c as Prepare, d as ProfileRef, f as Validator, i as EvalDefinition, l as PrepareContext, m as ValidatorResult, n as CommandResult, o as ExecOptions, p as ValidatorContext, r as EvalCaseDefinition, s as HarnessRef, t as CaseCache, u as PrepareValue } from "./types-CUVZ1A_F.cjs";
-import { _ as EvalVerdict, a as EvalDistribution, c as EvalRun, d as EvalSandbox, f as EvalSource, g as EvalUsage, h as EvalTrialStatus, i as EvalComparison, l as EvalRunStatus, m as EvalTrial, n as EvalCell, o as EvalHarness, p as EvalTask, r as EvalCellHistoryEntry, s as EvalJournalEntry, t as EvalArtifact, u as EvalRunSummary } from "./evals-WMHOpPxd.cjs";
+import { a as layer, i as DEFAULT_BASE_URL, n as AnpordClient, o as make, r as ClientOptions, t as AnpordApi } from "./client-CsGxLUZq.cjs";
+import { a as EvalTaskDefinition, c as Prepare, d as ProfileRef, f as Validator, i as EvalDefinition, l as PrepareContext, m as ValidatorResult, n as CommandResult, o as ExecOptions, p as ValidatorContext, r as EvalCaseDefinition, s as HarnessRef, t as CaseCache, u as PrepareValue } from "./types-Dvd_z5ge.cjs";
+import { _ as EvalVerdict, a as EvalDistribution, c as EvalRun, d as EvalSandbox, f as EvalSource, g as EvalUsage, h as EvalTrialStatus, i as EvalComparison, l as EvalRunStatus, m as EvalTrial, n as EvalCell, o as EvalHarness, p as EvalTask, r as EvalCellHistoryEntry, s as EvalJournalEntry, t as EvalArtifact, u as EvalRunSummary } from "./evals-Dw2Ek-mx.cjs";
 import { n as McpCallSchema, t as McpCall } from "./calls-DJGZJPrF.cjs";
 import { n as CliCallSchema, t as CliCall } from "./calls-6Z1EoEa-.cjs";
 import { empty, files, repo } from "./source.cjs";

package/dist/index.d.mts

@@ -1,6 +1,6 @@
-import { a as layer, i as DEFAULT_BASE_URL, n as AnpordClient, o as make, r as ClientOptions, t as AnpordApi } from "./client-ClWUDcPC.mjs";
-import { a as EvalTaskDefinition, c as Prepare, d as ProfileRef, f as Validator, i as EvalDefinition, l as PrepareContext, m as ValidatorResult, n as CommandResult, o as ExecOptions, p as ValidatorContext, r as EvalCaseDefinition, s as HarnessRef, t as CaseCache, u as PrepareValue } from "./types-Cw5k37Uk.mjs";
-import { _ as EvalVerdict, a as EvalDistribution, c as EvalRun, d as EvalSandbox, f as EvalSource, g as EvalUsage, h as EvalTrialStatus, i as EvalComparison, l as EvalRunStatus, m as EvalTrial, n as EvalCell, o as EvalHarness, p as EvalTask, r as EvalCellHistoryEntry, s as EvalJournalEntry, t as EvalArtifact, u as EvalRunSummary } from "./evals-WMHOpPxd.mjs";
+import { a as layer, i as DEFAULT_BASE_URL, n as AnpordClient, o as make, r as ClientOptions, t as AnpordApi } from "./client-C7iEhiDm.mjs";
+import { a as EvalTaskDefinition, c as Prepare, d as ProfileRef, f as Validator, i as EvalDefinition, l as PrepareContext, m as ValidatorResult, n as CommandResult, o as ExecOptions, p as ValidatorContext, r as EvalCaseDefinition, s as HarnessRef, t as CaseCache, u as PrepareValue } from "./types-D-h67MVJ.mjs";
+import { _ as EvalVerdict, a as EvalDistribution, c as EvalRun, d as EvalSandbox, f as EvalSource, g as EvalUsage, h as EvalTrialStatus, i as EvalComparison, l as EvalRunStatus, m as EvalTrial, n as EvalCell, o as EvalHarness, p as EvalTask, r as EvalCellHistoryEntry, s as EvalJournalEntry, t as EvalArtifact, u as EvalRunSummary } from "./evals-Dw2Ek-mx.mjs";
 import { n as McpCallSchema, t as McpCall } from "./calls-DJGZJPrF.mjs";
 import { n as CliCallSchema, t as CliCall } from "./calls-6Z1EoEa-.mjs";
 import { empty, files, repo } from "./source.mjs";

package/dist/index.mjs

@@ -1,6 +1,6 @@
-import { i as make, n as DEFAULT_BASE_URL, r as layer, t as AnpordApi } from "./client-DCSRP55Q.mjs";
+import { i as make, n as DEFAULT_BASE_URL, r as layer, t as AnpordApi } from "./client-D0l3kwWV.mjs";
 import { i as tokenMatcher, n as MissingApiKey, r as asAnpordError, t as AnpordError } from "./errors-B0YknR5V.mjs";
-import { a as suite, i as sourceUrlOf, t as compileDefinition } from "./compiler-BKchnKps.mjs";
+import { a as suite, i as sourceUrlOf, t as compileDefinition } from "./compiler-DSQRHvXS.mjs";
 import { n as files, r as repo, t as empty } from "./source-BP11pzVV.mjs";
 import { Cache, Cause, Clock, Context, Data, DateTime, Duration, Effect, Exit, Layer, ManagedRuntime, Option, Redacted, Schema } from "effect";
 import { FetchHttpClient } from "@effect/platform";

package/dist/{runtime-B5it0Siy.cjs → runtime-Bfs2j0jW.cjs}

@@ -1,5 +1,5 @@
-const require_eval_validations = require("./eval-validations-D0Cz6xSg.cjs");
-const require_api_mocks = require("./api-mocks-BjnewAya.cjs");
+const require_eval_validations = require("./eval-validations-CZm7vVtQ.cjs");
+const require_api_mocks = require("./api-mocks-CgiC7T9U.cjs");
 let effect = require("effect");
 let _effect_platform = require("@effect/platform");
 let _effect_platform_node_NodeFileSystem = require("@effect/platform-node/NodeFileSystem");

package/dist/{runtime-ChWJywJY.mjs → runtime-CYr8RbsW.mjs}

@@ -1,5 +1,5 @@
-import { a as validationCapture, i as unavailableValue } from "./eval-validations-CUG2PvR_.mjs";
-import { i as API_READY, n as API_MANIFEST, t as API_JOURNAL } from "./api-mocks-C3SsliJZ.mjs";
+import { o as unavailableValue, s as validationCapture } from "./eval-validations-DNO-qwkU.mjs";
+import { i as API_READY, n as API_MANIFEST, t as API_JOURNAL } from "./api-mocks-BYWvJp9v.mjs";
 import { Clock, Data, Deferred, Effect, Option, Ref, Schema } from "effect";
 import { FileSystem, HttpRouter, HttpServerRequest, HttpServerResponse } from "@effect/platform";
 import { layer } from "@effect/platform-node/NodeFileSystem";

package/dist/source.d.cts

@@ -1,4 +1,4 @@
-import { f as EvalSource } from "./evals-WMHOpPxd.cjs";
+import { f as EvalSource } from "./evals-Dw2Ek-mx.cjs";
 //#region src/evals/source.d.ts
 declare const repo: (spec: string) => EvalSource;
 declare const files: (contents: Readonly<Record<string, string>>) => EvalSource;

package/dist/source.d.mts

@@ -1,4 +1,4 @@
-import { f as EvalSource } from "./evals-WMHOpPxd.mjs";
+import { f as EvalSource } from "./evals-Dw2Ek-mx.mjs";
 //#region src/evals/source.d.ts
 declare const repo: (spec: string) => EvalSource;
 declare const files: (contents: Readonly<Record<string, string>>) => EvalSource;

package/dist/{types-Cw5k37Uk.d.mts → types-D-h67MVJ.d.mts}

@@ -1,7 +1,7 @@
 import "./errors-aGrd21jy.mjs";
-import { t as ApiCall } from "./api-mocks-xXKzUFc9.mjs";
+import { t as ApiCall } from "./api-mocks-BenVx0e8.mjs";
 import { a as EvalJudge, i as EvalUser, r as EvalTurn } from "./eval-turns-CTCoSuB6.mjs";
-import { f as EvalSource, o as EvalHarness } from "./evals-WMHOpPxd.mjs";
+import { f as EvalSource, o as EvalHarness } from "./evals-Dw2Ek-mx.mjs";
 import { t as McpCall } from "./calls-DJGZJPrF.mjs";
 import { n as McpServerDefinition } from "./define-HK0eeSNn.mjs";
 import { t as ApiDefinition } from "./define-NZvmQuIv.mjs";
@@ -200,6 +200,7 @@ declare const PublicStartEvalRequest: Schema.Struct<{
     variables: Schema.optional<Schema.Record$<typeof Schema.String, Schema.refine<string, typeof Schema.String>>>;
     verify: Schema.NullOr<Schema.refine<string, typeof Schema.String>>;
   }>>>>>;
+  executeLocally: Schema.optional<typeof Schema.Boolean>;
   name: Schema.optional<Schema.filter<Schema.filter<typeof Schema.String>>>;
   prompt: Schema.refine<string, typeof Schema.String>;
   tasks: Schema.filter<Schema.filter<Schema.Array$<Schema.refine<{
@@ -207,7 +208,6 @@ declare const PublicStartEvalRequest: Schema.Struct<{
     readonly model: string;
     readonly sandbox?: "daytona" | "e2b" | "upstash" | "modal" | "cloudflare" | "vercel" | undefined;
     readonly profile?: {
-      readonly run?: string | undefined;
       readonly name: string;
       readonly env?: {
         readonly [x: string]: string;
@@ -216,6 +216,7 @@ declare const PublicStartEvalRequest: Schema.Struct<{
         readonly [x: string]: string;
       };
       readonly install?: string | undefined;
+      readonly run?: string | undefined;
       readonly systemPrompt?: string | undefined;
     } | undefined;
   }, Schema.Struct<{
@@ -229,7 +230,7 @@ declare const PublicStartEvalRequest: Schema.Struct<{
       run: Schema.optional<typeof Schema.String>;
       systemPrompt: Schema.optional<typeof Schema.String>;
     }>>;
-    sandbox: Schema.optional<Schema.Literal<["daytona", "e2b", "upstash", "modal", "cloudflare", "vercel"]>>;
+    sandbox: Schema.optional<Schema.SchemaClass<"daytona" | "e2b" | "upstash" | "modal" | "cloudflare" | "vercel", "daytona" | "e2b" | "upstash" | "modal" | "cloudflare" | "vercel", never>>;
   }>>>>>;
   trials: Schema.filter<typeof Schema.Int>;
 }>;

package/dist/{types-CUVZ1A_F.d.cts → types-Dvd_z5ge.d.cts}

@@ -1,7 +1,7 @@
 import "./errors-aGrd21jy.cjs";
-import { t as ApiCall } from "./api-mocks-xXKzUFc9.cjs";
+import { t as ApiCall } from "./api-mocks-BenVx0e8.cjs";
 import { a as EvalJudge, i as EvalUser, r as EvalTurn } from "./eval-turns-CTCoSuB6.cjs";
-import { f as EvalSource, o as EvalHarness } from "./evals-WMHOpPxd.cjs";
+import { f as EvalSource, o as EvalHarness } from "./evals-Dw2Ek-mx.cjs";
 import { t as McpCall } from "./calls-DJGZJPrF.cjs";
 import { n as McpServerDefinition } from "./define-HK0eeSNn.cjs";
 import { t as ApiDefinition } from "./define-NZvmQuIv.cjs";
@@ -200,6 +200,7 @@ declare const PublicStartEvalRequest: Schema.Struct<{
     variables: Schema.optional<Schema.Record$<typeof Schema.String, Schema.refine<string, typeof Schema.String>>>;
     verify: Schema.NullOr<Schema.refine<string, typeof Schema.String>>;
   }>>>>>;
+  executeLocally: Schema.optional<typeof Schema.Boolean>;
   name: Schema.optional<Schema.filter<Schema.filter<typeof Schema.String>>>;
   prompt: Schema.refine<string, typeof Schema.String>;
   tasks: Schema.filter<Schema.filter<Schema.Array$<Schema.refine<{
@@ -207,7 +208,6 @@ declare const PublicStartEvalRequest: Schema.Struct<{
     readonly model: string;
     readonly sandbox?: "daytona" | "e2b" | "upstash" | "modal" | "cloudflare" | "vercel" | undefined;
     readonly profile?: {
-      readonly run?: string | undefined;
       readonly name: string;
       readonly env?: {
         readonly [x: string]: string;
@@ -216,6 +216,7 @@ declare const PublicStartEvalRequest: Schema.Struct<{
         readonly [x: string]: string;
       };
       readonly install?: string | undefined;
+      readonly run?: string | undefined;
       readonly systemPrompt?: string | undefined;
     } | undefined;
   }, Schema.Struct<{
@@ -229,7 +230,7 @@ declare const PublicStartEvalRequest: Schema.Struct<{
       run: Schema.optional<typeof Schema.String>;
       systemPrompt: Schema.optional<typeof Schema.String>;
     }>>;
-    sandbox: Schema.optional<Schema.Literal<["daytona", "e2b", "upstash", "modal", "cloudflare", "vercel"]>>;
+    sandbox: Schema.optional<Schema.SchemaClass<"daytona" | "e2b" | "upstash" | "modal" | "cloudflare" | "vercel", "daytona" | "e2b" | "upstash" | "modal" | "cloudflare" | "vercel", never>>;
   }>>>>>;
   trials: Schema.filter<typeof Schema.Int>;
 }>;

package/dist/validator-runtime.cjs

@@ -1,9 +1,9 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_eval_validations = require("./eval-validations-D0Cz6xSg.cjs");
+const require_eval_validations = require("./eval-validations-CZm7vVtQ.cjs");
 const require_api_context = require("./api-context.cjs");
 let node_fs_promises = require("node:fs/promises");
-let node_async_hooks = require("node:async_hooks");
 let node_child_process = require("node:child_process");
+let node_async_hooks = require("node:async_hooks");
 let node_util = require("node:util");
 //#region src/evals/validator-decode.ts
 const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);