anpord 0.1.19 → 0.1.21

package/dist/{evals-api-DZSr9rtP.cjs → evals-api-CNoLlXL5.cjs}

@@ -1,41 +1,8 @@
 const require_eval_judges = require("./eval-judges-1zF4PVqV.cjs");
-const require_eval_validations = require("./eval-validations-D0Cz6xSg.cjs");
-const require_eval_turns = require("./eval-turns-Bv8wRQ-b.cjs");
+const require_eval_validations = require("./eval-validations-CZm7vVtQ.cjs");
+const require_eval_turns = require("./eval-turns-DcJ8_1qr.cjs");
 let effect = require("effect");
 let _effect_platform = require("@effect/platform");
-//#region ../schema/src/domain/errors.ts
-var NotFound = class extends effect.Schema.TaggedError()("NotFound", { message: effect.Schema.String }, _effect_platform.HttpApiSchema.annotations({ status: 404 })) {};
-var Conflict = class extends effect.Schema.TaggedError()("Conflict", { message: effect.Schema.String }, _effect_platform.HttpApiSchema.annotations({ status: 409 })) {};
-var BadRequest = class extends effect.Schema.TaggedError()("BadRequest", { message: effect.Schema.String }, _effect_platform.HttpApiSchema.annotations({ status: 400 })) {};
-var Unauthorized = class extends effect.Schema.TaggedError()("Unauthorized", { message: effect.Schema.String }, _effect_platform.HttpApiSchema.annotations({ status: 401 })) {};
-var Forbidden = class extends effect.Schema.TaggedError()("Forbidden", { message: effect.Schema.String }, _effect_platform.HttpApiSchema.annotations({ status: 403 })) {};
-effect.Schema.TaggedError()("InternalError", { message: effect.Schema.String }, _effect_platform.HttpApiSchema.annotations({ status: 500 }));
-//#endregion
-//#region ../schema/src/domain/eval-limits.ts
-const PROMPT_LIMIT = 16384;
-const VERIFY_LIMIT = 8192;
-const NAME_LIMIT = 200;
-const VARIABLE_VALUE_LIMIT = 2048;
-const EvalPrompt = effect.Schema.String.pipe(effect.Schema.maxLength(PROMPT_LIMIT), effect.Schema.annotations({
-	description: "What the agent is asked to do.",
-	identifier: "EvalPrompt",
-	message: () => `A prompt must be at most ${PROMPT_LIMIT} characters, because it is quoted into one sandbox command line.`
-}));
-const EvalVerify = effect.Schema.String.pipe(effect.Schema.maxLength(VERIFY_LIMIT), effect.Schema.annotations({
-	description: "The shell command that decides whether a trial passed.",
-	identifier: "EvalVerify",
-	message: () => `A verifier must be at most ${VERIFY_LIMIT} characters, because it is quoted into one sandbox command line.`
-}));
-const EvalCaseName = effect.Schema.String.pipe(effect.Schema.maxLength(NAME_LIMIT), effect.Schema.annotations({
-	description: "What a case is called.",
-	identifier: "EvalCaseName",
-	message: () => `A case name must be at most ${NAME_LIMIT} characters.`
-}));
-const EvalVariableValue = effect.Schema.String.pipe(effect.Schema.maxLength(VARIABLE_VALUE_LIMIT), effect.Schema.annotations({
-	identifier: "EvalVariableValue",
-	message: () => `A variable value must be at most ${VARIABLE_VALUE_LIMIT} characters, because it is substituted into the prompt before it is quoted.`
-}));
-//#endregion
 //#region ../schema/src/domain/credentials.ts
 const CredentialScope = effect.Schema.Literal("organization", "personal");
 const CredentialStatus = effect.Schema.Literal("active", "invalid");
@@ -52,9 +19,9 @@ const CredentialAuthMethod = effect.Schema.Struct({
 	kind: effect.Schema.Literal("secret", "device", "env"),
 	label: effect.Schema.String
 });
-effect.Schema.Struct({
+const CredentialIntegration = effect.Schema.Struct({
 	authMethods: effect.Schema.Array(CredentialAuthMethod),
-	category: effect.Schema.Literal("harness", "judge", "sandbox"),
+	category: effect.Schema.Literal("harness", "model", "sandbox"),
 	id: effect.Schema.String,
 	label: effect.Schema.String
 });
@@ -65,7 +32,7 @@ effect.Schema.Struct({
 	description: "Who else in the organization has an integration connected.",
 	identifier: "IntegrationAwareness"
 });
-effect.Schema.Struct({
+const CredentialConnection = effect.Schema.Struct({
 	authMethodId: effect.Schema.String,
 	createdAt: effect.Schema.DateTimeUtc,
 	id: effect.Schema.String,
@@ -120,6 +87,57 @@ effect.Schema.Struct({
 	status: effect.Schema.Literal("pending", "complete", "failed", "expired")
 });
 //#endregion
+//#region ../schema/src/domain/errors.ts
+var NotFound = class extends effect.Schema.TaggedError()("NotFound", { message: effect.Schema.String }, _effect_platform.HttpApiSchema.annotations({ status: 404 })) {};
+var Conflict = class extends effect.Schema.TaggedError()("Conflict", { message: effect.Schema.String }, _effect_platform.HttpApiSchema.annotations({ status: 409 })) {};
+var BadRequest = class extends effect.Schema.TaggedError()("BadRequest", { message: effect.Schema.String }, _effect_platform.HttpApiSchema.annotations({ status: 400 })) {};
+var Unauthorized = class extends effect.Schema.TaggedError()("Unauthorized", { message: effect.Schema.String }, _effect_platform.HttpApiSchema.annotations({ status: 401 })) {};
+var Forbidden = class extends effect.Schema.TaggedError()("Forbidden", { message: effect.Schema.String }, _effect_platform.HttpApiSchema.annotations({ status: 403 })) {};
+effect.Schema.TaggedError()("InternalError", { message: effect.Schema.String }, _effect_platform.HttpApiSchema.annotations({ status: 500 }));
+//#endregion
+//#region ../schema/src/internal/authentication.ts
+var CurrentActor = class extends effect.Context.Tag("@anpord/schema/CurrentActor")() {};
+_effect_platform.HttpApiMiddleware.Tag()("@anpord/schema/Authentication", {
+	failure: Unauthorized,
+	provides: CurrentActor,
+	security: { session: _effect_platform.HttpApiSecurity.apiKey({
+		in: "cookie",
+		key: "anpord.session_token"
+	}) }
+});
+//#endregion
+//#region ../schema/src/public/authentication.ts
+var ApiKeyAuthentication = class extends _effect_platform.HttpApiMiddleware.Tag()("@anpord/schema/ApiKeyAuthentication", {
+	failure: Unauthorized,
+	provides: CurrentActor,
+	security: { bearer: _effect_platform.HttpApiSecurity.bearer }
+}) {};
+//#endregion
+//#region ../schema/src/domain/eval-limits.ts
+const PROMPT_LIMIT = 16384;
+const VERIFY_LIMIT = 8192;
+const NAME_LIMIT = 200;
+const VARIABLE_VALUE_LIMIT = 2048;
+const EvalPrompt = effect.Schema.String.pipe(effect.Schema.maxLength(PROMPT_LIMIT), effect.Schema.annotations({
+	description: "What the agent is asked to do.",
+	identifier: "EvalPrompt",
+	message: () => `A prompt must be at most ${PROMPT_LIMIT} characters, because it is quoted into one sandbox command line.`
+}));
+const EvalVerify = effect.Schema.String.pipe(effect.Schema.maxLength(VERIFY_LIMIT), effect.Schema.annotations({
+	description: "The shell command that decides whether a trial passed.",
+	identifier: "EvalVerify",
+	message: () => `A verifier must be at most ${VERIFY_LIMIT} characters, because it is quoted into one sandbox command line.`
+}));
+const EvalCaseName = effect.Schema.String.pipe(effect.Schema.maxLength(NAME_LIMIT), effect.Schema.annotations({
+	description: "What a case is called.",
+	identifier: "EvalCaseName",
+	message: () => `A case name must be at most ${NAME_LIMIT} characters.`
+}));
+const EvalVariableValue = effect.Schema.String.pipe(effect.Schema.maxLength(VARIABLE_VALUE_LIMIT), effect.Schema.annotations({
+	identifier: "EvalVariableValue",
+	message: () => `A variable value must be at most ${VARIABLE_VALUE_LIMIT} characters, because it is substituted into the prompt before it is quoted.`
+}));
+//#endregion
 //#region ../schema/src/domain/eval-trigger.ts
 const EvalTrigger = effect.Schema.Struct({
 	source: effect.Schema.Literal("dashboard", "api", "cli", "ci", "mcp"),
@@ -181,10 +199,11 @@ const PROFILE_HARNESS_RULE = "The command harness needs a profile with a run com
 //#endregion
 //#region ../schema/src/domain/evals.ts
 const EvalHarness = require_eval_judges.EvalHarness;
-const EvalSandbox = effect.Schema.Literal("daytona", "e2b", "upstash", "modal", "cloudflare", "vercel");
-EvalSandbox.literals;
+const EvalSandbox = effect.Schema.Literal("daytona", "e2b", "upstash", "modal", "cloudflare", "vercel", "local");
+const HOSTED_SANDBOXES = EvalSandbox.literals.filter((sandbox) => sandbox !== "local");
 const EvalTrialStatus = effect.Schema.Literal("queued", "running", "passed", "failed", "void");
 const EvalRunStatus = effect.Schema.Literal("running", "finished", "failed");
+const EvalExecutor = effect.Schema.Literal("client");
 const EvalSource = effect.Schema.Union(effect.Schema.Struct({ kind: effect.Schema.Literal("empty") }), effect.Schema.Struct({
 	kind: effect.Schema.Literal("repo"),
 	ref: effect.Schema.NullOr(effect.Schema.String),
@@ -289,7 +308,7 @@ effect.Schema.Struct({
 	tasks: effect.Schema.Array(EvalTaskRequest).pipe(effect.Schema.minItems(1), effect.Schema.maxItems(20)),
 	trials: effect.Schema.Int.pipe(effect.Schema.between(1, 10))
 });
-const OccurredAtMillis = effect.Schema.NullOr(effect.Schema.Number);
+const OccurredAtMillis$1 = effect.Schema.NullOr(effect.Schema.Number);
 const EvalUsage = effect.Schema.Struct({
 	cacheReadTokens: effect.Schema.Int,
 	cacheWriteTokens: effect.Schema.Int,
@@ -305,18 +324,18 @@ const EvalJournalEntry = effect.Schema.Union(effect.Schema.Struct({
 	_tag: effect.Schema.Literal("command"),
 	command: effect.Schema.String,
 	exitCode: effect.Schema.NullOr(effect.Schema.Int),
-	finishedAtMillis: OccurredAtMillis,
+	finishedAtMillis: OccurredAtMillis$1,
 	output: effect.Schema.String,
 	outputTruncated: effect.Schema.optional(effect.Schema.Boolean),
-	startedAtMillis: OccurredAtMillis
+	startedAtMillis: OccurredAtMillis$1
 }), effect.Schema.Struct({
 	_tag: effect.Schema.Literal("message"),
-	finishedAtMillis: OccurredAtMillis,
+	finishedAtMillis: OccurredAtMillis$1,
 	text: effect.Schema.String,
 	usage: effect.Schema.optional(effect.Schema.NullOr(EvalUsage))
 }), effect.Schema.Struct({
 	_tag: effect.Schema.Literal("toolCall"),
-	finishedAtMillis: OccurredAtMillis,
+	finishedAtMillis: OccurredAtMillis$1,
 	input: effect.Schema.optional(effect.Schema.String),
 	name: effect.Schema.String,
 	output: effect.Schema.optional(effect.Schema.String),
@@ -324,11 +343,11 @@ const EvalJournalEntry = effect.Schema.Union(effect.Schema.Struct({
 	outputTruncated: effect.Schema.optional(effect.Schema.Boolean),
 	inputTruncated: effect.Schema.optional(effect.Schema.Boolean),
 	errorTruncated: effect.Schema.optional(effect.Schema.Boolean),
-	startedAtMillis: effect.Schema.optional(OccurredAtMillis),
+	startedAtMillis: effect.Schema.optional(OccurredAtMillis$1),
 	status: effect.Schema.NullOr(effect.Schema.String)
 }), effect.Schema.Struct({
 	_tag: effect.Schema.Literal("fileChange"),
-	finishedAtMillis: OccurredAtMillis,
+	finishedAtMillis: OccurredAtMillis$1,
 	paths: effect.Schema.Array(effect.Schema.String)
 })).annotations({
 	description: "A normalized event recorded from the harness trajectory.",
@@ -475,6 +494,7 @@ const EvalRun = effect.Schema.Struct({
 	cases: effect.Schema.Array(effect.Schema.String),
 	cells: effect.Schema.Array(EvalCell),
 	costs: effect.Schema.NullOr(EvalCosts),
+	executedBy: effect.Schema.optionalWith(effect.Schema.NullOr(EvalExecutor), { default: () => null }),
 	failure: effect.Schema.NullOr(effect.Schema.String),
 	finishedAt: effect.Schema.NullOr(EvalTimestamp),
 	id: effect.Schema.String,
@@ -529,6 +549,14 @@ const EvalCellHistoryEntry = effect.Schema.Struct({
 	description: "A previous scored result for the same cell identity.",
 	identifier: "EvalCellHistoryEntry"
 });
+const RunSubscription = effect.Schema.Struct({
+	expiresAtMillis: effect.Schema.Number,
+	tag: effect.Schema.String,
+	token: effect.Schema.String
+}).annotations({
+	description: "A scoped, read-only token for watching one run in real time.",
+	identifier: "RunSubscription"
+});
 //#endregion
 //#region ../schema/src/domain/eval-playground.ts
 const PlaygroundCaseView = effect.Schema.Struct({
@@ -610,23 +638,74 @@ const ModelCatalogue = effect.Schema.Struct({
 	identifier: "ModelCatalogue"
 });
 //#endregion
-//#region ../schema/src/internal/authentication.ts
-var CurrentActor = class extends effect.Context.Tag("@anpord/schema/CurrentActor")() {};
-_effect_platform.HttpApiMiddleware.Tag()("@anpord/schema/Authentication", {
-	failure: Unauthorized,
-	provides: CurrentActor,
-	security: { session: _effect_platform.HttpApiSecurity.apiKey({
-		in: "cookie",
-		key: "anpord.session_token"
-	}) }
+//#region ../schema/src/domain/harness-event.ts
+const OccurredAtMillis = effect.Schema.optional(effect.Schema.Number);
+const HarnessUsage = effect.Schema.Struct({
+	cacheReadTokens: effect.Schema.Int,
+	cacheWriteTokens: effect.Schema.Int,
+	costUsd: effect.Schema.optional(effect.Schema.Number),
+	inputTokens: effect.Schema.Int,
+	outputTokens: effect.Schema.Int,
+	totalTokens: effect.Schema.Int
 });
+const HarnessEvent = effect.Schema.Union(effect.Schema.Struct({
+	_tag: effect.Schema.Literal("Started"),
+	at: OccurredAtMillis,
+	model: effect.Schema.String,
+	sessionId: effect.Schema.String
+}), effect.Schema.Struct({
+	_tag: effect.Schema.Literal("Message"),
+	at: OccurredAtMillis,
+	role: effect.Schema.Literal("assistant", "user"),
+	text: effect.Schema.String,
+	usage: effect.Schema.optional(HarnessUsage)
+}), effect.Schema.Struct({
+	_tag: effect.Schema.Literal("Command"),
+	at: OccurredAtMillis,
+	command: effect.Schema.String,
+	exitCode: effect.Schema.NullOr(effect.Schema.Int),
+	output: effect.Schema.String,
+	startedAt: effect.Schema.optional(effect.Schema.Number)
+}), effect.Schema.Struct({
+	_tag: effect.Schema.Literal("FileChange"),
+	at: OccurredAtMillis,
+	paths: effect.Schema.Array(effect.Schema.String)
+}), effect.Schema.Struct({
+	_tag: effect.Schema.Literal("ToolCall"),
+	at: OccurredAtMillis,
+	callId: effect.Schema.NullOr(effect.Schema.String),
+	input: effect.Schema.String,
+	name: effect.Schema.String,
+	output: effect.Schema.optional(effect.Schema.String),
+	error: effect.Schema.optional(effect.Schema.String),
+	startedAt: effect.Schema.optional(effect.Schema.Number),
+	status: effect.Schema.NullOr(effect.Schema.String)
+}), effect.Schema.Struct({
+	_tag: effect.Schema.Literal("Finished"),
+	at: OccurredAtMillis,
+	reason: effect.Schema.String
+}));
 //#endregion
-//#region ../schema/src/public/authentication.ts
-var ApiKeyAuthentication = class extends _effect_platform.HttpApiMiddleware.Tag()("@anpord/schema/ApiKeyAuthentication", {
-	failure: Unauthorized,
-	provides: CurrentActor,
-	security: { bearer: _effect_platform.HttpApiSecurity.bearer }
-}) {};
+//#region ../schema/src/domain/trial.ts
+const TrialStatus = effect.Schema.Literal("queued", "running", "passed", "failed", "void");
+effect.Schema.decodeUnknownOption(TrialStatus);
+const VerifyStepResult = effect.Schema.Struct({
+	command: effect.Schema.String,
+	exitCode: effect.Schema.Int
+});
+const TrialOutcome = effect.Schema.Struct({
+	artifacts: effect.Schema.optional(effect.Schema.Array(EvalArtifactMetadata)),
+	validations: effect.Schema.optional(require_eval_validations.EvalValidations),
+	judgments: effect.Schema.optional(effect.Schema.Array(require_eval_judges.EvalJudgment)),
+	commandCount: effect.Schema.Int,
+	exitCode: effect.Schema.Int,
+	modelMs: effect.Schema.Int,
+	passed: effect.Schema.Boolean,
+	sandboxMs: effect.Schema.Int,
+	status: TrialStatus,
+	verifySteps: effect.Schema.Array(VerifyStepResult),
+	voidFields: effect.Schema.Array(effect.Schema.String)
+});
 //#endregion
 //#region ../schema/src/public/evals-api.ts
 const EvalRunRequest = effect.Schema.Struct({ id: effect.Schema.String }).annotations({
@@ -644,7 +723,7 @@ const EvalModelsRequest = effect.Schema.Struct({
 	description: "Select a harness whose available models should be listed.",
 	identifier: "EvalModelsRequest"
 });
-const PublicEvalSandbox = EvalSandbox.annotations({
+const PublicEvalSandbox = effect.Schema.Literal(...HOSTED_SANDBOXES).annotations({
 	description: "The hosted sandbox a task runs in.",
 	identifier: "PublicEvalSandbox"
 });
@@ -680,6 +759,7 @@ const PublicEvalTask = effect.Schema.Struct({
 const PublicStartEvalRequest = effect.Schema.Struct({
 	trigger: effect.Schema.optional(EvalTrigger),
 	cases: effect.Schema.Array(PublicEvalCase).pipe(effect.Schema.minItems(1), effect.Schema.maxItems(100)),
+	executeLocally: effect.Schema.optional(effect.Schema.Boolean),
 	name: effect.Schema.optional(EvalName),
 	prompt: EvalPrompt,
 	tasks: effect.Schema.Array(PublicEvalTask).pipe(effect.Schema.minItems(1), effect.Schema.maxItems(20)),
@@ -688,7 +768,26 @@ const PublicStartEvalRequest = effect.Schema.Struct({
 	description: `Start a grid with at most 100 total case, task, and trial combinations.`,
 	identifier: "StartEvalRequest"
 });
-var PublicEvalsGroup = class extends _effect_platform.HttpApiGroup.make("evals").add(_effect_platform.HttpApiEndpoint.post("list", "/evals.list").setPayload(ListEvalsRequest).addSuccess(EvalRunPage).annotate(_effect_platform.OpenApi.Summary, "List eval runs").annotate(_effect_platform.OpenApi.Description, "Newest first. Pass the `next` cursor from a response to read the page after it; a null `next` means there are no more.")).add(_effect_platform.HttpApiEndpoint.post("start", "/evals.start").setPayload(PublicStartEvalRequest).addSuccess(StartedEval).annotate(_effect_platform.OpenApi.Summary, "Start an eval run").annotate(_effect_platform.OpenApi.Description, `Starts the grid and returns its id while trials continue in the background. ${PROFILE_HARNESS_RULE}`)).add(_effect_platform.HttpApiEndpoint.post("artifact", "/evals.artifact").setPayload(EvalArtifactRequest).addSuccess(EvalArtifact)).add(_effect_platform.HttpApiEndpoint.post("get", "/evals.get").setPayload(EvalRunRequest).addSuccess(EvalRun).annotate(_effect_platform.OpenApi.Summary, "Get an eval run")).add(_effect_platform.HttpApiEndpoint.post("cellHistory", "/evals.cellHistory").setPayload(EvalCellRequest).addSuccess(effect.Schema.Array(EvalCellHistoryEntry)).annotate(_effect_platform.OpenApi.Summary, "List a cell's history").annotate(_effect_platform.OpenApi.Description, "Returns the 20 most recent results.")).add(_effect_platform.HttpApiEndpoint.post("rerunCell", "/evals.rerunCell").setPayload(effect.Schema.extend(EvalRunRequest, effect.Schema.extend(EvalCellRequest, RerunCellRequest))).addSuccess(StartedEval).annotate(_effect_platform.OpenApi.Summary, "Rerun one cell")).add(_effect_platform.HttpApiEndpoint.post("models", "/evals.models").setPayload(EvalModelsRequest).addSuccess(ModelCatalogue).annotate(_effect_platform.OpenApi.Summary, "List models available to the harness").annotate(_effect_platform.OpenApi.Description, "The command harness has no catalogue of its own, so its list is empty: the model is whatever the profile's run command reads from ANPORD_MODEL.")).addError(BadRequest).addError(Conflict).addError(Forbidden).addError(NotFound).middleware(ApiKeyAuthentication).annotate(_effect_platform.OpenApi.Title, "Evals").annotate(_effect_platform.OpenApi.Description, "Run cases across harness, model, and sandbox combinations and compare the results with their baselines.") {};
+const ReportedTrial = effect.Schema.Struct({
+	caseName: EvalCaseName,
+	events: effect.Schema.Array(HarnessEvent),
+	ordinal: effect.Schema.Int.pipe(effect.Schema.nonNegative()),
+	outcome: TrialOutcome,
+	sandboxId: effect.Schema.optional(effect.Schema.NullOr(effect.Schema.String)),
+	taskIndex: effect.Schema.Int.pipe(effect.Schema.nonNegative()),
+	usage: effect.Schema.optional(effect.Schema.NullOr(HarnessUsage))
+}).annotations({
+	description: "One trial a client ran and is reporting the result of.",
+	identifier: "ReportedTrial"
+});
+const ReportTrialRequest = effect.Schema.Struct({
+	id: effect.Schema.String,
+	trial: ReportedTrial
+}).annotations({
+	description: "A trial the caller ran, named by the run it belongs to.",
+	identifier: "ReportTrialRequest"
+});
+var PublicEvalsGroup = class extends _effect_platform.HttpApiGroup.make("evals").add(_effect_platform.HttpApiEndpoint.post("list", "/evals.list").setPayload(ListEvalsRequest).addSuccess(EvalRunPage).annotate(_effect_platform.OpenApi.Summary, "List eval runs").annotate(_effect_platform.OpenApi.Description, "Newest first. Pass the `next` cursor from a response to read the page after it; a null `next` means there are no more.")).add(_effect_platform.HttpApiEndpoint.post("start", "/evals.start").setPayload(PublicStartEvalRequest).addSuccess(StartedEval).annotate(_effect_platform.OpenApi.Summary, "Start an eval run").annotate(_effect_platform.OpenApi.Description, `Starts the grid and returns its id while trials continue in the background. ${PROFILE_HARNESS_RULE}`)).add(_effect_platform.HttpApiEndpoint.post("reportTrial", "/evals.reportTrial").setPayload(ReportTrialRequest).addSuccess(effect.Schema.Void).annotate(_effect_platform.OpenApi.Summary, "Report a trial run outside the platform").annotate(_effect_platform.OpenApi.Description, "For a run started with executeLocally. The result is recorded as reported: it was produced somewhere the platform cannot inspect, so it is marked and kept out of baselines.")).add(_effect_platform.HttpApiEndpoint.post("finishRun", "/evals.finishRun").setPayload(EvalRunRequest).addSuccess(EvalRun).annotate(_effect_platform.OpenApi.Summary, "Settle a run whose trials the caller ran").annotate(_effect_platform.OpenApi.Description, "Settles the run from the trials reported so far. A run left unsettled is swept like any other abandoned work.")).add(_effect_platform.HttpApiEndpoint.post("artifact", "/evals.artifact").setPayload(EvalArtifactRequest).addSuccess(EvalArtifact)).add(_effect_platform.HttpApiEndpoint.post("get", "/evals.get").setPayload(EvalRunRequest).addSuccess(EvalRun).annotate(_effect_platform.OpenApi.Summary, "Get an eval run")).add(_effect_platform.HttpApiEndpoint.post("subscription", "/evals.subscription").setPayload(EvalRunRequest).addSuccess(RunSubscription).annotate(_effect_platform.OpenApi.Summary, "Watch an eval run as it moves").annotate(_effect_platform.OpenApi.Description, "Returns a read-only token scoped to this run, for following it in real time. Short-lived: request another when it expires.")).add(_effect_platform.HttpApiEndpoint.post("cellHistory", "/evals.cellHistory").setPayload(EvalCellRequest).addSuccess(effect.Schema.Array(EvalCellHistoryEntry)).annotate(_effect_platform.OpenApi.Summary, "List a cell's history").annotate(_effect_platform.OpenApi.Description, "Returns the 20 most recent results.")).add(_effect_platform.HttpApiEndpoint.post("rerunCell", "/evals.rerunCell").setPayload(effect.Schema.extend(EvalRunRequest, effect.Schema.extend(EvalCellRequest, RerunCellRequest))).addSuccess(StartedEval).annotate(_effect_platform.OpenApi.Summary, "Rerun one cell")).add(_effect_platform.HttpApiEndpoint.post("models", "/evals.models").setPayload(EvalModelsRequest).addSuccess(ModelCatalogue).annotate(_effect_platform.OpenApi.Summary, "List models available to the harness").annotate(_effect_platform.OpenApi.Description, "The command harness has no catalogue of its own, so its list is empty: the model is whatever the profile's run command reads from ANPORD_MODEL.")).addError(BadRequest).addError(Conflict).addError(Forbidden).addError(NotFound).middleware(ApiKeyAuthentication).annotate(_effect_platform.OpenApi.Title, "Evals").annotate(_effect_platform.OpenApi.Description, "Run cases across harness, model, and sandbox combinations and compare the results with their baselines.") {};
 //#endregion
 Object.defineProperty(exports, "ApiKeyAuthentication", {
 	enumerable: true,
@@ -708,6 +807,30 @@ Object.defineProperty(exports, "Conflict", {
 		return Conflict;
 	}
 });
+Object.defineProperty(exports, "CredentialConnection", {
+	enumerable: true,
+	get: function() {
+		return CredentialConnection;
+	}
+});
+Object.defineProperty(exports, "CredentialIntegration", {
+	enumerable: true,
+	get: function() {
+		return CredentialIntegration;
+	}
+});
+Object.defineProperty(exports, "CredentialScope", {
+	enumerable: true,
+	get: function() {
+		return CredentialScope;
+	}
+});
+Object.defineProperty(exports, "CredentialValues", {
+	enumerable: true,
+	get: function() {
+		return CredentialValues;
+	}
+});
 Object.defineProperty(exports, "EnvName", {
 	enumerable: true,
 	get: function() {
@@ -720,6 +843,12 @@ Object.defineProperty(exports, "EvalRun", {
 		return EvalRun;
 	}
 });
+Object.defineProperty(exports, "EvalSandbox", {
+	enumerable: true,
+	get: function() {
+		return EvalSandbox;
+	}
+});
 Object.defineProperty(exports, "EvalSourceFiles", {
 	enumerable: true,
 	get: function() {
@@ -744,6 +873,12 @@ Object.defineProperty(exports, "Forbidden", {
 		return Forbidden;
 	}
 });
+Object.defineProperty(exports, "HarnessEvent", {
+	enumerable: true,
+	get: function() {
+		return HarnessEvent;
+	}
+});
 Object.defineProperty(exports, "HarnessProfile", {
 	enumerable: true,
 	get: function() {

package/dist/index.cjs

@@ -1,7 +1,7 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_client = require("./client-COfGoWvI.cjs");
+const require_client = require("./client-BZDftC1x.cjs");
 const require_errors = require("./errors-BX1wry8K.cjs");
-const require_compiler = require("./compiler-CPpRRAdH.cjs");
+const require_compiler = require("./compiler-Cy85vNbp.cjs");
 const require_source = require("./source-aW5nSo2h.cjs");
 let effect = require("effect");
 let _effect_platform = require("@effect/platform");

package/dist/index.d.cts

@@ -1,6 +1,6 @@
-import { a as layer, i as DEFAULT_BASE_URL, n as AnpordClient, o as make, r as ClientOptions, t as AnpordApi } from "./client-OTrC6QXZ.cjs";
-import { a as EvalTaskDefinition, c as Prepare, d as ProfileRef, f as Validator, i as EvalDefinition, l as PrepareContext, m as ValidatorResult, n as CommandResult, o as ExecOptions, p as ValidatorContext, r as EvalCaseDefinition, s as HarnessRef, t as CaseCache, u as PrepareValue } from "./types-DCUo5on_.cjs";
-import { _ as EvalVerdict, a as EvalDistribution, c as EvalRun, d as EvalSandbox, f as EvalSource, g as EvalUsage, h as EvalTrialStatus, i as EvalComparison, l as EvalRunStatus, m as EvalTrial, n as EvalCell, o as EvalHarness, p as EvalTask, r as EvalCellHistoryEntry, s as EvalJournalEntry, t as EvalArtifact, u as EvalRunSummary } from "./evals-CSipRo7i.cjs";
+import { a as layer, i as DEFAULT_BASE_URL, n as AnpordClient, o as make, r as ClientOptions, t as AnpordApi } from "./client-CsGxLUZq.cjs";
+import { a as EvalTaskDefinition, c as Prepare, d as ProfileRef, f as Validator, i as EvalDefinition, l as PrepareContext, m as ValidatorResult, n as CommandResult, o as ExecOptions, p as ValidatorContext, r as EvalCaseDefinition, s as HarnessRef, t as CaseCache, u as PrepareValue } from "./types-Dvd_z5ge.cjs";
+import { _ as EvalVerdict, a as EvalDistribution, c as EvalRun, d as EvalSandbox, f as EvalSource, g as EvalUsage, h as EvalTrialStatus, i as EvalComparison, l as EvalRunStatus, m as EvalTrial, n as EvalCell, o as EvalHarness, p as EvalTask, r as EvalCellHistoryEntry, s as EvalJournalEntry, t as EvalArtifact, u as EvalRunSummary } from "./evals-Dw2Ek-mx.cjs";
 import { n as McpCallSchema, t as McpCall } from "./calls-DJGZJPrF.cjs";
 import { n as CliCallSchema, t as CliCall } from "./calls-6Z1EoEa-.cjs";
 import { empty, files, repo } from "./source.cjs";

package/dist/index.d.mts

@@ -1,6 +1,6 @@
-import { a as layer, i as DEFAULT_BASE_URL, n as AnpordClient, o as make, r as ClientOptions, t as AnpordApi } from "./client-BucAp4c0.mjs";
-import { a as EvalTaskDefinition, c as Prepare, d as ProfileRef, f as Validator, i as EvalDefinition, l as PrepareContext, m as ValidatorResult, n as CommandResult, o as ExecOptions, p as ValidatorContext, r as EvalCaseDefinition, s as HarnessRef, t as CaseCache, u as PrepareValue } from "./types-LW0tX4MF.mjs";
-import { _ as EvalVerdict, a as EvalDistribution, c as EvalRun, d as EvalSandbox, f as EvalSource, g as EvalUsage, h as EvalTrialStatus, i as EvalComparison, l as EvalRunStatus, m as EvalTrial, n as EvalCell, o as EvalHarness, p as EvalTask, r as EvalCellHistoryEntry, s as EvalJournalEntry, t as EvalArtifact, u as EvalRunSummary } from "./evals-CSipRo7i.mjs";
+import { a as layer, i as DEFAULT_BASE_URL, n as AnpordClient, o as make, r as ClientOptions, t as AnpordApi } from "./client-C7iEhiDm.mjs";
+import { a as EvalTaskDefinition, c as Prepare, d as ProfileRef, f as Validator, i as EvalDefinition, l as PrepareContext, m as ValidatorResult, n as CommandResult, o as ExecOptions, p as ValidatorContext, r as EvalCaseDefinition, s as HarnessRef, t as CaseCache, u as PrepareValue } from "./types-D-h67MVJ.mjs";
+import { _ as EvalVerdict, a as EvalDistribution, c as EvalRun, d as EvalSandbox, f as EvalSource, g as EvalUsage, h as EvalTrialStatus, i as EvalComparison, l as EvalRunStatus, m as EvalTrial, n as EvalCell, o as EvalHarness, p as EvalTask, r as EvalCellHistoryEntry, s as EvalJournalEntry, t as EvalArtifact, u as EvalRunSummary } from "./evals-Dw2Ek-mx.mjs";
 import { n as McpCallSchema, t as McpCall } from "./calls-DJGZJPrF.mjs";
 import { n as CliCallSchema, t as CliCall } from "./calls-6Z1EoEa-.mjs";
 import { empty, files, repo } from "./source.mjs";

package/dist/index.mjs

@@ -1,6 +1,6 @@
-import { i as make, n as DEFAULT_BASE_URL, r as layer, t as AnpordApi } from "./client-sWrfA0jZ.mjs";
+import { i as make, n as DEFAULT_BASE_URL, r as layer, t as AnpordApi } from "./client-D0l3kwWV.mjs";
 import { i as tokenMatcher, n as MissingApiKey, r as asAnpordError, t as AnpordError } from "./errors-B0YknR5V.mjs";
-import { a as suite, i as sourceUrlOf, t as compileDefinition } from "./compiler-q6rnLW2F.mjs";
+import { a as suite, i as sourceUrlOf, t as compileDefinition } from "./compiler-DSQRHvXS.mjs";
 import { n as files, r as repo, t as empty } from "./source-BP11pzVV.mjs";
 import { Cache, Cause, Clock, Context, Data, DateTime, Duration, Effect, Exit, Layer, ManagedRuntime, Option, Redacted, Schema } from "effect";
 import { FetchHttpClient } from "@effect/platform";

package/dist/{runtime-B5it0Siy.cjs → runtime-Bfs2j0jW.cjs}

@@ -1,5 +1,5 @@
-const require_eval_validations = require("./eval-validations-D0Cz6xSg.cjs");
-const require_api_mocks = require("./api-mocks-BjnewAya.cjs");
+const require_eval_validations = require("./eval-validations-CZm7vVtQ.cjs");
+const require_api_mocks = require("./api-mocks-CgiC7T9U.cjs");
 let effect = require("effect");
 let _effect_platform = require("@effect/platform");
 let _effect_platform_node_NodeFileSystem = require("@effect/platform-node/NodeFileSystem");

package/dist/{runtime-ChWJywJY.mjs → runtime-CYr8RbsW.mjs}

@@ -1,5 +1,5 @@
-import { a as validationCapture, i as unavailableValue } from "./eval-validations-CUG2PvR_.mjs";
-import { i as API_READY, n as API_MANIFEST, t as API_JOURNAL } from "./api-mocks-C3SsliJZ.mjs";
+import { o as unavailableValue, s as validationCapture } from "./eval-validations-DNO-qwkU.mjs";
+import { i as API_READY, n as API_MANIFEST, t as API_JOURNAL } from "./api-mocks-BYWvJp9v.mjs";
 import { Clock, Data, Deferred, Effect, Option, Ref, Schema } from "effect";
 import { FileSystem, HttpRouter, HttpServerRequest, HttpServerResponse } from "@effect/platform";
 import { layer } from "@effect/platform-node/NodeFileSystem";

package/dist/source.d.cts

@@ -1,4 +1,4 @@
-import { f as EvalSource } from "./evals-CSipRo7i.cjs";
+import { f as EvalSource } from "./evals-Dw2Ek-mx.cjs";
 //#region src/evals/source.d.ts
 declare const repo: (spec: string) => EvalSource;
 declare const files: (contents: Readonly<Record<string, string>>) => EvalSource;

package/dist/source.d.mts

@@ -1,4 +1,4 @@
-import { f as EvalSource } from "./evals-CSipRo7i.mjs";
+import { f as EvalSource } from "./evals-Dw2Ek-mx.mjs";
 //#region src/evals/source.d.ts
 declare const repo: (spec: string) => EvalSource;
 declare const files: (contents: Readonly<Record<string, string>>) => EvalSource;

package/dist/{types-LW0tX4MF.d.mts → types-D-h67MVJ.d.mts}

@@ -1,7 +1,7 @@
 import "./errors-aGrd21jy.mjs";
-import { t as ApiCall } from "./api-mocks-CAQFqUgY.mjs";
+import { t as ApiCall } from "./api-mocks-BenVx0e8.mjs";
 import { a as EvalJudge, i as EvalUser, r as EvalTurn } from "./eval-turns-CTCoSuB6.mjs";
-import { f as EvalSource, o as EvalHarness } from "./evals-CSipRo7i.mjs";
+import { f as EvalSource, o as EvalHarness } from "./evals-Dw2Ek-mx.mjs";
 import { t as McpCall } from "./calls-DJGZJPrF.mjs";
 import { n as McpServerDefinition } from "./define-HK0eeSNn.mjs";
 import { t as ApiDefinition } from "./define-NZvmQuIv.mjs";
@@ -20,8 +20,8 @@ declare const PublicStartEvalRequest: Schema.Struct<{
     readonly source?: {
       readonly kind: "empty";
     } | {
-      readonly url: string;
       readonly kind: "repo";
+      readonly url: string;
       readonly ref: string | null;
     } | {
       readonly kind: "files";
@@ -76,10 +76,10 @@ declare const PublicStartEvalRequest: Schema.Struct<{
         }[] | undefined;
       }[];
       readonly judges: readonly ({
-        readonly harness?: undefined;
-        readonly model: string;
         readonly name: string;
         readonly kind: "judge";
+        readonly harness?: undefined;
+        readonly model: string;
         readonly prompt: string;
         readonly provider: "openai";
         readonly expected?: string | undefined;
@@ -89,10 +89,10 @@ declare const PublicStartEvalRequest: Schema.Struct<{
         readonly threshold: number;
         readonly timeoutMs: number;
       } | {
-        readonly harness: "cursor" | "codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen";
-        readonly model: string;
         readonly name: string;
         readonly kind: "judge";
+        readonly harness: "cursor" | "codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen";
+        readonly model: string;
         readonly prompt: string;
         readonly provider?: undefined;
         readonly expected?: string | undefined;
@@ -200,24 +200,25 @@ declare const PublicStartEvalRequest: Schema.Struct<{
     variables: Schema.optional<Schema.Record$<typeof Schema.String, Schema.refine<string, typeof Schema.String>>>;
     verify: Schema.NullOr<Schema.refine<string, typeof Schema.String>>;
   }>>>>>;
+  executeLocally: Schema.optional<typeof Schema.Boolean>;
   name: Schema.optional<Schema.filter<Schema.filter<typeof Schema.String>>>;
   prompt: Schema.refine<string, typeof Schema.String>;
   tasks: Schema.filter<Schema.filter<Schema.Array$<Schema.refine<{
     readonly harness: "cursor" | "codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen" | "command";
     readonly model: string;
+    readonly sandbox?: "daytona" | "e2b" | "upstash" | "modal" | "cloudflare" | "vercel" | undefined;
     readonly profile?: {
-      readonly run?: string | undefined;
       readonly name: string;
-      readonly files: {
-        readonly [x: string]: string;
-      };
       readonly env?: {
         readonly [x: string]: string;
       } | undefined;
+      readonly files: {
+        readonly [x: string]: string;
+      };
       readonly install?: string | undefined;
+      readonly run?: string | undefined;
       readonly systemPrompt?: string | undefined;
     } | undefined;
-    readonly sandbox?: "daytona" | "e2b" | "upstash" | "modal" | "cloudflare" | "vercel" | undefined;
   }, Schema.Struct<{
     harness: Schema.Literal<["codex", "opencode", "pi", "fx", "claude", "gemini", "qwen", "cursor", "command"]>;
     model: Schema.filter<typeof Schema.String>;
@@ -229,7 +230,7 @@ declare const PublicStartEvalRequest: Schema.Struct<{
       run: Schema.optional<typeof Schema.String>;
       systemPrompt: Schema.optional<typeof Schema.String>;
     }>>;
-    sandbox: Schema.optional<Schema.Literal<["daytona", "e2b", "upstash", "modal", "cloudflare", "vercel"]>>;
+    sandbox: Schema.optional<Schema.SchemaClass<"daytona" | "e2b" | "upstash" | "modal" | "cloudflare" | "vercel", "daytona" | "e2b" | "upstash" | "modal" | "cloudflare" | "vercel", never>>;
   }>>>>>;
   trials: Schema.filter<typeof Schema.Int>;
 }>;