anpord 0.1.19 → 0.1.21

package/dist/{evals-api-u98TxdqN.mjs → evals-api-CK3F1KoI.mjs}

@@ -1,41 +1,8 @@
 import { n as EvalJudgment, r as EvalHarness$1, t as EvalJudge } from "./eval-judges-Bee_ABJt.mjs";
-import { t as EvalValidations } from "./eval-validations-CUG2PvR_.mjs";
-import { r as EvalUser } from "./eval-turns-hf6tbL-E.mjs";
+import { n as EvalValidations } from "./eval-validations-DNO-qwkU.mjs";
+import { i as EvalUser } from "./eval-turns-BC-GMHD-.mjs";
 import { Context, Schema } from "effect";
 import { HttpApiEndpoint, HttpApiGroup, HttpApiMiddleware, HttpApiSchema, HttpApiSecurity, OpenApi } from "@effect/platform";
-//#region ../schema/src/domain/errors.ts
-var NotFound = class extends Schema.TaggedError()("NotFound", { message: Schema.String }, HttpApiSchema.annotations({ status: 404 })) {};
-var Conflict = class extends Schema.TaggedError()("Conflict", { message: Schema.String }, HttpApiSchema.annotations({ status: 409 })) {};
-var BadRequest = class extends Schema.TaggedError()("BadRequest", { message: Schema.String }, HttpApiSchema.annotations({ status: 400 })) {};
-var Unauthorized = class extends Schema.TaggedError()("Unauthorized", { message: Schema.String }, HttpApiSchema.annotations({ status: 401 })) {};
-var Forbidden = class extends Schema.TaggedError()("Forbidden", { message: Schema.String }, HttpApiSchema.annotations({ status: 403 })) {};
-Schema.TaggedError()("InternalError", { message: Schema.String }, HttpApiSchema.annotations({ status: 500 }));
-//#endregion
-//#region ../schema/src/domain/eval-limits.ts
-const PROMPT_LIMIT = 16384;
-const VERIFY_LIMIT = 8192;
-const NAME_LIMIT = 200;
-const VARIABLE_VALUE_LIMIT = 2048;
-const EvalPrompt = Schema.String.pipe(Schema.maxLength(PROMPT_LIMIT), Schema.annotations({
-	description: "What the agent is asked to do.",
-	identifier: "EvalPrompt",
-	message: () => `A prompt must be at most ${PROMPT_LIMIT} characters, because it is quoted into one sandbox command line.`
-}));
-const EvalVerify = Schema.String.pipe(Schema.maxLength(VERIFY_LIMIT), Schema.annotations({
-	description: "The shell command that decides whether a trial passed.",
-	identifier: "EvalVerify",
-	message: () => `A verifier must be at most ${VERIFY_LIMIT} characters, because it is quoted into one sandbox command line.`
-}));
-const EvalCaseName = Schema.String.pipe(Schema.maxLength(NAME_LIMIT), Schema.annotations({
-	description: "What a case is called.",
-	identifier: "EvalCaseName",
-	message: () => `A case name must be at most ${NAME_LIMIT} characters.`
-}));
-const EvalVariableValue = Schema.String.pipe(Schema.maxLength(VARIABLE_VALUE_LIMIT), Schema.annotations({
-	identifier: "EvalVariableValue",
-	message: () => `A variable value must be at most ${VARIABLE_VALUE_LIMIT} characters, because it is substituted into the prompt before it is quoted.`
-}));
-//#endregion
 //#region ../schema/src/domain/credentials.ts
 const CredentialScope = Schema.Literal("organization", "personal");
 const CredentialStatus = Schema.Literal("active", "invalid");
@@ -52,9 +19,9 @@ const CredentialAuthMethod = Schema.Struct({
 	kind: Schema.Literal("secret", "device", "env"),
 	label: Schema.String
 });
-Schema.Struct({
+const CredentialIntegration = Schema.Struct({
 	authMethods: Schema.Array(CredentialAuthMethod),
-	category: Schema.Literal("harness", "judge", "sandbox"),
+	category: Schema.Literal("harness", "model", "sandbox"),
 	id: Schema.String,
 	label: Schema.String
 });
@@ -65,7 +32,7 @@ Schema.Struct({
 	description: "Who else in the organization has an integration connected.",
 	identifier: "IntegrationAwareness"
 });
-Schema.Struct({
+const CredentialConnection = Schema.Struct({
 	authMethodId: Schema.String,
 	createdAt: Schema.DateTimeUtc,
 	id: Schema.String,
@@ -120,6 +87,57 @@ Schema.Struct({
 	status: Schema.Literal("pending", "complete", "failed", "expired")
 });
 //#endregion
+//#region ../schema/src/domain/errors.ts
+var NotFound = class extends Schema.TaggedError()("NotFound", { message: Schema.String }, HttpApiSchema.annotations({ status: 404 })) {};
+var Conflict = class extends Schema.TaggedError()("Conflict", { message: Schema.String }, HttpApiSchema.annotations({ status: 409 })) {};
+var BadRequest = class extends Schema.TaggedError()("BadRequest", { message: Schema.String }, HttpApiSchema.annotations({ status: 400 })) {};
+var Unauthorized = class extends Schema.TaggedError()("Unauthorized", { message: Schema.String }, HttpApiSchema.annotations({ status: 401 })) {};
+var Forbidden = class extends Schema.TaggedError()("Forbidden", { message: Schema.String }, HttpApiSchema.annotations({ status: 403 })) {};
+Schema.TaggedError()("InternalError", { message: Schema.String }, HttpApiSchema.annotations({ status: 500 }));
+//#endregion
+//#region ../schema/src/internal/authentication.ts
+var CurrentActor = class extends Context.Tag("@anpord/schema/CurrentActor")() {};
+HttpApiMiddleware.Tag()("@anpord/schema/Authentication", {
+	failure: Unauthorized,
+	provides: CurrentActor,
+	security: { session: HttpApiSecurity.apiKey({
+		in: "cookie",
+		key: "anpord.session_token"
+	}) }
+});
+//#endregion
+//#region ../schema/src/public/authentication.ts
+var ApiKeyAuthentication = class extends HttpApiMiddleware.Tag()("@anpord/schema/ApiKeyAuthentication", {
+	failure: Unauthorized,
+	provides: CurrentActor,
+	security: { bearer: HttpApiSecurity.bearer }
+}) {};
+//#endregion
+//#region ../schema/src/domain/eval-limits.ts
+const PROMPT_LIMIT = 16384;
+const VERIFY_LIMIT = 8192;
+const NAME_LIMIT = 200;
+const VARIABLE_VALUE_LIMIT = 2048;
+const EvalPrompt = Schema.String.pipe(Schema.maxLength(PROMPT_LIMIT), Schema.annotations({
+	description: "What the agent is asked to do.",
+	identifier: "EvalPrompt",
+	message: () => `A prompt must be at most ${PROMPT_LIMIT} characters, because it is quoted into one sandbox command line.`
+}));
+const EvalVerify = Schema.String.pipe(Schema.maxLength(VERIFY_LIMIT), Schema.annotations({
+	description: "The shell command that decides whether a trial passed.",
+	identifier: "EvalVerify",
+	message: () => `A verifier must be at most ${VERIFY_LIMIT} characters, because it is quoted into one sandbox command line.`
+}));
+const EvalCaseName = Schema.String.pipe(Schema.maxLength(NAME_LIMIT), Schema.annotations({
+	description: "What a case is called.",
+	identifier: "EvalCaseName",
+	message: () => `A case name must be at most ${NAME_LIMIT} characters.`
+}));
+const EvalVariableValue = Schema.String.pipe(Schema.maxLength(VARIABLE_VALUE_LIMIT), Schema.annotations({
+	identifier: "EvalVariableValue",
+	message: () => `A variable value must be at most ${VARIABLE_VALUE_LIMIT} characters, because it is substituted into the prompt before it is quoted.`
+}));
+//#endregion
 //#region ../schema/src/domain/eval-trigger.ts
 const EvalTrigger = Schema.Struct({
 	source: Schema.Literal("dashboard", "api", "cli", "ci", "mcp"),
@@ -181,10 +199,11 @@ const PROFILE_HARNESS_RULE = "The command harness needs a profile with a run com
 //#endregion
 //#region ../schema/src/domain/evals.ts
 const EvalHarness = EvalHarness$1;
-const EvalSandbox = Schema.Literal("daytona", "e2b", "upstash", "modal", "cloudflare", "vercel");
-EvalSandbox.literals;
+const EvalSandbox = Schema.Literal("daytona", "e2b", "upstash", "modal", "cloudflare", "vercel", "local");
+const HOSTED_SANDBOXES = EvalSandbox.literals.filter((sandbox) => sandbox !== "local");
 const EvalTrialStatus = Schema.Literal("queued", "running", "passed", "failed", "void");
 const EvalRunStatus = Schema.Literal("running", "finished", "failed");
+const EvalExecutor = Schema.Literal("client");
 const EvalSource = Schema.Union(Schema.Struct({ kind: Schema.Literal("empty") }), Schema.Struct({
 	kind: Schema.Literal("repo"),
 	ref: Schema.NullOr(Schema.String),
@@ -289,7 +308,7 @@ Schema.Struct({
 	tasks: Schema.Array(EvalTaskRequest).pipe(Schema.minItems(1), Schema.maxItems(20)),
 	trials: Schema.Int.pipe(Schema.between(1, 10))
 });
-const OccurredAtMillis = Schema.NullOr(Schema.Number);
+const OccurredAtMillis$1 = Schema.NullOr(Schema.Number);
 const EvalUsage = Schema.Struct({
 	cacheReadTokens: Schema.Int,
 	cacheWriteTokens: Schema.Int,
@@ -305,18 +324,18 @@ const EvalJournalEntry = Schema.Union(Schema.Struct({
 	_tag: Schema.Literal("command"),
 	command: Schema.String,
 	exitCode: Schema.NullOr(Schema.Int),
-	finishedAtMillis: OccurredAtMillis,
+	finishedAtMillis: OccurredAtMillis$1,
 	output: Schema.String,
 	outputTruncated: Schema.optional(Schema.Boolean),
-	startedAtMillis: OccurredAtMillis
+	startedAtMillis: OccurredAtMillis$1
 }), Schema.Struct({
 	_tag: Schema.Literal("message"),
-	finishedAtMillis: OccurredAtMillis,
+	finishedAtMillis: OccurredAtMillis$1,
 	text: Schema.String,
 	usage: Schema.optional(Schema.NullOr(EvalUsage))
 }), Schema.Struct({
 	_tag: Schema.Literal("toolCall"),
-	finishedAtMillis: OccurredAtMillis,
+	finishedAtMillis: OccurredAtMillis$1,
 	input: Schema.optional(Schema.String),
 	name: Schema.String,
 	output: Schema.optional(Schema.String),
@@ -324,11 +343,11 @@ const EvalJournalEntry = Schema.Union(Schema.Struct({
 	outputTruncated: Schema.optional(Schema.Boolean),
 	inputTruncated: Schema.optional(Schema.Boolean),
 	errorTruncated: Schema.optional(Schema.Boolean),
-	startedAtMillis: Schema.optional(OccurredAtMillis),
+	startedAtMillis: Schema.optional(OccurredAtMillis$1),
 	status: Schema.NullOr(Schema.String)
 }), Schema.Struct({
 	_tag: Schema.Literal("fileChange"),
-	finishedAtMillis: OccurredAtMillis,
+	finishedAtMillis: OccurredAtMillis$1,
 	paths: Schema.Array(Schema.String)
 })).annotations({
 	description: "A normalized event recorded from the harness trajectory.",
@@ -475,6 +494,7 @@ const EvalRun = Schema.Struct({
 	cases: Schema.Array(Schema.String),
 	cells: Schema.Array(EvalCell),
 	costs: Schema.NullOr(EvalCosts),
+	executedBy: Schema.optionalWith(Schema.NullOr(EvalExecutor), { default: () => null }),
 	failure: Schema.NullOr(Schema.String),
 	finishedAt: Schema.NullOr(EvalTimestamp),
 	id: Schema.String,
@@ -529,6 +549,14 @@ const EvalCellHistoryEntry = Schema.Struct({
 	description: "A previous scored result for the same cell identity.",
 	identifier: "EvalCellHistoryEntry"
 });
+const RunSubscription = Schema.Struct({
+	expiresAtMillis: Schema.Number,
+	tag: Schema.String,
+	token: Schema.String
+}).annotations({
+	description: "A scoped, read-only token for watching one run in real time.",
+	identifier: "RunSubscription"
+});
 //#endregion
 //#region ../schema/src/domain/eval-playground.ts
 const PlaygroundCaseView = Schema.Struct({
@@ -610,23 +638,74 @@ const ModelCatalogue = Schema.Struct({
 	identifier: "ModelCatalogue"
 });
 //#endregion
-//#region ../schema/src/internal/authentication.ts
-var CurrentActor = class extends Context.Tag("@anpord/schema/CurrentActor")() {};
-HttpApiMiddleware.Tag()("@anpord/schema/Authentication", {
-	failure: Unauthorized,
-	provides: CurrentActor,
-	security: { session: HttpApiSecurity.apiKey({
-		in: "cookie",
-		key: "anpord.session_token"
-	}) }
+//#region ../schema/src/domain/harness-event.ts
+const OccurredAtMillis = Schema.optional(Schema.Number);
+const HarnessUsage = Schema.Struct({
+	cacheReadTokens: Schema.Int,
+	cacheWriteTokens: Schema.Int,
+	costUsd: Schema.optional(Schema.Number),
+	inputTokens: Schema.Int,
+	outputTokens: Schema.Int,
+	totalTokens: Schema.Int
 });
+const HarnessEvent = Schema.Union(Schema.Struct({
+	_tag: Schema.Literal("Started"),
+	at: OccurredAtMillis,
+	model: Schema.String,
+	sessionId: Schema.String
+}), Schema.Struct({
+	_tag: Schema.Literal("Message"),
+	at: OccurredAtMillis,
+	role: Schema.Literal("assistant", "user"),
+	text: Schema.String,
+	usage: Schema.optional(HarnessUsage)
+}), Schema.Struct({
+	_tag: Schema.Literal("Command"),
+	at: OccurredAtMillis,
+	command: Schema.String,
+	exitCode: Schema.NullOr(Schema.Int),
+	output: Schema.String,
+	startedAt: Schema.optional(Schema.Number)
+}), Schema.Struct({
+	_tag: Schema.Literal("FileChange"),
+	at: OccurredAtMillis,
+	paths: Schema.Array(Schema.String)
+}), Schema.Struct({
+	_tag: Schema.Literal("ToolCall"),
+	at: OccurredAtMillis,
+	callId: Schema.NullOr(Schema.String),
+	input: Schema.String,
+	name: Schema.String,
+	output: Schema.optional(Schema.String),
+	error: Schema.optional(Schema.String),
+	startedAt: Schema.optional(Schema.Number),
+	status: Schema.NullOr(Schema.String)
+}), Schema.Struct({
+	_tag: Schema.Literal("Finished"),
+	at: OccurredAtMillis,
+	reason: Schema.String
+}));
 //#endregion
-//#region ../schema/src/public/authentication.ts
-var ApiKeyAuthentication = class extends HttpApiMiddleware.Tag()("@anpord/schema/ApiKeyAuthentication", {
-	failure: Unauthorized,
-	provides: CurrentActor,
-	security: { bearer: HttpApiSecurity.bearer }
-}) {};
+//#region ../schema/src/domain/trial.ts
+const TrialStatus = Schema.Literal("queued", "running", "passed", "failed", "void");
+Schema.decodeUnknownOption(TrialStatus);
+const VerifyStepResult = Schema.Struct({
+	command: Schema.String,
+	exitCode: Schema.Int
+});
+const TrialOutcome = Schema.Struct({
+	artifacts: Schema.optional(Schema.Array(EvalArtifactMetadata)),
+	validations: Schema.optional(EvalValidations),
+	judgments: Schema.optional(Schema.Array(EvalJudgment)),
+	commandCount: Schema.Int,
+	exitCode: Schema.Int,
+	modelMs: Schema.Int,
+	passed: Schema.Boolean,
+	sandboxMs: Schema.Int,
+	status: TrialStatus,
+	verifySteps: Schema.Array(VerifyStepResult),
+	voidFields: Schema.Array(Schema.String)
+});
 //#endregion
 //#region ../schema/src/public/evals-api.ts
 const EvalRunRequest = Schema.Struct({ id: Schema.String }).annotations({
@@ -644,7 +723,7 @@ const EvalModelsRequest = Schema.Struct({
 	description: "Select a harness whose available models should be listed.",
 	identifier: "EvalModelsRequest"
 });
-const PublicEvalSandbox = EvalSandbox.annotations({
+const PublicEvalSandbox = Schema.Literal(...HOSTED_SANDBOXES).annotations({
 	description: "The hosted sandbox a task runs in.",
 	identifier: "PublicEvalSandbox"
 });
@@ -680,6 +759,7 @@ const PublicEvalTask = Schema.Struct({
 const PublicStartEvalRequest = Schema.Struct({
 	trigger: Schema.optional(EvalTrigger),
 	cases: Schema.Array(PublicEvalCase).pipe(Schema.minItems(1), Schema.maxItems(100)),
+	executeLocally: Schema.optional(Schema.Boolean),
 	name: Schema.optional(EvalName),
 	prompt: EvalPrompt,
 	tasks: Schema.Array(PublicEvalTask).pipe(Schema.minItems(1), Schema.maxItems(20)),
@@ -688,6 +768,25 @@ const PublicStartEvalRequest = Schema.Struct({
 	description: `Start a grid with at most 100 total case, task, and trial combinations.`,
 	identifier: "StartEvalRequest"
 });
-var PublicEvalsGroup = class extends HttpApiGroup.make("evals").add(HttpApiEndpoint.post("list", "/evals.list").setPayload(ListEvalsRequest).addSuccess(EvalRunPage).annotate(OpenApi.Summary, "List eval runs").annotate(OpenApi.Description, "Newest first. Pass the `next` cursor from a response to read the page after it; a null `next` means there are no more.")).add(HttpApiEndpoint.post("start", "/evals.start").setPayload(PublicStartEvalRequest).addSuccess(StartedEval).annotate(OpenApi.Summary, "Start an eval run").annotate(OpenApi.Description, `Starts the grid and returns its id while trials continue in the background. ${PROFILE_HARNESS_RULE}`)).add(HttpApiEndpoint.post("artifact", "/evals.artifact").setPayload(EvalArtifactRequest).addSuccess(EvalArtifact)).add(HttpApiEndpoint.post("get", "/evals.get").setPayload(EvalRunRequest).addSuccess(EvalRun).annotate(OpenApi.Summary, "Get an eval run")).add(HttpApiEndpoint.post("cellHistory", "/evals.cellHistory").setPayload(EvalCellRequest).addSuccess(Schema.Array(EvalCellHistoryEntry)).annotate(OpenApi.Summary, "List a cell's history").annotate(OpenApi.Description, "Returns the 20 most recent results.")).add(HttpApiEndpoint.post("rerunCell", "/evals.rerunCell").setPayload(Schema.extend(EvalRunRequest, Schema.extend(EvalCellRequest, RerunCellRequest))).addSuccess(StartedEval).annotate(OpenApi.Summary, "Rerun one cell")).add(HttpApiEndpoint.post("models", "/evals.models").setPayload(EvalModelsRequest).addSuccess(ModelCatalogue).annotate(OpenApi.Summary, "List models available to the harness").annotate(OpenApi.Description, "The command harness has no catalogue of its own, so its list is empty: the model is whatever the profile's run command reads from ANPORD_MODEL.")).addError(BadRequest).addError(Conflict).addError(Forbidden).addError(NotFound).middleware(ApiKeyAuthentication).annotate(OpenApi.Title, "Evals").annotate(OpenApi.Description, "Run cases across harness, model, and sandbox combinations and compare the results with their baselines.") {};
+const ReportedTrial = Schema.Struct({
+	caseName: EvalCaseName,
+	events: Schema.Array(HarnessEvent),
+	ordinal: Schema.Int.pipe(Schema.nonNegative()),
+	outcome: TrialOutcome,
+	sandboxId: Schema.optional(Schema.NullOr(Schema.String)),
+	taskIndex: Schema.Int.pipe(Schema.nonNegative()),
+	usage: Schema.optional(Schema.NullOr(HarnessUsage))
+}).annotations({
+	description: "One trial a client ran and is reporting the result of.",
+	identifier: "ReportedTrial"
+});
+const ReportTrialRequest = Schema.Struct({
+	id: Schema.String,
+	trial: ReportedTrial
+}).annotations({
+	description: "A trial the caller ran, named by the run it belongs to.",
+	identifier: "ReportTrialRequest"
+});
+var PublicEvalsGroup = class extends HttpApiGroup.make("evals").add(HttpApiEndpoint.post("list", "/evals.list").setPayload(ListEvalsRequest).addSuccess(EvalRunPage).annotate(OpenApi.Summary, "List eval runs").annotate(OpenApi.Description, "Newest first. Pass the `next` cursor from a response to read the page after it; a null `next` means there are no more.")).add(HttpApiEndpoint.post("start", "/evals.start").setPayload(PublicStartEvalRequest).addSuccess(StartedEval).annotate(OpenApi.Summary, "Start an eval run").annotate(OpenApi.Description, `Starts the grid and returns its id while trials continue in the background. ${PROFILE_HARNESS_RULE}`)).add(HttpApiEndpoint.post("reportTrial", "/evals.reportTrial").setPayload(ReportTrialRequest).addSuccess(Schema.Void).annotate(OpenApi.Summary, "Report a trial run outside the platform").annotate(OpenApi.Description, "For a run started with executeLocally. The result is recorded as reported: it was produced somewhere the platform cannot inspect, so it is marked and kept out of baselines.")).add(HttpApiEndpoint.post("finishRun", "/evals.finishRun").setPayload(EvalRunRequest).addSuccess(EvalRun).annotate(OpenApi.Summary, "Settle a run whose trials the caller ran").annotate(OpenApi.Description, "Settles the run from the trials reported so far. A run left unsettled is swept like any other abandoned work.")).add(HttpApiEndpoint.post("artifact", "/evals.artifact").setPayload(EvalArtifactRequest).addSuccess(EvalArtifact)).add(HttpApiEndpoint.post("get", "/evals.get").setPayload(EvalRunRequest).addSuccess(EvalRun).annotate(OpenApi.Summary, "Get an eval run")).add(HttpApiEndpoint.post("subscription", "/evals.subscription").setPayload(EvalRunRequest).addSuccess(RunSubscription).annotate(OpenApi.Summary, "Watch an eval run as it moves").annotate(OpenApi.Description, "Returns a read-only token scoped to this run, for following it in real time. Short-lived: request another when it expires.")).add(HttpApiEndpoint.post("cellHistory", "/evals.cellHistory").setPayload(EvalCellRequest).addSuccess(Schema.Array(EvalCellHistoryEntry)).annotate(OpenApi.Summary, "List a cell's history").annotate(OpenApi.Description, "Returns the 20 most recent results.")).add(HttpApiEndpoint.post("rerunCell", "/evals.rerunCell").setPayload(Schema.extend(EvalRunRequest, Schema.extend(EvalCellRequest, RerunCellRequest))).addSuccess(StartedEval).annotate(OpenApi.Summary, "Rerun one cell")).add(HttpApiEndpoint.post("models", "/evals.models").setPayload(EvalModelsRequest).addSuccess(ModelCatalogue).annotate(OpenApi.Summary, "List models available to the harness").annotate(OpenApi.Description, "The command harness has no catalogue of its own, so its list is empty: the model is whatever the profile's run command reads from ANPORD_MODEL.")).addError(BadRequest).addError(Conflict).addError(Forbidden).addError(NotFound).middleware(ApiKeyAuthentication).annotate(OpenApi.Title, "Evals").annotate(OpenApi.Description, "Run cases across harness, model, and sandbox combinations and compare the results with their baselines.") {};
 //#endregion
-export { EvalValidator as a, PROFILE_LIMITS as c, EvalSourceFiles as d, EvalTrigger as f, NotFound as g, Forbidden as h, EvalRun as i, ProfilePath as l, Conflict as m, PublicStartEvalRequest as n, EnvName as o, BadRequest as p, ApiKeyAuthentication as r, HarnessProfile as s, PublicEvalsGroup as t, profileFitsHarness as u };
+export { CredentialValues as S, Forbidden as _, EvalSandbox as a, CredentialIntegration as b, HarnessProfile as c, profileFitsHarness as d, EvalSourceFiles as f, Conflict as g, BadRequest as h, EvalRun as i, PROFILE_LIMITS as l, ApiKeyAuthentication as m, PublicStartEvalRequest as n, EvalValidator as o, EvalTrigger as p, HarnessEvent as r, EnvName as s, PublicEvalsGroup as t, ProfilePath as u, NotFound as v, CredentialScope as x, CredentialConnection as y };