@alpic-ai/tracks 0.6.0 → 0.9.1

package/dist/main.js

@@ -1,8 +1,8 @@
 #!/usr/bin/env node
 import { NodeRuntime, NodeServices } from "@effect/platform-node";
 import { HttpApi, HttpApiClient, HttpApiEndpoint, HttpApiGroup, HttpApiMiddleware, HttpApiSchema, HttpApiSecurity, OpenApi } from "effect/http-api";
-import { Config, Console, Context, Deferred, Effect, Layer, Option, Redacted, Result, Schema, SchemaTransformation, Stream, Struct } from "effect";
-import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/http";
+import { Config, Console, Context, Deferred, Effect, FileSystem, Layer, Option, Path, Redacted, Result, Schedule, Schema, SchemaTransformation, Stream, Struct } from "effect";
+import { FetchHttpClient, HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http";
 import { isIPv4 } from "node:net";
 import { Argument, CliError, Command, Flag } from "effect/cli";
 import { createServer } from "node:http";
@@ -30,8 +30,7 @@ const List = (item) => Schema.Struct({
 	data: Schema.Array(item),
 	has_more: Schema.Boolean.annotate({ description: "Whether more items are available after this page." })
 });
-//#endregion
-//#region ../contract/src/validation.ts
+// ../contract/src/validation.ts
 const RequestIssue = Schema.Struct({
 	path: Schema.Array(Schema.Union([Schema.String, Schema.Number])).annotate({ description: "Keys leading to the wrong value inside that part of the request, such as `[\"agent\", \"model\"]`. Empty when the problem concerns that part as a whole." }),
 	message: Schema.String.annotate({ description: "What is wrong with the value." })
@@ -53,8 +52,7 @@ var InvalidRequest = class extends Schema.TaggedError()("InvalidRequest", {
 	description: "The request is invalid: a value is missing, has the wrong type or is out of range. Fix it before sending the request again."
 }) {};
 var RequestValidation = class extends HttpApiMiddleware.Service()("@tracks/contract/RequestValidation", { error: InvalidRequest }) {};
-//#endregion
-//#region ../contract/src/connections.ts
+// ../contract/src/connections.ts
 const ConnectionId = Schema.String.pipe(Schema.brand("ConnectionId")).annotate({
 	identifier: "ConnectionId",
 	description: "Unique identifier of the connection."
@@ -144,20 +142,10 @@ var ConnectionsApi = class extends HttpApiGroup.make("connections").add(HttpApiE
 	title: "Connections",
 	description: "Let agents use MCP servers as one of their users, without their credentials entering the sandbox."
 })).middleware(ApiKeyAuthorization).middleware(RequestValidation) {};
-//#endregion
-//#region ../contract/src/files.ts
+// ../contract/src/files.ts
 const FileId = Schema.String.check(Schema.isPattern(/^[0-9a-f-]{36}$/)).pipe(Schema.brand("FileId")).annotate({ description: "Reference returned when requesting a file upload." });
 const FileName = Schema.String.check(Schema.isPattern(/^[A-Za-z0-9][A-Za-z0-9._ -]{0,127}$/)).annotate({ description: "Filename, without directories; up to 128 ASCII letters, digits, spaces, dots, hyphens or underscores, starting with a letter or digit." });
-const FileContentType = Schema.Literals([
-	"image/png",
-	"image/jpeg",
-	"image/webp",
-	"image/gif",
-	"application/pdf",
-	"text/plain",
-	"text/csv",
-	"application/json"
-]);
+const FileContentType = Schema.String.check(Schema.isPattern(/^[\w!#$&^.+-]{1,127}\/[\w!#$&^.+-]{1,127}$/)).annotate({ description: "MIME type, without parameters; `application/octet-stream` when unknown." });
 const FileSize = Schema.Int.check(Schema.isBetween({
 	minimum: 1,
 	maximum: 20971520
@@ -192,24 +180,27 @@ var FilesApi = class extends HttpApiGroup.make("files").add(HttpApiEndpoint.post
 	}).pipe(HttpApiSchema.status(201))
 }).annotateMerge(OpenApi.annotations({
 	summary: "Request a file upload",
-	description: "Request with `name`, `content_type` and exact byte `size`, e.g. `{\"name\":\"photo.png\",\"content_type\":\"image/png\",\"size\":12345}`. Supports PNG, JPEG, WebP, GIF, PDF, text, CSV and JSON, up to 20 MiB per file. PUT raw bytes (not JSON or base64) to the returned `url` with every returned header within 15 minutes; browsers set `content-length` from the body. Uploads are write-once; repeating the PUT returns 412. Pass the returned `id` in `input_files` when creating a run. Files in the landing zone expire after 24 hours and are deleted asynchronously. Files attached to a run are retained permanently and their references can be reused by your organization."
-}))).middleware(ApiKeyAuthorization).middleware(RequestValidation) {};
-//#endregion
-//#region ../contract/src/runs.ts
+	description: "Request with `name`, `content_type` and exact byte `size`, e.g. `{\"name\":\"photo.png\",\"content_type\":\"image/png\",\"size\":12345}`. Any type of file, up to 20 MiB each. PUT raw bytes (not JSON or base64) to the returned `url` with every returned header within 15 minutes; browsers set `content-length` from the body. Uploads are write-once; repeating the PUT returns 412. Pass the returned `id` in `input_files` when creating a run. Files in the landing zone expire after 24 hours and are deleted asynchronously. Files attached to a run are retained permanently and their references can be reused by your organization."
+}))).annotateMerge(OpenApi.annotations({
+	title: "Files",
+	description: "Upload files to place in a run's sandbox."
+})).middleware(ApiKeyAuthorization).middleware(RequestValidation) {};
+// ../contract/src/runs.ts
 const RunId = Schema.String.pipe(Schema.brand("RunId")).annotate({
 	identifier: "RunId",
 	description: "Unique identifier of the run."
 });
 const Harness = Schema.Literals([
 	"claude-code",
+	"claude-desktop",
 	"codex",
 	"codex-app",
 	"cursor"
 ]).annotate({
 	identifier: "Harness",
-	description: "The coding agent to run: `claude-code` (Claude Code), `codex` (OpenAI's Codex CLI), `codex-app` (the Codex mode of OpenAI's ChatGPT desktop app, with its browser, computer-use and document tools) or `cursor` (Cursor's CLI)."
+	description: "The coding agent to run: `claude-code` (Claude Code), `claude-desktop` (Cowork, the agent workspace of Anthropic's Claude desktop app, for documents, research and longer tasks), `codex` (OpenAI's Codex CLI), `codex-app` (the Codex mode of OpenAI's ChatGPT desktop app, with its browser, computer-use and document tools) or `cursor` (Cursor's CLI)."
 });
-const Agent = Schema.Struct({
+const Agent$1 = Schema.Struct({
 	harness: Harness,
 	model: Schema.NonEmptyString.annotate({ description: "Model the agent uses, such as `claude-sonnet-5-5`, or a Cursor model id such as `gpt-5.6-luna-low`." })
 }).annotate({
@@ -333,14 +324,23 @@ const SkillsInput = Schema.Array(SkillInput).check(Schema.isMaxLength(MAX_SKILLS
 	path: [index, "name"],
 	issue: "Expected a name not reserved for the agent"
 }] : [])), Schema.makeFilter((skills) => skills.flatMap((skill) => skill.files).reduce((total, file) => total + encoder.encode(file.content).length, 0) <= MAX_SKILLS_BYTES || `Expected at most ${MAX_SKILLS_BYTES / 1024 / 1024} MB of skills together`)).annotate({ description: `Skills the agent can load, at most ${MAX_SKILLS} and ${MAX_SKILLS_BYTES / 1024 / 1024} MB together.` });
+const BROWSER_MCP_SERVER = "tracks-browser";
 const EnvironmentInput = Schema.Struct({
 	repository,
 	variables: Schema.OptionFromOptionalKey(Variables$1),
 	setup,
 	teardown,
 	skills: Schema.OptionFromOptionalKey(SkillsInput),
-	mcp_servers: Schema.OptionFromOptionalKey(McpServersInput)
-}).annotate({
+	mcp_servers: Schema.OptionFromOptionalKey(McpServersInput),
+	browser: Schema.Boolean.pipe(Schema.withDecodingDefaultKey(Effect.succeed(false))).annotate({ description: "Whether the agent gets a browser: a headless Chromium in its sandbox, driven through an MCP server named `tracks-browser`, whose pages can offer it tools through WebMCP. Screenshots of the pages it visits are kept with the run. Ignored by the Codex app and Claude Cowork, which always use their own browser. Defaults to false." })
+}).check(Schema.makeFilter((environment) => !environment.browser ? [] : Option.getOrElse(environment.mcp_servers, () => []).flatMap((server, index) => server.name === BROWSER_MCP_SERVER ? [{
+	path: [
+		"mcp_servers",
+		index,
+		"name"
+	],
+	issue: `Expected a name other than ${BROWSER_MCP_SERVER}, the browser's MCP server`
+}] : []))).annotate({
 	identifier: "EnvironmentInput",
 	description: "What the agent works on."
 });
@@ -366,7 +366,8 @@ const RunEnvironment = Schema.Struct({
 	setup,
 	teardown,
 	skills: Schema.Array(RunSkill),
-	mcp_servers: Schema.Array(RunMcpServer)
+	mcp_servers: Schema.Array(RunMcpServer),
+	browser: Schema.Boolean.annotate({ description: "Whether the agent was given a headless Chromium in its sandbox." })
 }).annotate({
 	identifier: "Environment",
 	description: "What the agent works on."
@@ -392,16 +393,17 @@ const LabelFilters = Schema.Array(Schema.String.check(Schema.isPattern(LABEL_FIL
 	decode: (filters) => Object.fromEntries(filters.map(keyAndValueOf)),
 	encode: (labels) => Object.entries(labels).map(([key, value]) => `${key}:${value}`)
 })));
+const MAX_BUDGET_USD = 1e4;
 const Budget = Schema.Finite.check(Schema.isBetween({
 	minimum: .01,
-	maximum: 1e4
+	maximum: MAX_BUDGET_USD
 })).annotate({ description: "Maximum the run may spend, in US dollars. Defaults to a standard budget when omitted. The run ends with `budget_exceeded` once it is reached." });
 const MIN_IDLE_TIMEOUT_SECONDS = 120;
 const MAX_IDLE_TIMEOUT_SECONDS = 900;
 const MIN_TIME_LIMIT_SECONDS = 60;
 const MAX_TIME_LIMIT_SECONDS = 43200;
 const CreateRunInput = Schema.Struct({
-	agent: Agent,
+	agent: Agent$1,
 	prompt: Schema.NonEmptyString.annotate({ description: "What you ask the agent to do first." }),
 	input_files: Schema.optionalKey(Schema.Array(FileId).check(Schema.isMaxLength(20)).annotate({ description: "Up to 20 uploaded file ids, staged in the sandbox before setup and referenced in the first prompt." })),
 	environment: EnvironmentInput,
@@ -465,19 +467,21 @@ const EndReason = Schema.Literals([
 const FailureKind = Schema.Literals([
 	"repository_checkout_failed",
 	"setup_failed",
+	"browser_not_started",
 	"sandbox_not_found",
 	"process_lost",
 	"sandbox_unavailable",
+	"agent_stopped",
 	"unexpected"
 ]).annotate({
 	identifier: "FailureKind",
-	description: "What kind of thing went wrong: the repository could not be checked out, the environment's setup command failed, the sandbox disappeared, the agent's process was lost, the sandbox provider failed, or something unexpected."
+	description: "What kind of thing went wrong: the repository could not be checked out, the environment's setup command failed, the browser did not start, the sandbox disappeared, the agent's process was lost, the sandbox provider failed, the agent of a run that accepts no follow-ups stopped with an error, or something unexpected."
 });
 const runFields = {
 	id: RunId,
 	input_files: Schema.Array(InputFile),
 	labels: Labels.annotate({ description: "Labels attached to the run when it was created." }),
-	agent: Agent,
+	agent: Agent$1,
 	environment: RunEnvironment,
 	budget_usd: Schema.Number.annotate({ description: "Maximum the run may spend, in US dollars." }),
 	one_shot: Schema.Boolean.annotate({ description: "Whether the run ends as soon as its first prompt finishes." }),
@@ -530,6 +534,10 @@ var RunNotFound = class extends Schema.TaggedError()("RunNotFound", { message: S
 	httpApiStatus: 404,
 	description: "No run with this id exists in your organization."
 }) {};
+var ScreenshotNotFound = class extends Schema.TaggedError()("ScreenshotNotFound", { message: Schema.String }, {
+	httpApiStatus: 404,
+	description: "This event has no screenshot."
+}) {};
 var ConnectionUnusable = class extends Schema.TaggedError()("ConnectionUnusable", { message: Schema.String }, {
 	httpApiStatus: 422,
 	description: "A `connection_id` in `environment.mcp_servers` does not exist or is not `active`."
@@ -557,13 +565,13 @@ var RunNotIdle = class extends Schema.TaggedError()("RunNotIdle", { message: Sch
 	httpApiStatus: 409,
 	description: "The run accepts no prompt now: it is one-shot, has ended, or is still working on a prompt."
 }) {};
-const Count = Schema.Natural;
+const Count$1 = Schema.Natural;
 const RunUsage = Schema.Struct({
 	cost_usd: Schema.Number.annotate({ description: "Total cost of the run so far, in US dollars." }),
 	models: Schema.Array(Schema.Struct({
 		provider: Schema.String.annotate({ description: "Company serving the model, such as `anthropic`." }),
 		model: Schema.String,
-		quantities: Schema.Record(Schema.String, Count).annotate({ description: "Units consumed, keyed by unit as the provider counts them, such as input tokens or web searches." }),
+		quantities: Schema.Record(Schema.String, Count$1).annotate({ description: "Units consumed, keyed by unit as the provider counts them, such as input tokens or web searches." }),
 		cost_usd: Schema.Number.annotate({ description: "Cost of this model's usage, in US dollars." })
 	})).annotate({ description: "Model usage, one entry per model." }),
 	sandbox: Schema.Struct({
@@ -578,18 +586,23 @@ const Frame = Schema.Struct({
 	run_id: RunId,
 	seq: Schema.Int.annotate({ description: "Position of the line in the run's output, starting at 1. Always increasing, but some numbers may be skipped." }),
 	prompt: PromptIndex.annotate({ description: "Index of the prompt the agent was working on." }),
+	process: Schema.String.annotate({ description: "Program in the sandbox that printed the line: `prompt-1`, `prompt-2`… for the agent working on each prompt, `browser` for its browser." }),
 	captured_at: Schema.DateTimeUtc.annotate({ description: "When the line was captured." }),
 	channel: Schema.Literals(["stdout", "stderr"]).annotate({ description: "Output stream the line came from." }),
 	body: Schema.String.annotate({ description: "The line, exactly as the agent printed it." })
 }).annotate({
 	identifier: "Frame",
-	description: "One line of the agent's output."
+	description: "One line of output from the agent or its browser."
 });
 const Outcome = Schema.Literals(["succeeded", "failed"]).annotate({
 	identifier: "Outcome",
 	description: "Whether the tool call or subagent succeeded."
 });
 const ToolCallId = Schema.String.annotate({ description: "Identifier of the tool call, unique in its run." });
+const Screenshot = Schema.Struct({ page_url: Schema.String.annotate({ description: "Address of the page shown." }) }).annotate({
+	identifier: "Screenshot",
+	description: "An image of the page the agent's browser showed; get it from `GET /runs/{id}/events/{seq}/screenshot`, with this event's `seq`."
+});
 const eventFields = {
 	run_id: RunId,
 	seq: Schema.Int.annotate({ description: "Position of the event in the run, starting at 1. Always increasing, but some numbers may be skipped." }),
@@ -639,7 +652,8 @@ const Event = Schema.Union([
 		tool_call_id: ToolCallId,
 		outcome: Outcome,
 		output: Schema.String.annotate({ description: "What the tool returned, possibly truncated." }),
-		structured: Schema.OptionFromOptionalKey(Schema.Json.annotate({ description: "The structured content an MCP tool returned beside its text, unchanged. Absent when the tool returned none or it was too large." }))
+		structured: Schema.OptionFromOptionalKey(Schema.Json.annotate({ description: "The structured content an MCP tool returned beside its text, unchanged. Absent when the tool returned none or it was too large." })),
+		screenshot: Schema.OptionFromOptionalKey(Screenshot.annotate({ description: "The page a browser tool call left its browser on, as the harness captured it. Absent when it captured none." }))
 	}).annotate({
 		identifier: "ToolEndedEvent",
 		description: "A tool call ended; its `tool_call_id` is the one of the `tool_started` event."
@@ -660,6 +674,27 @@ const Event = Schema.Union([
 		identifier: "SubagentEndedEvent",
 		description: "A subagent ended."
 	}),
+	Schema.Struct({
+		...eventFields,
+		type: Schema.Literal("page_visited"),
+		tool_call_id: Schema.OptionFromOptionalKey(ToolCallId.annotate({ description: "The browser tool call that led to the page, when the agent had made one." })),
+		page_id: Schema.String.annotate({ description: "Identifier of the page in the run: a later visit to the same page keeps it." }),
+		version: Schema.Int.annotate({ description: "How many times the page was seen changed, this one included, starting at 1." }),
+		url: Schema.String.annotate({ description: "URL of the page." }),
+		title: Schema.String.annotate({ description: "Title of the page, empty when it has none." }),
+		reason: Schema.Literals([
+			"arrival",
+			"settled",
+			"after_submit",
+			"error",
+			"after_action",
+			"prompt_end"
+		]).annotate({ description: "Why it was captured: on arrival, once settled, after a form was submitted, on an error, after a browser action, or when the prompt ended." }),
+		screenshot: Schema.OptionFromOptionalKey(Screenshot.annotate({ description: "The page as the browser captured it. Absent when it captured none; its image may be missing when the sandbox was gone before it was kept." }))
+	}).annotate({
+		identifier: "PageVisitedEvent",
+		description: "The agent's browser showed a page, new or changed."
+	}),
 	Schema.Struct({
 		...eventFields,
 		type: Schema.Literal("stop"),
@@ -788,6 +823,16 @@ var RunsApi = class extends HttpApiGroup.make("runs").add(HttpApiEndpoint.post("
 }).annotateMerge(OpenApi.annotations({
 	summary: "List a run's events",
 	description: "Returns what the agent did, in order: each prompt, then its messages, reasoning, tool calls and subagents, then a `stop` once the agent stopped. Events read alike whatever the harness, and are read from the run's output: `frame_seqs` points at the lines each one comes from. To get the next page, pass the `seq` of the last event received as `after`."
+})), HttpApiEndpoint.get("getScreenshot", "/runs/:id/events/:seq/screenshot", {
+	params: {
+		id: RunId,
+		seq: Schema.Int.check(Schema.isGreaterThanOrEqualTo(1))
+	},
+	success: Schema.Uint8Array.pipe(HttpApiSchema.asUint8Array({ contentType: "image/jpeg" })),
+	error: [RunNotFound, ScreenshotNotFound]
+}).annotateMerge(OpenApi.annotations({
+	summary: "Get a screenshot",
+	description: "Returns the image of a page the agent's browser showed, for a `tool_ended` or `page_visited` event that has a `screenshot`. Its content type is the image's own, usually `image/jpeg`."
 })), HttpApiEndpoint.get("followFrames", "/runs/:id/frames/stream", {
 	params: { id: RunId },
 	query: { after: OptionalQueryParam(Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), "Send only lines whose `seq` is greater than this.") },
@@ -808,8 +853,329 @@ var RunsApi = class extends HttpApiGroup.make("runs").add(HttpApiEndpoint.post("
 	title: "Runs",
 	description: "Start coding agents and follow their work."
 })).middleware(ApiKeyAuthorization).middleware(RequestValidation) {};
-//#endregion
-//#region ../contract/src/system.ts
+// ../contract/src/experiments.ts
+const ExperimentId = Schema.String.pipe(Schema.brand("ExperimentId")).annotate({
+	identifier: "ExperimentId",
+	description: "Unique identifier of the experiment."
+});
+const ExecutionId = Schema.String.pipe(Schema.brand("ExecutionId")).annotate({
+	identifier: "ExecutionId",
+	description: "Unique identifier of the execution."
+});
+const Agent = Schema.Struct({
+	harness: Harness,
+	model: Schema.String.annotate({ description: "Model the agent uses, such as `claude-sonnet-5-5`." })
+}).annotate({
+	identifier: "ExperimentAgent",
+	description: "A coding agent and the model it uses."
+});
+const MAX_ID_LENGTH = 128;
+const idChosenFor = (item, example) => Schema.NonEmptyString.check(Schema.isMaxLength(MAX_ID_LENGTH)).annotate({ description: `Identifier you choose for the ${item}, up to ${MAX_ID_LENGTH} characters, such as \`${example}\`. Results refer to it.` });
+const withUniqueIds = (item, items) => Schema.Array(item).check(Schema.makeFilter((all) => {
+	const ids = all.map((each) => each.id);
+	return new Set(ids).size === ids.length || `Expected each of the ${items}' ids once`;
+}));
+const libraryIds = (items) => Schema.Array(Schema.String).annotate({ description: `Ids of ${items} from your library, as listed by \`GET /library\`.` });
+const Evaluation = Schema.Struct({
+	id: idChosenFor("evaluation, unique in its scenario", "tests-pass"),
+	name: Schema.String.annotate({ description: "Short name of what is checked." }),
+	judging_instructions: Schema.String.annotate({ description: "The yes-or-no question a judge answers from what the agent did, such as `Do the tests pass?`." }),
+	weight: Schema.Int.check(Schema.isGreaterThanOrEqualTo(1)).annotate({ description: "How much it counts in the run's score. Defaults to 1." })
+}).annotate({
+	identifier: "Evaluation",
+	description: "One criterion every run of the scenario is judged on."
+});
+const MAX_INPUT_FILES = 20;
+const Scenario = Schema.Struct({
+	id: idChosenFor("scenario, unique in the experiment", "checkout"),
+	name: Schema.String.annotate({ description: "Short name of the scenario." }),
+	prompt: Schema.String.annotate({ description: "What the agent is asked to do." }),
+	input_files: Schema.Array(FileId).check(Schema.isMaxLength(MAX_INPUT_FILES)).annotate({ description: `Up to ${MAX_INPUT_FILES} uploaded file ids, given to each of the scenario's runs. Kept for good once saved. Defaults to none.` }),
+	evaluations: withUniqueIds(Evaluation, "evaluations").annotate({ description: "What a run is judged on. Runs of a scenario without evaluations are not judged." }),
+	passing_score: Schema.OptionFromOptionalKey(Schema.Int.check(Schema.isBetween({
+		minimum: 0,
+		maximum: 100
+	})).annotate({ description: "Score out of 100, weighted by the evaluations' weights, a run needs to pass. Omit it so a run passes only when it passes every evaluation." }))
+}).annotate({
+	identifier: "Scenario",
+	description: "One task given to the agents."
+});
+const AgentTools = Schema.Struct({
+	id: idChosenFor("agent tool configuration, unique in the experiment", "with-docs-mcp"),
+	name: Schema.String.annotate({ description: "Short name of the agent tools, such as `No agent tools` for the baseline." }),
+	skills: libraryIds("skills"),
+	mcp_servers: libraryIds("MCP servers"),
+	command_line_tools: libraryIds("command-line tools"),
+	browser: Schema.Boolean.pipe(Schema.withDecodingDefaultKey(Effect.succeed(false))).annotate({ description: "Whether agents without a browser of their own get one, a headless Chromium in their sandbox whose pages can offer them tools through WebMCP. The Codex app and Claude Cowork always use their own browser. Defaults to false." })
+}).annotate({
+	identifier: "AgentTools",
+	description: "Skills, MCP servers, command-line tools and a browser given to the agent. Keep one with none as the baseline the others are compared to."
+});
+const ExcludedVariant = Schema.Struct({
+	repository_id: Schema.OptionFromOptionalKey(Schema.String.annotate({ description: "Id of one of the experiment's repositories. Omit it for no repository." })),
+	agent_tools_id: Schema.String.annotate({ description: "Id of one of the experiment's agent tool configurations." })
+}).annotate({
+	identifier: "ExcludedVariant",
+	description: "A combination of repository and agent tools not to run."
+});
+const RunSettings = Schema.Struct({
+	repetitions: Schema.Int.check(Schema.isGreaterThanOrEqualTo(1)).annotate({ description: "Identical runs started for each scenario, variant and agent, to compare results beyond chance." }),
+	time_limit_seconds: Schema.Finite.check(Schema.isBetween({
+		minimum: 0,
+		exclusiveMinimum: true,
+		maximum: MAX_TIME_LIMIT_SECONDS
+	})).annotate({ description: `Longest each run may last, in seconds, above 0 and at most ${MAX_TIME_LIMIT_SECONDS}.` }),
+	budget_usd: Schema.Finite.check(Schema.isBetween({
+		minimum: 0,
+		exclusiveMinimum: true,
+		maximum: MAX_BUDGET_USD
+	})).annotate({ description: `Most each run may spend, in US dollars, above 0 and at most ${MAX_BUDGET_USD}.` }),
+	setup: script("Shell script run in each run's sandbox after checkout and before the prompt."),
+	teardown: script("Shell script run in each run's sandbox after the run ends."),
+	variables: Schema.Array(Schema.Struct({
+		name: Schema.String,
+		value: Schema.String
+	})).annotate({ description: "Environment variables set in every process of each run's agent, each name once. Defaults to none." })
+}).annotate({
+	identifier: "RunSettings",
+	description: "What every run of the experiment shares."
+});
+const experimentFields = {
+	name: Schema.String.annotate({ description: "Name of the experiment." }),
+	scenarios: withUniqueIds(Scenario, "scenarios"),
+	agents: Schema.Array(Agent).annotate({ description: "Agents every scenario is given to, each once." }),
+	repositories: libraryIds("repositories").annotate({ description: "Ids of repositories from your library, as listed by `GET /library`, each once, tried in turn with every agent tool configuration. Defaults to none: runs start without a repository." }),
+	agent_tools: withUniqueIds(AgentTools, "agent_tools").annotate({ description: "Agent tool configurations each tried in turn with every repository." }),
+	excluded_variants: Schema.Array(ExcludedVariant).annotate({ description: "Combinations of repository and agent tools not to run. Every other combination is a variant, run for each scenario and agent. Defaults to none." }),
+	run_settings: RunSettings
+};
+const Experiment = Schema.Struct({
+	id: ExperimentId,
+	...experimentFields
+}).annotate({
+	identifier: "Experiment",
+	description: "Scenarios given to several agents in several variants, each run judged on the scenario's evaluations."
+});
+const EvaluationInput = Evaluation.mapFields(Struct.evolve({ weight: (weight) => weight.pipe(Schema.withDecodingDefaultKey(Effect.succeed(1))) })).annotate({ identifier: "EvaluationInput" });
+const ScenarioInput = Scenario.mapFields(Struct.evolve({
+	input_files: (inputFiles) => inputFiles.pipe(Schema.withDecodingDefaultKey(Effect.succeed([]))),
+	evaluations: () => withUniqueIds(EvaluationInput, "evaluations")
+})).annotate({ identifier: "ScenarioInput" });
+const RunSettingsInput = RunSettings.mapFields(Struct.evolve({ variables: (variables) => variables.pipe(Schema.withDecodingDefaultKey(Effect.succeed([]))) })).annotate({ identifier: "RunSettingsInput" });
+const unknownExclusions = (input) => input.excluded_variants.flatMap((excluded, index) => [...input.agent_tools.some((agentTools) => agentTools.id === excluded.agent_tools_id) ? [] : [{
+	path: [
+		"excluded_variants",
+		index,
+		"agent_tools_id"
+	],
+	issue: "Expected the id of one of the agent tool configurations"
+}], ...Option.match(excluded.repository_id, {
+	onNone: () => true,
+	onSome: (repository) => input.repositories.includes(repository)
+}) ? [] : [{
+	path: [
+		"excluded_variants",
+		index,
+		"repository_id"
+	],
+	issue: "Expected one of the repositories"
+}]]);
+const ExperimentInput = Schema.Struct(experimentFields).mapFields(Struct.evolve({
+	scenarios: () => withUniqueIds(ScenarioInput, "scenarios"),
+	repositories: (repositories) => repositories.check(Schema.isUnique()).pipe(Schema.withDecodingDefaultKey(Effect.succeed([]))),
+	excluded_variants: (excluded) => excluded.pipe(Schema.withDecodingDefaultKey(Effect.succeed([]))),
+	run_settings: () => RunSettingsInput
+})).check(Schema.makeFilter(unknownExclusions)).annotate({ identifier: "ExperimentInput" });
+const Count = Schema.Natural;
+const Execution = Schema.Struct({
+	id: ExecutionId,
+	experiment_id: ExperimentId,
+	number: Schema.Int.annotate({ description: "Its place among the experiment's executions, from 1." }),
+	launched_at: Schema.DateTimeUtc.annotate({ description: "When the execution was launched." }),
+	canceled_at: Schema.OptionFromOptionalKey(Schema.DateTimeUtc.annotate({ description: "When the execution was canceled. Absent unless it was." })),
+	pending: Schema.Boolean.annotate({ description: "Whether some of its runs have not ended yet or wait for their evaluation results. Its pass rate is final once it is `false`, canceled or not." }),
+	slots: Schema.Struct({
+		total: Count.annotate({ description: "Runs the execution plans." }),
+		running: Count.annotate({ description: "Runs not started yet or not ended." }),
+		judging: Count.annotate({ description: "Ended runs waiting for their evaluation results." }),
+		judged: Count.annotate({ description: "Runs with a verdict." }),
+		passed: Count.annotate({ description: "Runs that passed their scenario." })
+	}).annotate({ description: "How many of its slots, one per scenario, variant, agent and repetition, stand where." }),
+	pass_rate: Schema.OptionFromOptionalKey(Schema.Int.check(Schema.isBetween({
+		minimum: 0,
+		maximum: 100
+	})).annotate({ description: "Runs that passed out of those with a verdict, as a whole percent. Absent until one run has a verdict." })),
+	cost_usd: Schema.Number.annotate({ description: "What its runs cost so far, in US dollars." })
+}).annotate({
+	identifier: "Execution",
+	description: "One launch of an experiment: a run for every scenario, variant, agent and repetition, with the experiment as it was when launched."
+});
+const NotEvaluatedReason = Schema.Literals([
+	"never_started",
+	"no_evaluations",
+	"run_not_completed",
+	"agent_failed",
+	"evidence_too_long",
+	"too_many_attempts"
+]).annotate({ description: "Why the run has no verdict: it never started, its scenario has no evaluations, it did not end `completed`, its agent stopped with an error such as its model provider being unreachable, what it did was too long to judge, or judging failed too many times." });
+const Slot = Schema.Struct({
+	id: Schema.String.annotate({ description: "Unique identifier of the slot." }),
+	scenario_id: Schema.String,
+	repository_id: Schema.OptionFromOptionalKey(Schema.String.annotate({ description: "Absent when the run has no repository." })),
+	agent_tools_id: Schema.String,
+	agent: Agent,
+	repetition: Schema.Int.annotate({ description: "Which of the identical runs it is, from 1." }),
+	run_id: Schema.OptionFromOptionalKey(RunId.annotate({ description: "The run, once started." })),
+	outcome: Schema.Literals([
+		"running",
+		"judging",
+		"judged",
+		"not_evaluated"
+	]).annotate({ description: "`running` while its run waits to start or has not ended, `judging` while it waits for evaluation results, `judged` once they are all in, or `not_evaluated` when it never will be." }),
+	verdict: Schema.OptionFromOptionalKey(Schema.Literals(["passed", "failed"]).annotate({ description: "Whether the run passed its scenario, once judged." })),
+	score: Schema.OptionFromOptionalKey(Schema.Number.annotate({ description: "Weighted share of evaluations passed, out of 100, once every evaluation result is in." })),
+	not_evaluated_reason: Schema.OptionFromOptionalKey(NotEvaluatedReason),
+	refusal: Schema.OptionFromOptionalKey(Schema.String.annotate({ description: "Why its run could not start." })),
+	evaluation_results: Schema.Array(Schema.Struct({
+		evaluation_id: Schema.String,
+		passed: Schema.Boolean
+	})).annotate({ description: "Each evaluation judged so far, and whether the run passed it." })
+}).annotate({
+	identifier: "Slot",
+	description: "One run an execution plans, for one scenario, variant, agent and repetition, and how it was judged."
+});
+var ExperimentNotFound = class extends Schema.TaggedError()("ExperimentNotFound", { message: Schema.String }, {
+	httpApiStatus: 404,
+	description: "No experiment with this id exists in your organization."
+}) {};
+var ExecutionNotFound = class extends Schema.TaggedError()("ExecutionNotFound", { message: Schema.String }, {
+	httpApiStatus: 404,
+	description: "No execution with this id exists in your organization."
+}) {};
+var ExperimentIncomplete = class extends Schema.TaggedError()("ExperimentIncomplete", {
+	message: Schema.String,
+	missing: Schema.Array(Schema.String).annotate({ description: "Each thing to fix before launching, in a sentence." })
+}, {
+	httpApiStatus: 422,
+	description: "The experiment cannot be launched as it is: `missing` says what to fix."
+}) {};
+const experimentParams = { id: ExperimentId };
+var ExperimentsApi = class extends HttpApiGroup.make("experiments").add(HttpApiEndpoint.post("create", "/experiments", {
+	payload: ExperimentInput,
+	success: Experiment.pipe(HttpApiSchema.status(201)),
+	error: FileUnavailable
+}).annotateMerge(OpenApi.annotations({
+	summary: "Create an experiment",
+	description: "Creates an experiment from its scenarios, agents, variants and run settings. Launch it to start its runs."
+})), HttpApiEndpoint.get("list", "/experiments", { success: List(Experiment) }).annotateMerge(OpenApi.annotations({
+	summary: "List experiments",
+	description: "Returns every experiment."
+})), HttpApiEndpoint.get("get", "/experiments/:id", {
+	params: experimentParams,
+	success: Experiment,
+	error: ExperimentNotFound
+}).annotateMerge(OpenApi.annotations({
+	summary: "Get an experiment",
+	description: "Returns an experiment."
+})), HttpApiEndpoint.put("update", "/experiments/:id", {
+	params: experimentParams,
+	payload: ExperimentInput,
+	success: Experiment,
+	error: [ExperimentNotFound, FileUnavailable]
+}).annotateMerge(OpenApi.annotations({
+	summary: "Update an experiment",
+	description: "Replaces the whole experiment: send it complete, as `GET /experiments/{id}` returns it. Executions already launched keep the experiment as it was."
+})), HttpApiEndpoint.delete("delete", "/experiments/:id", {
+	params: experimentParams,
+	success: HttpApiSchema.NoContent,
+	error: ExperimentNotFound
+}).annotateMerge(OpenApi.annotations({
+	summary: "Delete an experiment",
+	description: "Deletes an experiment. Its executions, their runs and results stay."
+}))).annotateMerge(OpenApi.annotations({
+	title: "Experiments",
+	description: "Define scenarios to give several agents in several variants, judged on what they did."
+})).middleware(ApiKeyAuthorization).middleware(RequestValidation) {};
+const executionParams = { id: ExecutionId };
+var ExecutionsApi = class extends HttpApiGroup.make("executions").add(HttpApiEndpoint.post("launch", "/experiments/:id/executions", {
+	params: experimentParams,
+	success: Execution.pipe(HttpApiSchema.status(201)),
+	error: [
+		ExperimentNotFound,
+		ExperimentIncomplete,
+		InsufficientBalance,
+		OrganizationBlocked
+	]
+}).annotateMerge(OpenApi.annotations({
+	summary: "Launch an experiment",
+	description: "Starts an execution of the experiment as it is now: a run for every scenario, variant, agent and repetition, at most 500, started in the background. Each run is judged on its scenario's evaluations once it ends. Executions start only while your organization's balance is above $0."
+})), HttpApiEndpoint.get("list", "/experiments/:id/executions", {
+	params: experimentParams,
+	success: List(Execution)
+}).annotateMerge(OpenApi.annotations({
+	summary: "List an experiment's executions",
+	description: "Returns the experiment's executions, the latest launched first, even after it was deleted."
+})), HttpApiEndpoint.get("get", "/executions/:id", {
+	params: executionParams,
+	success: Execution,
+	error: ExecutionNotFound
+}).annotateMerge(OpenApi.annotations({
+	summary: "Get an execution",
+	description: "Returns an execution, where its runs stand and its pass rate. Poll it until it is no longer `pending`."
+})), HttpApiEndpoint.get("listSlots", "/executions/:id/slots", {
+	params: executionParams,
+	success: List(Slot),
+	error: ExecutionNotFound
+}).annotateMerge(OpenApi.annotations({
+	summary: "List an execution's slots",
+	description: "Returns every run the execution plans, with its run once started and its verdict once judged. Read a run's work with the runs endpoints."
+})), HttpApiEndpoint.post("cancel", "/executions/:id/cancel", {
+	params: executionParams,
+	success: Execution.pipe(HttpApiSchema.status(202)),
+	error: ExecutionNotFound
+}).annotateMerge(OpenApi.annotations({
+	summary: "Cancel an execution",
+	description: "Starts none of its runs left and cancels those started. Canceling an execution already canceled does nothing."
+}))).annotateMerge(OpenApi.annotations({
+	title: "Executions",
+	description: "Launch experiments and read their results."
+})).middleware(ApiKeyAuthorization).middleware(RequestValidation) {};
+// ../contract/src/library.ts
+const id = Schema.String.annotate({ description: "Id to pick it in an experiment." });
+const Library = Schema.Struct({
+	repositories: Schema.Array(Schema.Struct({
+		id,
+		name: Schema.String,
+		url: Schema.String.annotate({ description: "URL of the Git repository." }),
+		ref: Schema.String.annotate({ description: "Branch, tag or commit checked out." })
+	})),
+	skills: Schema.Array(Schema.Struct({
+		id,
+		name: Schema.String,
+		description: Schema.String
+	})),
+	mcp_servers: Schema.Array(Schema.Struct({
+		id,
+		name: Schema.String.annotate({ description: "Name the agent knows the MCP server by." }),
+		url: Schema.String,
+		connected: Schema.Boolean.annotate({ description: "Whether runs can use it: always for one without an account, otherwise once connected in the web app." })
+	})),
+	command_line_tools: Schema.Array(Schema.Struct({
+		id,
+		name: Schema.String
+	}))
+}).annotate({
+	identifier: "Library",
+	description: "What your organization keeps for its experiments, added in the web app."
+});
+var LibraryApi = class extends HttpApiGroup.make("library").add(HttpApiEndpoint.get("get", "/library", { success: Library }).annotateMerge(OpenApi.annotations({
+	summary: "Get the library",
+	description: "Returns the repositories, skills, MCP servers and command-line tools your experiments can pick, by id."
+}))).annotateMerge(OpenApi.annotations({
+	title: "Library",
+	description: "What your experiments can give their agents."
+})).middleware(ApiKeyAuthorization).middleware(RequestValidation) {};
+// ../contract/src/system.ts
 const EgressIps = Schema.Struct({ ips: Schema.Array(Schema.String).annotate({ description: "IPv4 addresses, empty when the API runs locally." }) }).annotate({ identifier: "EgressIps" });
 var SystemApi = class extends HttpApiGroup.make("system", { topLevel: true }).add(HttpApiEndpoint.get("health", "/health", { success: HttpApiSchema.NoContent }).annotateMerge(OpenApi.annotations({
 	summary: "Check health",
@@ -818,23 +1184,23 @@ var SystemApi = class extends HttpApiGroup.make("system", { topLevel: true }).ad
 	summary: "List egress IPs",
 	description: "The fixed addresses we call MCP servers and their authorization servers from. Allow them on an MCP server that only accepts known addresses. Commands the agent runs in its sandbox, such as cloning the repository, come from other addresses. No key required."
 }))).annotateMerge(OpenApi.annotations({ title: "System" })) {};
-//#endregion
-//#region ../contract/src/api.ts
-var TracksApi = class extends HttpApi.make("tracks").add(RunsApi).add(FilesApi).add(ConnectionsApi).add(SystemApi).annotate(HttpApi.ParseOptions, { errors: "all" }).annotateMerge(OpenApi.annotations({
+// ../contract/src/api.ts
+var TracksApi = class extends HttpApi.make("tracks").add(RunsApi).add(FilesApi).add(ConnectionsApi).add(ExperimentsApi).add(ExecutionsApi).add(LibraryApi).add(SystemApi).annotate(HttpApi.ParseOptions, { errors: "all" }).annotateMerge(OpenApi.annotations({
 	title: "Tracks API",
-	description: "Run coding agents such as Claude Code in isolated sandboxes with an API key of your organization. Create a run with a prompt, then follow its output and what it costs."
+	description: "Run coding agents such as Claude Code in isolated sandboxes with an API key of your organization. Create a run with a prompt, then follow its output and what it costs, or launch an experiment to compare agents on your scenarios."
 })) {};
-//#endregion
-//#region ../contract/src/client.ts
+// ../contract/src/client.ts
 const CLI_USER_AGENT = "tracks";
 var ApiClient = class ApiClient extends Context.Service()("@tracks/contract/ApiClient") {
 	static layer = (options) => Layer.effect(ApiClient, HttpApiClient.make(TracksApi, { transformClient: HttpClient.mapRequest((request) => {
 		const located = HttpClientRequest.prependUrl(request, options.baseUrl.href);
 		return options.userAgent === void 0 ? located : HttpClientRequest.setHeader(located, "user-agent", options.userAgent);
-	}) })).pipe(Layer.provide(HttpApiMiddleware.layerClient(ApiKeyAuthorization, ({ next, request }) => next(HttpClientRequest.bearerToken(request, Redacted.value(options.apiKey))))));
+	}) })).pipe(Layer.provide(HttpApiMiddleware.layerClient(ApiKeyAuthorization, ({ next, request }) => next(Option.match(options.apiKey, {
+		onNone: () => request,
+		onSome: (apiKey) => HttpClientRequest.bearerToken(request, Redacted.value(apiKey))
+	})))));
 };
-//#endregion
-//#region ../env/src/env.ts
+// ../env/src/env.ts
 const Secret = Schema.Redacted(Schema.NonEmptyString);
 const StripeKey = (prefix) => Schema.Redacted(Schema.String.check(Schema.isStartingWith(prefix)));
 const Port = Schema.Int.check(Schema.isBetween({
@@ -846,7 +1212,7 @@ const LOOPBACK_HOSTS = /* @__PURE__ */ new Set([
 	"127.0.0.1",
 	"[::1]"
 ]);
-const TRACKS_API_PATH = "/api";
+const TRACKS_PUBLIC_API_URL = `https://tracks.alpic.ai/api`;
 const SecureUrl = Schema.URLFromString.check(Schema.makeFilter((url) => url.protocol === "https:" || LOOPBACK_HOSTS.has(url.hostname) || "must use https"));
 const ENCRYPTION_KEY_BYTES = 32;
 const EncryptionKey = Schema.Redacted(Schema.NonEmptyString.check(Schema.makeFilter((key) => Buffer.from(key, "base64").length === ENCRYPTION_KEY_BYTES || `must be ${ENCRYPTION_KEY_BYTES} random bytes in base64: openssl rand -base64 ${ENCRYPTION_KEY_BYTES}`)));
@@ -871,16 +1237,18 @@ const Secrets = Schema.Struct({
 	TYPESAFE_API_KEY: Secret,
 	DATABASE_PASSWORD: Secret,
 	CONNECTION_ENCRYPTION_KEY: EncryptionKey,
+	TRAFFIC_ENCRYPTION_KEY: EncryptionKey,
 	STRIPE_SECRET_KEY: StripeKey("rk_"),
 	STRIPE_WEBHOOK_SECRET: StripeKey("whsec_"),
 	IDENTITY_PROVIDER_CLIENT_ID: Secret,
 	IDENTITY_PROVIDER_CLIENT_SECRET: Secret,
 	SESSION_SECRET: Secret,
+	VERCEL_CLIENT_SECRET: Secret,
 	SENTRY_DSN: Schema.optionalKey(Schema.Redacted(SentryDsn))
 });
 const Variables = Schema.Struct({
 	PORT: Port.pipe(Schema.withDecodingDefaultKey(Effect.succeed(3e3))),
-	TRACKS_API_URL: Schema.URLFromString.pipe(Schema.withDecodingDefaultKey(Effect.succeed(`https://agent-runner.lab.alpic.ai${TRACKS_API_PATH}`))),
+	TRACKS_API_URL: Schema.URLFromString.pipe(Schema.withDecodingDefaultKey(Effect.succeed(TRACKS_PUBLIC_API_URL))),
 	DAYTONA_API_URL: Schema.NonEmptyString.pipe(Schema.withDecodingDefaultKey(Effect.succeed("https://app.daytona.io/api"))),
 	AGENT_RUNNER_URL: SecureUrl,
 	FILES_BUCKET: Schema.NonEmptyString,
@@ -904,7 +1272,12 @@ const Variables = Schema.Struct({
 	DEPLOYMENT_ENVIRONMENT: Schema.NonEmptyString.pipe(Schema.withDecodingDefaultKey(Effect.succeed("local"))),
 	OTLP_METRICS_URL: Schema.optionalKey(Schema.URLFromString),
 	RELEASE: Schema.optionalKey(Schema.NonEmptyString),
-	IDENTITY_PROVIDER_URL: SecureUrl.pipe(Schema.withDecodingDefaultKey(Effect.succeed("https://id.alpic.ai")))
+	POSTHOG_KEY: Schema.optionalKey(Schema.NonEmptyString),
+	POSTHOG_HOST: SecureUrl.pipe(Schema.withDecodingDefaultKey(Effect.succeed("https://us.i.posthog.com"))),
+	CLOUDFLARE_CLIENT_ID: Schema.NonEmptyString,
+	VERCEL_CLIENT_ID: Schema.NonEmptyString,
+	VERCEL_INTEGRATION_SLUG: Schema.NonEmptyString,
+	IDENTITY_PROVIDER_URL: SecureUrl.pipe(Schema.withDecodingDefaultKey(Effect.succeed("https://idp.alpic.ai")))
 });
 const Env = Schema.Struct({
 	...Secrets.fields,
@@ -940,6 +1313,13 @@ const IDENTITY_ENV = [
 	"IDENTITY_PROVIDER_CLIENT_SECRET",
 	"SESSION_SECRET"
 ];
+const TRAFFIC_ENV = [
+	"CLOUDFLARE_CLIENT_ID",
+	"VERCEL_CLIENT_ID",
+	"VERCEL_INTEGRATION_SLUG",
+	"VERCEL_CLIENT_SECRET",
+	"TRAFFIC_ENCRYPTION_KEY"
+];
 const AGENT_RUNNER_ENV = [
 	"FILES_BUCKET",
 	"FILES_ENDPOINT",
@@ -952,6 +1332,8 @@ const AGENT_RUNNER_ENV = [
 const LOG_ENV = ["LOG_FORMAT", "LOG_LEVEL"];
 const OBSERVABILITY_ENV = [
 	"SENTRY_DSN",
+	"POSTHOG_KEY",
+	"POSTHOG_HOST",
 	"DEPLOYMENT_ENVIRONMENT",
 	"OTLP_METRICS_URL",
 	"RELEASE"
@@ -964,17 +1346,15 @@ const SERVICE_ENV = [
 	...OBSERVABILITY_ENV,
 	...AGENT_RUNNER_ENV,
 	...PAYMENTS_ENV,
-	...IDENTITY_ENV
+	...IDENTITY_ENV,
+	...TRAFFIC_ENV
 ];
-const CLI_ENV = ["TRACKS_API_KEY", "TRACKS_API_URL"];
 const isDatabaseVariable = (name) => DATABASE_ENV.some((databaseName) => databaseName === name);
 SERVICE_ENV.flatMap((name) => isSecret(name) && !isDatabaseVariable(name) ? [name] : []);
 const readEnv = (...names) => Config.schema(Env.mapFields(Struct.pick(names)));
-//#endregion
-//#region package.json
-var version = "0.6.0";
-//#endregion
-//#region src/connections.ts
+// package.json
+var version = "0.9.1";
+// src/connections.ts
 const encodeConnection = Schema.encodeEffect(Schema.fromJsonString(Connection, { space: 2 }));
 const encodeConnections = Schema.encodeEffect(Schema.fromJsonString(List(Connection), { space: 2 }));
 const connectionId = Argument.String("id").pipe(Argument.withDescription("Connection id"), Argument.withSchema(ConnectionId));
@@ -1037,7 +1417,7 @@ const confirmationOf = (params, id) => {
 		onNone: () => Effect.fail(new ConsentFailed({ reason: params.get("error") ?? "no confirmation came back" }))
 	});
 };
-const create$1 = Command.make("create", { server: Argument.String("server-url").pipe(Argument.withDescription("URL of the MCP server, in https")) }, Effect.fn("connections.create")(function* ({ server }) {
+const create$2 = Command.make("create", { server: Argument.String("server-url").pipe(Argument.withDescription("URL of the MCP server, in https")) }, Effect.fn("connections.create")(function* ({ server }) {
 	const client = yield* ApiClient;
 	const callback = yield* loopbackCallback;
 	const created = yield* client.connections.create({ payload: {
@@ -1057,11 +1437,11 @@ const create$1 = Command.make("create", { server: Argument.String("server-url").
 	});
 	yield* Console.log(yield* encodeConnection(connection));
 }, Effect.scoped)).pipe(Command.withDescription("Connect to an MCP server as one of its users: consent in your browser, then use the connection in runs"));
-const list = Command.make("list", {}, Effect.fn("connections.list")(function* () {
+const list$2 = Command.make("list", {}, Effect.fn("connections.list")(function* () {
 	const client = yield* ApiClient;
 	yield* Console.log(yield* encodeConnections(yield* client.connections.list()));
 })).pipe(Command.withDescription("List connections"));
-const get$1 = Command.make("get", { id: connectionId }, Effect.fn("connections.get")(function* ({ id }) {
+const get$2 = Command.make("get", { id: connectionId }, Effect.fn("connections.get")(function* ({ id }) {
 	const client = yield* ApiClient;
 	yield* Console.log(yield* encodeConnection(yield* client.connections.get({ params: { id } })));
 })).pipe(Command.withDescription("Show a connection"));
@@ -1069,13 +1449,150 @@ const deleteConnection = Command.make("delete", { id: connectionId }, Effect.fn(
 	yield* (yield* ApiClient).connections.delete({ params: { id } });
 })).pipe(Command.withDescription("Delete a connection and the access it holds"));
 const connections = Command.make("connections").pipe(Command.withDescription("Manage connections to MCP servers"), Command.withSubcommands([
+	create$2,
+	list$2,
+	get$2,
+	deleteConnection
+]));
+// src/experiments.ts
+const encodeExperiment = Schema.encodeEffect(Schema.fromJsonString(Experiment, { space: 2 }));
+const encodeExperiments = Schema.encodeEffect(Schema.fromJsonString(List(Experiment), { space: 2 }));
+const encodeExecution = Schema.encodeEffect(Schema.fromJsonString(Execution, { space: 2 }));
+const encodeExecutions = Schema.encodeEffect(Schema.fromJsonString(List(Execution), { space: 2 }));
+const encodeSlots = Schema.encodeEffect(Schema.fromJsonString(List(Slot), { space: 2 }));
+const experimentId = Argument.String("id").pipe(Argument.withDescription("Experiment id"), Argument.withSchema(ExperimentId));
+const executionId = Argument.String("id").pipe(Argument.withDescription("Execution id"), Argument.withSchema(ExecutionId));
+const definition = Argument.FileSchema("file", ExperimentInput).pipe(Argument.withDescription("JSON file of the experiment, as `tracks experiments get` prints it, without its id"));
+const POLL_INTERVAL = "10 seconds";
+const POLL_RETRIES = {
+	schedule: Schedule.exponential("2 seconds").pipe(Schedule.jittered),
+	times: 5
+};
+var PassRateTooLow = class extends Schema.TaggedError()("PassRateTooLow", {
+	passRate: Schema.Option(Schema.Number),
+	minimum: Schema.Number
+}) {
+	get message() {
+		const reached = Option.match(this.passRate, {
+			onNone: () => "no run was judged",
+			onSome: (rate) => `${rate}%`
+		});
+		return `The execution's pass rate is below ${this.minimum}%: ${reached}`;
+	}
+};
+const create$1 = Command.make("create", { definition }, Effect.fn("experiments.create")(function* ({ definition: payload }) {
+	const client = yield* ApiClient;
+	yield* Console.log(yield* encodeExperiment(yield* client.experiments.create({ payload })));
+})).pipe(Command.withDescription("Create an experiment from a JSON file"));
+const list$1 = Command.make("list", {}, Effect.fn("experiments.list")(function* () {
+	const client = yield* ApiClient;
+	yield* Console.log(yield* encodeExperiments(yield* client.experiments.list()));
+})).pipe(Command.withDescription("List experiments"));
+const get$1 = Command.make("get", { id: experimentId }, Effect.fn("experiments.get")(function* ({ id }) {
+	const client = yield* ApiClient;
+	yield* Console.log(yield* encodeExperiment(yield* client.experiments.get({ params: { id } })));
+})).pipe(Command.withDescription("Show an experiment"));
+const update = Command.make("update", {
+	id: experimentId,
+	definition
+}, Effect.fn("experiments.update")(function* ({ id, definition: payload }) {
+	const client = yield* ApiClient;
+	yield* Console.log(yield* encodeExperiment(yield* client.experiments.update({
+		params: { id },
+		payload
+	})));
+})).pipe(Command.withDescription("Replace a whole experiment with a JSON file"));
+const deleteExperiment = Command.make("delete", { id: experimentId }, Effect.fn("experiments.delete")(function* ({ id }) {
+	yield* (yield* ApiClient).experiments.delete({ params: { id } });
+})).pipe(Command.withDescription("Delete an experiment; its executions and their results stay"));
+const launch = Command.make("launch", {
+	id: experimentId,
+	wait: Flag.Boolean("wait").pipe(Flag.withAlias("w"), Flag.withDescription(`Wait until no run is pending, checking every ${POLL_INTERVAL}, then print the execution`), Flag.withDefault(false)),
+	minPassRate: Flag.Int("min-pass-rate").pipe(Flag.withSchema(Schema.Int.check(Schema.isBetween({
+		minimum: 0,
+		maximum: 100
+	}))), Flag.withDescription("Wait, then exit with code 2 when the pass rate ends below this percentage, from 0 to 100"), Flag.optional)
+}, Effect.fn("experiments.launch")(function* ({ id, wait, minPassRate }) {
+	const client = yield* ApiClient;
+	const launched = yield* client.executions.launch({ params: { id } });
+	if (!wait && Option.isNone(minPassRate)) return yield* Console.log(yield* encodeExecution(launched));
+	yield* Console.error(`Launched execution ${launched.id}: ${launched.slots.total} runs`);
+	const done = yield* client.executions.get({ params: { id: launched.id } }).pipe(Effect.retry({
+		...POLL_RETRIES,
+		while: (error) => error._tag === "HttpClientError" && error.reason._tag === "TransportError"
+	}), Effect.tap(({ slots }) => Console.error(`${slots.running} running, ${slots.judging} judging, ${slots.judged} judged`)), Effect.repeat({
+		schedule: Schedule.spaced(POLL_INTERVAL),
+		until: (execution) => !execution.pending
+	}), Effect.tapErrorTag("HttpClientError", () => Console.error(`Execution ${launched.id} keeps running: check it with tracks executions get ${launched.id}`)));
+	yield* Console.log(yield* encodeExecution(done));
+	if (Option.isSome(minPassRate) && !Option.exists(done.pass_rate, (rate) => rate >= minPassRate.value)) return yield* new PassRateTooLow({
+		passRate: done.pass_rate,
+		minimum: minPassRate.value
+	});
+})).pipe(Command.withDescription("Launch an experiment: start a run for every scenario, variant, agent and repetition"));
+const executionsOfExperiment = Command.make("executions", { id: experimentId }, Effect.fn("experiments.executions")(function* ({ id }) {
+	const client = yield* ApiClient;
+	yield* Console.log(yield* encodeExecutions(yield* client.executions.list({ params: { id } })));
+})).pipe(Command.withDescription("List an experiment's executions"));
+const experiments = Command.make("experiments").pipe(Command.withDescription("Compare agents on your scenarios"), Command.withSubcommands([
 	create$1,
-	list,
+	list$1,
 	get$1,
-	deleteConnection
+	update,
+	deleteExperiment,
+	launch,
+	executionsOfExperiment
+]));
+const getExecution = Command.make("get", { id: executionId }, Effect.fn("executions.get")(function* ({ id }) {
+	const client = yield* ApiClient;
+	yield* Console.log(yield* encodeExecution(yield* client.executions.get({ params: { id } })));
+})).pipe(Command.withDescription("Show an execution, where its runs stand and its pass rate"));
+const slots = Command.make("slots", { id: executionId }, Effect.fn("executions.slots")(function* ({ id }) {
+	const client = yield* ApiClient;
+	yield* Console.log(yield* encodeSlots(yield* client.executions.listSlots({ params: { id } })));
+})).pipe(Command.withDescription("List every run an execution plans, with its verdict once judged"));
+const cancel$1 = Command.make("cancel", { id: executionId }, Effect.fn("executions.cancel")(function* ({ id }) {
+	const client = yield* ApiClient;
+	yield* Console.log(yield* encodeExecution(yield* client.executions.cancel({ params: { id } })));
+})).pipe(Command.withDescription("Cancel an execution: start none of its runs left and cancel those started"));
+const executions = Command.make("executions").pipe(Command.withDescription("Follow and cancel experiments' executions"), Command.withSubcommands([
+	getExecution,
+	slots,
+	cancel$1
 ]));
-//#endregion
-//#region src/runs.ts
+// src/library.ts
+const encodeLibrary = Schema.encodeEffect(Schema.fromJsonString(Library, { space: 2 }));
+const library = Command.make("library", {}, Effect.fn("library")(function* () {
+	const client = yield* ApiClient;
+	yield* Console.log(yield* encodeLibrary(yield* client.library.get()));
+})).pipe(Command.withDescription("List the repositories, skills, MCP servers and command-line tools experiments can pick"));
+// src/files.ts
+const CONTENT_TYPES = {
+	".png": "image/png",
+	".jpg": "image/jpeg",
+	".jpeg": "image/jpeg",
+	".webp": "image/webp",
+	".gif": "image/gif",
+	".pdf": "application/pdf",
+	".txt": "text/plain",
+	".md": "text/markdown",
+	".csv": "text/csv",
+	".json": "application/json"
+};
+const uploadFile = Effect.fn("uploadFile")(function* (localPath) {
+	const fileSystem = yield* FileSystem.FileSystem;
+	const path = yield* Path.Path;
+	const contentType = CONTENT_TYPES[path.extname(localPath).toLowerCase()] ?? "application/octet-stream";
+	const bytes = yield* fileSystem.readFile(localPath);
+	const upload = yield* (yield* ApiClient).files.createUpload({ payload: {
+		name: path.basename(localPath),
+		content_type: contentType,
+		size: bytes.byteLength
+	} });
+	yield* (yield* HttpClient.HttpClient).execute(HttpClientRequest.put(upload.url).pipe(HttpClientRequest.bodyUint8Array(bytes, contentType), HttpClientRequest.setHeaders(upload.headers))).pipe(Effect.flatMap(HttpClientResponse.filterStatusOk));
+	return upload.id;
+});
+// src/runs.ts
 const encodeRun = Schema.encodeEffect(Schema.fromJsonString(Run, { space: 2 }));
 const encodeFrames = Schema.encodeEffect(Schema.fromJsonString(List(Frame), { space: 2 }));
 const encodeEvents = Schema.encodeEffect(Schema.fromJsonString(List(Event), { space: 2 }));
@@ -1120,6 +1637,7 @@ const fixedHeaderOf = (entry) => {
 		value: entry.slice(colon + 1).trim()
 	});
 };
+const labelFlag = (description) => Flag.String("label").pipe(Flag.withDescription(description), Flag.atLeast(0), Flag.withSchema(LabelFilters));
 var FixedHeaderUnmatched = class extends Schema.TaggedError()("FixedHeaderUnmatched", { server: Schema.String }) {
 	get message() {
 		return `--mcp-header ${this.server}=… needs an --mcp ${this.server}=URL`;
@@ -1129,6 +1647,7 @@ const create = Command.make("create", {
 	prompt: Argument.String("prompt").pipe(Argument.withDescription("Prompt sent to the agent")),
 	harness: Flag.Literals("harness", Harness.literals).pipe(Flag.withAlias("H"), Flag.withDescription("Harness running the agent")),
 	model: Flag.String("model").pipe(Flag.withAlias("m"), Flag.withDescription("Model used by the harness")),
+	file: Flag.File("file", { mustExist: true }).pipe(Flag.withAlias("f"), Flag.withDescription("Local file uploaded and placed in the sandbox before setup, its path given to the agent with the prompt; repeat for several"), Flag.atLeast(0)),
 	repository: Flag.String("repository").pipe(Flag.withAlias("r"), Flag.withDescription("Git repository cloned into the sandbox"), Flag.optional),
 	env: Flag.String("env").pipe(Flag.withAlias("e"), Flag.withDescription("Environment variable set in the agent's processes, as KEY=VALUE, or KEY to pass its local value"), Flag.filterMap(variableOf, (entry) => `${entry} is not set in the local environment`), Flag.atLeast(0)),
 	setup: Flag.String("setup").pipe(Flag.withDescription("Shell script run in the sandbox after checkout, before the first prompt; from a file: --setup \"$(cat setup.sh)\""), Flag.optional),
@@ -1143,12 +1662,14 @@ const create = Command.make("create", {
 		})),
 		onSome: Effect.succeed
 	})), Flag.atLeast(0)),
+	browser: Flag.Boolean("browser").pipe(Flag.withDescription("Give the agent a browser, a headless Chromium in its sandbox whose pages can offer it tools through WebMCP. The Codex app and Claude Cowork always use their own browser instead"), Flag.withDefault(false)),
 	budget: Flag.Finite("budget").pipe(Flag.withAlias("b"), Flag.withDescription("Most the run may spend, in US dollars"), Flag.optional),
 	oneShot: Flag.Boolean("one-shot").pipe(Flag.withDescription("End the run once its first prompt finishes (default); --no-one-shot waits for follow-ups"), Flag.optional),
 	idleTimeout: Flag.Int("idle-timeout").pipe(Flag.withDescription("Seconds an idle run waits for a follow-up before it ends"), Flag.optional),
 	timeLimit: Flag.Int("time-limit").pipe(Flag.withDescription("Longest the run may last, in seconds, before it is stopped"), Flag.optional),
-	idempotencyKey: Flag.String("idempotency-key").pipe(Flag.withDescription("Retry safely: running again with the same key returns the run already created"), Flag.optional)
-}, Effect.fn("runs.create")(function* ({ prompt, harness, model, repository, env, setup, teardown, mcp, mcpHeader, budget, oneShot, idleTimeout, timeLimit, idempotencyKey }) {
+	idempotencyKey: Flag.String("idempotency-key").pipe(Flag.withDescription("Retry safely: running again with the same key returns the run already created"), Flag.optional),
+	label: labelFlag("Label attached to the run to find it again, as KEY:VALUE, such as ticket:ENG-42; repeat for several")
+}, Effect.fn("runs.create")(function* ({ prompt, harness, model, file, repository, env, setup, teardown, mcp, mcpHeader, browser, budget, oneShot, idleTimeout, timeLimit, idempotencyKey, label }) {
 	const unmatched = mcpHeader.find((header) => !mcp.some((server) => server.url !== void 0 && server.name === header.server));
 	if (unmatched !== void 0) return yield* new FixedHeaderUnmatched({ server: unmatched.server });
 	const mcpServers = mcp.map((server) => {
@@ -1159,6 +1680,7 @@ const create = Command.make("create", {
 			headers: Option.some(Object.fromEntries(headers.map(({ name, value }) => [name, value])))
 		};
 	});
+	const inputFiles = yield* Effect.forEach(file, uploadFile, { concurrency: "unbounded" });
 	const run = yield* (yield* ApiClient).runs.create({
 		headers: { "idempotency-key": idempotencyKey },
 		payload: {
@@ -1167,22 +1689,38 @@ const create = Command.make("create", {
 				model
 			},
 			prompt,
+			input_files: inputFiles,
 			environment: {
 				repository,
 				variables: Option.some(Object.fromEntries(env.map(({ name, value }) => [name, value]))),
 				setup,
 				teardown,
 				skills: Option.none(),
-				mcp_servers: Option.some(mcpServers)
+				mcp_servers: Option.some(mcpServers),
+				browser
 			},
 			budget_usd: budget,
 			one_shot: oneShot,
 			idle_timeout_seconds: idleTimeout,
-			time_limit_seconds: timeLimit
+			time_limit_seconds: timeLimit,
+			labels: label
 		}
 	});
 	yield* Console.log(yield* encodeRun(run));
 })).pipe(Command.withDescription("Start a run"));
+const encodeRuns = Schema.encodeEffect(Schema.fromJsonString(List(Run), { space: 2 }));
+const list = Command.make("list", {
+	label: labelFlag("Only runs with this label, as KEY:VALUE; repeat to require several"),
+	after: Flag.String("after").pipe(Flag.withAlias("a"), Flag.withDescription("Only runs listed after the run with this id"), Flag.withSchema(RunId), Flag.optional),
+	limit: Flag.Int("limit").pipe(Flag.withAlias("l"), Flag.withDescription("Maximum runs returned"), Flag.optional)
+}, Effect.fn("runs.list")(function* ({ label, after, limit }) {
+	const page = yield* (yield* ApiClient).runs.list({ query: {
+		label: Object.keys(label).length === 0 ? Option.none() : Option.some(label),
+		after,
+		limit
+	} });
+	yield* Console.log(yield* encodeRuns(page));
+})).pipe(Command.withDescription("List your runs, most recently started first"));
 const get = Command.make("get", { id: runId }, Effect.fn("runs.get")(function* ({ id }) {
 	const run = yield* (yield* ApiClient).runs.get({ params: { id } });
 	yield* Console.log(yield* encodeRun(run));
@@ -1224,6 +1762,14 @@ const events = Command.make("events", {
 	yield* Console.log(yield* encodeEvents(page));
 })).pipe(Command.withDescription("List a run's events"));
 const printed = (sent, itemOf, encode) => sent.pipe(Stream.filterMap(itemOf), Stream.tap((item) => Effect.flatMap(encode(item), Console.log)), Stream.map((item) => item.seq), Stream.runLast);
+var RunUnfinished = class extends Schema.TaggedError()("RunUnfinished", {
+	id: RunId,
+	endReason: Schema.String
+}) {
+	get message() {
+		return `Run ${this.id} ended with ${this.endReason} before its agent finished`;
+	}
+};
 const follow = Command.make("follow", {
 	id: runId,
 	after: Flag.Int("after").pipe(Flag.withAlias("a"), Flag.withDescription("Only frames, or events with --events, with a greater seq"), Flag.optional),
@@ -1240,9 +1786,15 @@ const follow = Command.make("follow", {
 	let lastSeen = after;
 	while (true) {
 		lastSeen = Option.orElse(yield* followOnce(lastSeen), () => lastSeen);
-		if ((yield* client.runs.get({ params: { id } })).status === "ended") return;
+		const run = yield* client.runs.get({ params: { id } });
+		if (run.status !== "ended") continue;
+		if (run.end_reason === "completed" || run.end_reason === "idle_timeout") return;
+		return yield* new RunUnfinished({
+			id,
+			endReason: run.end_reason
+		});
 	}
-})).pipe(Command.withDescription("Print a run's frames, or its events with --events, as they come, one JSON object per line, until the run ends"));
+})).pipe(Command.withDescription("Print a run's frames, or its events with --events, as they come, one JSON object per line, until the run ends; exit with code 2 when it ended before its agent finished"));
 const sendPrompt = Command.make("send-prompt", {
 	id: runId,
 	text: Argument.String("text").pipe(Argument.withDescription("Follow-up prompt sent to the agent")),
@@ -1281,6 +1833,7 @@ const cancel = Command.make("cancel", { id: runId }, Effect.fn("runs.cancel")(fu
 })).pipe(Command.withDescription("Stop a run right away, even mid-prompt"));
 const runs = Command.make("runs").pipe(Command.withDescription("Manage runs"), Command.withSubcommands([
 	create,
+	list,
 	get,
 	sendPrompt,
 	prompts,
@@ -1291,27 +1844,36 @@ const runs = Command.make("runs").pipe(Command.withDescription("Manage runs"), C
 	end,
 	cancel
 ]));
-//#endregion
-//#region src/main.ts
+// src/tracks.ts
 const health = Command.make("health", {}, Effect.fn("health")(function* () {
 	yield* (yield* ApiClient).health();
 	yield* Console.log("ok");
 })).pipe(Command.withDescription("Check that the API is reachable"));
-const cli = Command.make("tracks").pipe(Command.withDescription("Run coding agents in sandboxes"), Command.withSubcommands([
+const tracks = Command.make("tracks").pipe(Command.withDescription("Run coding agents in sandboxes and compare them on your scenarios"), Command.withSubcommands([
 	health,
 	runs,
-	connections
+	connections,
+	experiments,
+	executions,
+	library
 ]));
-const ApiClientLayer = Layer.unwrap(Effect.map(readEnv(...CLI_ENV), (env) => ApiClient.layer({
-	baseUrl: env.TRACKS_API_URL,
-	apiKey: env.TRACKS_API_KEY,
-	userAgent: `${CLI_USER_AGENT}/${version}`
-}))).pipe(Layer.provide(FetchHttpClient.layer));
-const MainLayer = Layer.mergeAll(ApiClientLayer, NodeServices.layer);
-Command.run(cli, { version }).pipe(Effect.catch((error) => {
+// src/main.ts
+const apiKey = Effect.option(Effect.gen(function* () {
+	const { TRACKS_API_KEY } = yield* readEnv("TRACKS_API_KEY");
+	return TRACKS_API_KEY;
+}));
+const ApiClientLayer = Layer.unwrap(Effect.gen(function* () {
+	const { TRACKS_API_URL } = yield* readEnv("TRACKS_API_URL");
+	return ApiClient.layer({
+		baseUrl: TRACKS_API_URL,
+		apiKey: yield* apiKey,
+		userAgent: `${CLI_USER_AGENT}/${version}`
+	});
+}));
+const MainLayer = Layer.mergeAll(ApiClientLayer, NodeServices.layer).pipe(Layer.provideMerge(FetchHttpClient.layer));
+Command.run(tracks, { version }).pipe(Effect.catch((error) => {
 	const explainedByHelp = error._tag === "ShowHelp";
 	const line = "message" in error ? `${error._tag}: ${error.message}` : error._tag;
-	return Effect.andThen(explainedByHelp ? Effect.void : Console.error(line), Effect.sync(() => void (process.exitCode = 1)));
+	return Effect.andThen(explainedByHelp ? Effect.void : Console.error(line), Effect.sync(() => void (process.exitCode = error._tag === "RunUnfinished" || error._tag === "PassRateTooLow" ? 2 : 1)));
 }), Effect.provide(MainLayer), NodeRuntime.runMain);
-//#endregion
 export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@alpic-ai/tracks",
-  "version": "0.6.0",
+  "version": "0.9.1",
   "description": "Run coding agents in sandboxes from the command line",
   "license": "UNLICENSED",
   "repository": {
@@ -20,19 +20,22 @@
     "access": "public"
   },
   "dependencies": {
-    "@effect/platform-node": "^4.0.2",
-    "effect": "^4.0.2"
+    "@effect/platform-node": "^4.0.7",
+    "effect": "^4.0.7"
   },
   "devDependencies": {
+    "@effect/vitest": "^4.0.7",
     "@tracks/contract": "0.0.0",
+    "@tracks/docs": "0.0.0",
     "@tracks/env": "0.0.0",
-    "rolldown": "^1.2.13"
+    "rolldown": "^1.2.15"
   },
   "engines": {
     "node": ">=24"
   },
   "scripts": {
     "start": "node --import @tracks/env/preload src/main.ts",
-    "build": "rolldown --config rolldown.config.ts"
+    "build": "rolldown --config rolldown.config.ts",
+    "docs:generate": "node scripts/docs.ts"
   }
 }