@alpic-ai/tracks 0.7.1 → 0.10.0

package/dist/main.js

@@ -181,7 +181,10 @@ var FilesApi = class extends HttpApiGroup.make("files").add(HttpApiEndpoint.post
 }).annotateMerge(OpenApi.annotations({
 	summary: "Request a file upload",
 	description: "Request with `name`, `content_type` and exact byte `size`, e.g. `{\"name\":\"photo.png\",\"content_type\":\"image/png\",\"size\":12345}`. Any type of file, up to 20 MiB each. PUT raw bytes (not JSON or base64) to the returned `url` with every returned header within 15 minutes; browsers set `content-length` from the body. Uploads are write-once; repeating the PUT returns 412. Pass the returned `id` in `input_files` when creating a run. Files in the landing zone expire after 24 hours and are deleted asynchronously. Files attached to a run are retained permanently and their references can be reused by your organization."
-}))).middleware(ApiKeyAuthorization).middleware(RequestValidation) {};
+}))).annotateMerge(OpenApi.annotations({
+	title: "Files",
+	description: "Upload files to place in a run's sandbox."
+})).middleware(ApiKeyAuthorization).middleware(RequestValidation) {};
 // ../contract/src/runs.ts
 const RunId = Schema.String.pipe(Schema.brand("RunId")).annotate({
 	identifier: "RunId",
@@ -321,14 +324,23 @@ const SkillsInput = Schema.Array(SkillInput).check(Schema.isMaxLength(MAX_SKILLS
 	path: [index, "name"],
 	issue: "Expected a name not reserved for the agent"
 }] : [])), Schema.makeFilter((skills) => skills.flatMap((skill) => skill.files).reduce((total, file) => total + encoder.encode(file.content).length, 0) <= MAX_SKILLS_BYTES || `Expected at most ${MAX_SKILLS_BYTES / 1024 / 1024} MB of skills together`)).annotate({ description: `Skills the agent can load, at most ${MAX_SKILLS} and ${MAX_SKILLS_BYTES / 1024 / 1024} MB together.` });
+const BROWSER_MCP_SERVER = "tracks-browser";
 const EnvironmentInput = Schema.Struct({
 	repository,
 	variables: Schema.OptionFromOptionalKey(Variables$1),
 	setup,
 	teardown,
 	skills: Schema.OptionFromOptionalKey(SkillsInput),
-	mcp_servers: Schema.OptionFromOptionalKey(McpServersInput)
-}).annotate({
+	mcp_servers: Schema.OptionFromOptionalKey(McpServersInput),
+	browser: Schema.Boolean.pipe(Schema.withDecodingDefaultKey(Effect.succeed(false))).annotate({ description: "Whether the agent gets a browser: a headless Chromium in its sandbox, driven through an MCP server named `tracks-browser`, whose pages can offer it tools through WebMCP. Screenshots of the pages it visits are kept with the run. Ignored by the Codex app and Claude Cowork, which always use their own browser. Defaults to false." })
+}).check(Schema.makeFilter((environment) => !environment.browser ? [] : Option.getOrElse(environment.mcp_servers, () => []).flatMap((server, index) => server.name === BROWSER_MCP_SERVER ? [{
+	path: [
+		"mcp_servers",
+		index,
+		"name"
+	],
+	issue: `Expected a name other than ${BROWSER_MCP_SERVER}, the browser's MCP server`
+}] : []))).annotate({
 	identifier: "EnvironmentInput",
 	description: "What the agent works on."
 });
@@ -354,7 +366,8 @@ const RunEnvironment = Schema.Struct({
 	setup,
 	teardown,
 	skills: Schema.Array(RunSkill),
-	mcp_servers: Schema.Array(RunMcpServer)
+	mcp_servers: Schema.Array(RunMcpServer),
+	browser: Schema.Boolean.annotate({ description: "Whether the agent was given a headless Chromium in its sandbox." })
 }).annotate({
 	identifier: "Environment",
 	description: "What the agent works on."
@@ -454,13 +467,15 @@ const EndReason = Schema.Literals([
 const FailureKind = Schema.Literals([
 	"repository_checkout_failed",
 	"setup_failed",
+	"browser_not_started",
 	"sandbox_not_found",
 	"process_lost",
 	"sandbox_unavailable",
+	"agent_stopped",
 	"unexpected"
 ]).annotate({
 	identifier: "FailureKind",
-	description: "What kind of thing went wrong: the repository could not be checked out, the environment's setup command failed, the sandbox disappeared, the agent's process was lost, the sandbox provider failed, or something unexpected."
+	description: "What kind of thing went wrong: the repository could not be checked out, the environment's setup command failed, the browser did not start, the sandbox disappeared, the agent's process was lost, the sandbox provider failed, the agent of a run that accepts no follow-ups stopped with an error, or something unexpected."
 });
 const runFields = {
 	id: RunId,
@@ -519,6 +534,10 @@ var RunNotFound = class extends Schema.TaggedError()("RunNotFound", { message: S
 	httpApiStatus: 404,
 	description: "No run with this id exists in your organization."
 }) {};
+var ScreenshotNotFound = class extends Schema.TaggedError()("ScreenshotNotFound", { message: Schema.String }, {
+	httpApiStatus: 404,
+	description: "This event has no screenshot."
+}) {};
 var ConnectionUnusable = class extends Schema.TaggedError()("ConnectionUnusable", { message: Schema.String }, {
 	httpApiStatus: 422,
 	description: "A `connection_id` in `environment.mcp_servers` does not exist or is not `active`."
@@ -567,18 +586,23 @@ const Frame = Schema.Struct({
 	run_id: RunId,
 	seq: Schema.Int.annotate({ description: "Position of the line in the run's output, starting at 1. Always increasing, but some numbers may be skipped." }),
 	prompt: PromptIndex.annotate({ description: "Index of the prompt the agent was working on." }),
+	process: Schema.String.annotate({ description: "Program in the sandbox that printed the line: `prompt-1`, `prompt-2`… for the agent working on each prompt, `browser` for its browser." }),
 	captured_at: Schema.DateTimeUtc.annotate({ description: "When the line was captured." }),
 	channel: Schema.Literals(["stdout", "stderr"]).annotate({ description: "Output stream the line came from." }),
 	body: Schema.String.annotate({ description: "The line, exactly as the agent printed it." })
 }).annotate({
 	identifier: "Frame",
-	description: "One line of the agent's output."
+	description: "One line of output from the agent or its browser."
 });
 const Outcome = Schema.Literals(["succeeded", "failed"]).annotate({
 	identifier: "Outcome",
 	description: "Whether the tool call or subagent succeeded."
 });
 const ToolCallId = Schema.String.annotate({ description: "Identifier of the tool call, unique in its run." });
+const Screenshot = Schema.Struct({ page_url: Schema.String.annotate({ description: "Address of the page shown." }) }).annotate({
+	identifier: "Screenshot",
+	description: "An image of the page the agent's browser showed; get it from `GET /runs/{id}/events/{seq}/screenshot`, with this event's `seq`."
+});
 const eventFields = {
 	run_id: RunId,
 	seq: Schema.Int.annotate({ description: "Position of the event in the run, starting at 1. Always increasing, but some numbers may be skipped." }),
@@ -628,7 +652,8 @@ const Event = Schema.Union([
 		tool_call_id: ToolCallId,
 		outcome: Outcome,
 		output: Schema.String.annotate({ description: "What the tool returned, possibly truncated." }),
-		structured: Schema.OptionFromOptionalKey(Schema.Json.annotate({ description: "The structured content an MCP tool returned beside its text, unchanged. Absent when the tool returned none or it was too large." }))
+		structured: Schema.OptionFromOptionalKey(Schema.Json.annotate({ description: "The structured content an MCP tool returned beside its text, unchanged. Absent when the tool returned none or it was too large." })),
+		screenshot: Schema.OptionFromOptionalKey(Screenshot.annotate({ description: "The page a browser tool call left its browser on, as the harness captured it. Absent when it captured none." }))
 	}).annotate({
 		identifier: "ToolEndedEvent",
 		description: "A tool call ended; its `tool_call_id` is the one of the `tool_started` event."
@@ -649,6 +674,27 @@ const Event = Schema.Union([
 		identifier: "SubagentEndedEvent",
 		description: "A subagent ended."
 	}),
+	Schema.Struct({
+		...eventFields,
+		type: Schema.Literal("page_visited"),
+		tool_call_id: Schema.OptionFromOptionalKey(ToolCallId.annotate({ description: "The browser tool call that led to the page, when the agent had made one." })),
+		page_id: Schema.String.annotate({ description: "Identifier of the page in the run: a later visit to the same page keeps it." }),
+		version: Schema.Int.annotate({ description: "How many times the page was seen changed, this one included, starting at 1." }),
+		url: Schema.String.annotate({ description: "URL of the page." }),
+		title: Schema.String.annotate({ description: "Title of the page, empty when it has none." }),
+		reason: Schema.Literals([
+			"arrival",
+			"settled",
+			"after_submit",
+			"error",
+			"after_action",
+			"prompt_end"
+		]).annotate({ description: "Why it was captured: on arrival, once settled, after a form was submitted, on an error, after a browser action, or when the prompt ended." }),
+		screenshot: Schema.OptionFromOptionalKey(Screenshot.annotate({ description: "The page as the browser captured it. Absent when it captured none; its image may be missing when the sandbox was gone before it was kept." }))
+	}).annotate({
+		identifier: "PageVisitedEvent",
+		description: "The agent's browser showed a page, new or changed."
+	}),
 	Schema.Struct({
 		...eventFields,
 		type: Schema.Literal("stop"),
@@ -777,6 +823,16 @@ var RunsApi = class extends HttpApiGroup.make("runs").add(HttpApiEndpoint.post("
 }).annotateMerge(OpenApi.annotations({
 	summary: "List a run's events",
 	description: "Returns what the agent did, in order: each prompt, then its messages, reasoning, tool calls and subagents, then a `stop` once the agent stopped. Events read alike whatever the harness, and are read from the run's output: `frame_seqs` points at the lines each one comes from. To get the next page, pass the `seq` of the last event received as `after`."
+})), HttpApiEndpoint.get("getScreenshot", "/runs/:id/events/:seq/screenshot", {
+	params: {
+		id: RunId,
+		seq: Schema.Int.check(Schema.isGreaterThanOrEqualTo(1))
+	},
+	success: Schema.Uint8Array.pipe(HttpApiSchema.asUint8Array({ contentType: "image/jpeg" })),
+	error: [RunNotFound, ScreenshotNotFound]
+}).annotateMerge(OpenApi.annotations({
+	summary: "Get a screenshot",
+	description: "Returns the image of a page the agent's browser showed, for a `tool_ended` or `page_visited` event that has a `screenshot`. Its content type is the image's own, usually `image/jpeg`."
 })), HttpApiEndpoint.get("followFrames", "/runs/:id/frames/stream", {
 	params: { id: RunId },
 	query: { after: OptionalQueryParam(Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), "Send only lines whose `seq` is greater than this.") },
@@ -849,10 +905,11 @@ const AgentTools = Schema.Struct({
 	name: Schema.String.annotate({ description: "Short name of the agent tools, such as `No agent tools` for the baseline." }),
 	skills: libraryIds("skills"),
 	mcp_servers: libraryIds("MCP servers"),
-	command_line_tools: libraryIds("command-line tools")
+	command_line_tools: libraryIds("command-line tools"),
+	browser: Schema.Boolean.pipe(Schema.withDecodingDefaultKey(Effect.succeed(false))).annotate({ description: "Whether agents without a browser of their own get one, a headless Chromium in their sandbox whose pages can offer them tools through WebMCP. The Codex app and Claude Cowork always use their own browser. Defaults to false." })
 }).annotate({
 	identifier: "AgentTools",
-	description: "Skills, MCP servers and command-line tools given to the agent. Keep one with none as the baseline the others are compared to."
+	description: "Skills, MCP servers, command-line tools and a browser given to the agent. Keep one with none as the baseline the others are compared to."
 });
 const ExcludedVariant = Schema.Struct({
 	repository_id: Schema.OptionFromOptionalKey(Schema.String.annotate({ description: "Id of one of the experiment's repositories. Omit it for no repository." })),
@@ -943,7 +1000,7 @@ const Execution = Schema.Struct({
 		judging: Count.annotate({ description: "Ended runs waiting for their evaluation results." }),
 		judged: Count.annotate({ description: "Runs with a verdict." }),
 		passed: Count.annotate({ description: "Runs that passed their scenario." })
-	}).annotate({ description: "How many of its planned runs stand where." }),
+	}).annotate({ description: "How many of its slots, one per scenario, variant, agent and repetition, stand where." }),
 	pass_rate: Schema.OptionFromOptionalKey(Schema.Int.check(Schema.isBetween({
 		minimum: 0,
 		maximum: 100
@@ -1138,7 +1195,10 @@ var ApiClient = class ApiClient extends Context.Service()("@tracks/contract/ApiC
 	static layer = (options) => Layer.effect(ApiClient, HttpApiClient.make(TracksApi, { transformClient: HttpClient.mapRequest((request) => {
 		const located = HttpClientRequest.prependUrl(request, options.baseUrl.href);
 		return options.userAgent === void 0 ? located : HttpClientRequest.setHeader(located, "user-agent", options.userAgent);
-	}) })).pipe(Layer.provide(HttpApiMiddleware.layerClient(ApiKeyAuthorization, ({ next, request }) => next(HttpClientRequest.bearerToken(request, Redacted.value(options.apiKey))))));
+	}) })).pipe(Layer.provide(HttpApiMiddleware.layerClient(ApiKeyAuthorization, ({ next, request }) => next(Option.match(options.apiKey, {
+		onNone: () => request,
+		onSome: (apiKey) => HttpClientRequest.bearerToken(request, Redacted.value(apiKey))
+	})))));
 };
 // ../env/src/env.ts
 const Secret = Schema.Redacted(Schema.NonEmptyString);
@@ -1152,7 +1212,7 @@ const LOOPBACK_HOSTS = /* @__PURE__ */ new Set([
 	"127.0.0.1",
 	"[::1]"
 ]);
-const TRACKS_API_PATH = "/api";
+const TRACKS_PUBLIC_API_URL = `https://tracks.alpic.ai/api`;
 const SecureUrl = Schema.URLFromString.check(Schema.makeFilter((url) => url.protocol === "https:" || LOOPBACK_HOSTS.has(url.hostname) || "must use https"));
 const ENCRYPTION_KEY_BYTES = 32;
 const EncryptionKey = Schema.Redacted(Schema.NonEmptyString.check(Schema.makeFilter((key) => Buffer.from(key, "base64").length === ENCRYPTION_KEY_BYTES || `must be ${ENCRYPTION_KEY_BYTES} random bytes in base64: openssl rand -base64 ${ENCRYPTION_KEY_BYTES}`)));
@@ -1183,11 +1243,12 @@ const Secrets = Schema.Struct({
 	IDENTITY_PROVIDER_CLIENT_ID: Secret,
 	IDENTITY_PROVIDER_CLIENT_SECRET: Secret,
 	SESSION_SECRET: Secret,
+	VERCEL_CLIENT_SECRET: Secret,
 	SENTRY_DSN: Schema.optionalKey(Schema.Redacted(SentryDsn))
 });
 const Variables = Schema.Struct({
 	PORT: Port.pipe(Schema.withDecodingDefaultKey(Effect.succeed(3e3))),
-	TRACKS_API_URL: Schema.URLFromString.pipe(Schema.withDecodingDefaultKey(Effect.succeed(`https://agent-runner.lab.alpic.ai${TRACKS_API_PATH}`))),
+	TRACKS_API_URL: Schema.URLFromString.pipe(Schema.withDecodingDefaultKey(Effect.succeed(TRACKS_PUBLIC_API_URL))),
 	DAYTONA_API_URL: Schema.NonEmptyString.pipe(Schema.withDecodingDefaultKey(Effect.succeed("https://app.daytona.io/api"))),
 	AGENT_RUNNER_URL: SecureUrl,
 	FILES_BUCKET: Schema.NonEmptyString,
@@ -1214,6 +1275,8 @@ const Variables = Schema.Struct({
 	POSTHOG_KEY: Schema.optionalKey(Schema.NonEmptyString),
 	POSTHOG_HOST: SecureUrl.pipe(Schema.withDecodingDefaultKey(Effect.succeed("https://us.i.posthog.com"))),
 	CLOUDFLARE_CLIENT_ID: Schema.NonEmptyString,
+	VERCEL_CLIENT_ID: Schema.NonEmptyString,
+	VERCEL_INTEGRATION_SLUG: Schema.NonEmptyString,
 	IDENTITY_PROVIDER_URL: SecureUrl.pipe(Schema.withDecodingDefaultKey(Effect.succeed("https://idp.alpic.ai")))
 });
 const Env = Schema.Struct({
@@ -1250,7 +1313,13 @@ const IDENTITY_ENV = [
 	"IDENTITY_PROVIDER_CLIENT_SECRET",
 	"SESSION_SECRET"
 ];
-const TRAFFIC_ENV = ["CLOUDFLARE_CLIENT_ID", "TRAFFIC_ENCRYPTION_KEY"];
+const TRAFFIC_ENV = [
+	"CLOUDFLARE_CLIENT_ID",
+	"VERCEL_CLIENT_ID",
+	"VERCEL_INTEGRATION_SLUG",
+	"VERCEL_CLIENT_SECRET",
+	"TRAFFIC_ENCRYPTION_KEY"
+];
 const AGENT_RUNNER_ENV = [
 	"FILES_BUCKET",
 	"FILES_ENDPOINT",
@@ -1280,12 +1349,11 @@ const SERVICE_ENV = [
 	...IDENTITY_ENV,
 	...TRAFFIC_ENV
 ];
-const CLI_ENV = ["TRACKS_API_KEY", "TRACKS_API_URL"];
 const isDatabaseVariable = (name) => DATABASE_ENV.some((databaseName) => databaseName === name);
 SERVICE_ENV.flatMap((name) => isSecret(name) && !isDatabaseVariable(name) ? [name] : []);
 const readEnv = (...names) => Config.schema(Env.mapFields(Struct.pick(names)));
 // package.json
-var version = "0.7.1";
+var version = "0.10.0";
 // src/connections.ts
 const encodeConnection = Schema.encodeEffect(Schema.fromJsonString(Connection, { space: 2 }));
 const encodeConnections = Schema.encodeEffect(Schema.fromJsonString(List(Connection), { space: 2 }));
@@ -1369,7 +1437,7 @@ const create$2 = Command.make("create", { server: Argument.String("server-url").
 	});
 	yield* Console.log(yield* encodeConnection(connection));
 }, Effect.scoped)).pipe(Command.withDescription("Connect to an MCP server as one of its users: consent in your browser, then use the connection in runs"));
-const list$1 = Command.make("list", {}, Effect.fn("connections.list")(function* () {
+const list$2 = Command.make("list", {}, Effect.fn("connections.list")(function* () {
 	const client = yield* ApiClient;
 	yield* Console.log(yield* encodeConnections(yield* client.connections.list()));
 })).pipe(Command.withDescription("List connections"));
@@ -1382,7 +1450,7 @@ const deleteConnection = Command.make("delete", { id: connectionId }, Effect.fn(
 })).pipe(Command.withDescription("Delete a connection and the access it holds"));
 const connections = Command.make("connections").pipe(Command.withDescription("Manage connections to MCP servers"), Command.withSubcommands([
 	create$2,
-	list$1,
+	list$2,
 	get$2,
 	deleteConnection
 ]));
@@ -1396,6 +1464,10 @@ const experimentId = Argument.String("id").pipe(Argument.withDescription("Experi
 const executionId = Argument.String("id").pipe(Argument.withDescription("Execution id"), Argument.withSchema(ExecutionId));
 const definition = Argument.FileSchema("file", ExperimentInput).pipe(Argument.withDescription("JSON file of the experiment, as `tracks experiments get` prints it, without its id"));
 const POLL_INTERVAL = "10 seconds";
+const POLL_RETRIES = {
+	schedule: Schedule.exponential("2 seconds").pipe(Schedule.jittered),
+	times: 5
+};
 var PassRateTooLow = class extends Schema.TaggedError()("PassRateTooLow", {
 	passRate: Schema.Option(Schema.Number),
 	minimum: Schema.Number
@@ -1412,7 +1484,7 @@ const create$1 = Command.make("create", { definition }, Effect.fn("experiments.c
 	const client = yield* ApiClient;
 	yield* Console.log(yield* encodeExperiment(yield* client.experiments.create({ payload })));
 })).pipe(Command.withDescription("Create an experiment from a JSON file"));
-const list = Command.make("list", {}, Effect.fn("experiments.list")(function* () {
+const list$1 = Command.make("list", {}, Effect.fn("experiments.list")(function* () {
 	const client = yield* ApiClient;
 	yield* Console.log(yield* encodeExperiments(yield* client.experiments.list()));
 })).pipe(Command.withDescription("List experiments"));
@@ -1439,16 +1511,19 @@ const launch = Command.make("launch", {
 	minPassRate: Flag.Int("min-pass-rate").pipe(Flag.withSchema(Schema.Int.check(Schema.isBetween({
 		minimum: 0,
 		maximum: 100
-	}))), Flag.withDescription("Wait, then exit with an error when the pass rate ends below this percentage, from 0 to 100"), Flag.optional)
+	}))), Flag.withDescription("Wait, then exit with code 2 when the pass rate ends below this percentage, from 0 to 100"), Flag.optional)
 }, Effect.fn("experiments.launch")(function* ({ id, wait, minPassRate }) {
 	const client = yield* ApiClient;
 	const launched = yield* client.executions.launch({ params: { id } });
 	if (!wait && Option.isNone(minPassRate)) return yield* Console.log(yield* encodeExecution(launched));
 	yield* Console.error(`Launched execution ${launched.id}: ${launched.slots.total} runs`);
-	const done = yield* client.executions.get({ params: { id: launched.id } }).pipe(Effect.tap(({ slots }) => Console.error(`${slots.running} running, ${slots.judging} judging, ${slots.judged} judged`)), Effect.repeat({
+	const done = yield* client.executions.get({ params: { id: launched.id } }).pipe(Effect.retry({
+		...POLL_RETRIES,
+		while: (error) => error._tag === "HttpClientError" && error.reason._tag === "TransportError"
+	}), Effect.tap(({ slots }) => Console.error(`${slots.running} running, ${slots.judging} judging, ${slots.judged} judged`)), Effect.repeat({
 		schedule: Schedule.spaced(POLL_INTERVAL),
 		until: (execution) => !execution.pending
-	}));
+	}), Effect.tapErrorTag("HttpClientError", () => Console.error(`Execution ${launched.id} keeps running: check it with tracks executions get ${launched.id}`)));
 	yield* Console.log(yield* encodeExecution(done));
 	if (Option.isSome(minPassRate) && !Option.exists(done.pass_rate, (rate) => rate >= minPassRate.value)) return yield* new PassRateTooLow({
 		passRate: done.pass_rate,
@@ -1461,7 +1536,7 @@ const executionsOfExperiment = Command.make("executions", { id: experimentId },
 })).pipe(Command.withDescription("List an experiment's executions"));
 const experiments = Command.make("experiments").pipe(Command.withDescription("Compare agents on your scenarios"), Command.withSubcommands([
 	create$1,
-	list,
+	list$1,
 	get$1,
 	update,
 	deleteExperiment,
@@ -1562,6 +1637,7 @@ const fixedHeaderOf = (entry) => {
 		value: entry.slice(colon + 1).trim()
 	});
 };
+const labelFlag = (description) => Flag.String("label").pipe(Flag.withDescription(description), Flag.atLeast(0), Flag.withSchema(LabelFilters));
 var FixedHeaderUnmatched = class extends Schema.TaggedError()("FixedHeaderUnmatched", { server: Schema.String }) {
 	get message() {
 		return `--mcp-header ${this.server}=… needs an --mcp ${this.server}=URL`;
@@ -1586,12 +1662,14 @@ const create = Command.make("create", {
 		})),
 		onSome: Effect.succeed
 	})), Flag.atLeast(0)),
+	browser: Flag.Boolean("browser").pipe(Flag.withDescription("Give the agent a browser, a headless Chromium in its sandbox whose pages can offer it tools through WebMCP. The Codex app and Claude Cowork always use their own browser instead"), Flag.withDefault(false)),
 	budget: Flag.Finite("budget").pipe(Flag.withAlias("b"), Flag.withDescription("Most the run may spend, in US dollars"), Flag.optional),
 	oneShot: Flag.Boolean("one-shot").pipe(Flag.withDescription("End the run once its first prompt finishes (default); --no-one-shot waits for follow-ups"), Flag.optional),
 	idleTimeout: Flag.Int("idle-timeout").pipe(Flag.withDescription("Seconds an idle run waits for a follow-up before it ends"), Flag.optional),
 	timeLimit: Flag.Int("time-limit").pipe(Flag.withDescription("Longest the run may last, in seconds, before it is stopped"), Flag.optional),
-	idempotencyKey: Flag.String("idempotency-key").pipe(Flag.withDescription("Retry safely: running again with the same key returns the run already created"), Flag.optional)
-}, Effect.fn("runs.create")(function* ({ prompt, harness, model, file, repository, env, setup, teardown, mcp, mcpHeader, budget, oneShot, idleTimeout, timeLimit, idempotencyKey }) {
+	idempotencyKey: Flag.String("idempotency-key").pipe(Flag.withDescription("Retry safely: running again with the same key returns the run already created"), Flag.optional),
+	label: labelFlag("Label attached to the run to find it again, as KEY:VALUE, such as ticket:ENG-42; repeat for several")
+}, Effect.fn("runs.create")(function* ({ prompt, harness, model, file, repository, env, setup, teardown, mcp, mcpHeader, browser, budget, oneShot, idleTimeout, timeLimit, idempotencyKey, label }) {
 	const unmatched = mcpHeader.find((header) => !mcp.some((server) => server.url !== void 0 && server.name === header.server));
 	if (unmatched !== void 0) return yield* new FixedHeaderUnmatched({ server: unmatched.server });
 	const mcpServers = mcp.map((server) => {
@@ -1618,16 +1696,31 @@ const create = Command.make("create", {
 				setup,
 				teardown,
 				skills: Option.none(),
-				mcp_servers: Option.some(mcpServers)
+				mcp_servers: Option.some(mcpServers),
+				browser
 			},
 			budget_usd: budget,
 			one_shot: oneShot,
 			idle_timeout_seconds: idleTimeout,
-			time_limit_seconds: timeLimit
+			time_limit_seconds: timeLimit,
+			labels: label
 		}
 	});
 	yield* Console.log(yield* encodeRun(run));
 })).pipe(Command.withDescription("Start a run"));
+const encodeRuns = Schema.encodeEffect(Schema.fromJsonString(List(Run), { space: 2 }));
+const list = Command.make("list", {
+	label: labelFlag("Only runs with this label, as KEY:VALUE; repeat to require several"),
+	after: Flag.String("after").pipe(Flag.withAlias("a"), Flag.withDescription("Only runs listed after the run with this id"), Flag.withSchema(RunId), Flag.optional),
+	limit: Flag.Int("limit").pipe(Flag.withAlias("l"), Flag.withDescription("Maximum runs returned"), Flag.optional)
+}, Effect.fn("runs.list")(function* ({ label, after, limit }) {
+	const page = yield* (yield* ApiClient).runs.list({ query: {
+		label: Object.keys(label).length === 0 ? Option.none() : Option.some(label),
+		after,
+		limit
+	} });
+	yield* Console.log(yield* encodeRuns(page));
+})).pipe(Command.withDescription("List your runs, most recently started first"));
 const get = Command.make("get", { id: runId }, Effect.fn("runs.get")(function* ({ id }) {
 	const run = yield* (yield* ApiClient).runs.get({ params: { id } });
 	yield* Console.log(yield* encodeRun(run));
@@ -1669,6 +1762,14 @@ const events = Command.make("events", {
 	yield* Console.log(yield* encodeEvents(page));
 })).pipe(Command.withDescription("List a run's events"));
 const printed = (sent, itemOf, encode) => sent.pipe(Stream.filterMap(itemOf), Stream.tap((item) => Effect.flatMap(encode(item), Console.log)), Stream.map((item) => item.seq), Stream.runLast);
+var RunUnfinished = class extends Schema.TaggedError()("RunUnfinished", {
+	id: RunId,
+	endReason: Schema.String
+}) {
+	get message() {
+		return `Run ${this.id} ended with ${this.endReason} before its agent finished`;
+	}
+};
 const follow = Command.make("follow", {
 	id: runId,
 	after: Flag.Int("after").pipe(Flag.withAlias("a"), Flag.withDescription("Only frames, or events with --events, with a greater seq"), Flag.optional),
@@ -1685,9 +1786,15 @@ const follow = Command.make("follow", {
 	let lastSeen = after;
 	while (true) {
 		lastSeen = Option.orElse(yield* followOnce(lastSeen), () => lastSeen);
-		if ((yield* client.runs.get({ params: { id } })).status === "ended") return;
+		const run = yield* client.runs.get({ params: { id } });
+		if (run.status !== "ended") continue;
+		if (run.end_reason === "completed" || run.end_reason === "idle_timeout") return;
+		return yield* new RunUnfinished({
+			id,
+			endReason: run.end_reason
+		});
 	}
-})).pipe(Command.withDescription("Print a run's frames, or its events with --events, as they come, one JSON object per line, until the run ends"));
+})).pipe(Command.withDescription("Print a run's frames, or its events with --events, as they come, one JSON object per line, until the run ends; exit with code 2 when it ended before its agent finished"));
 const sendPrompt = Command.make("send-prompt", {
 	id: runId,
 	text: Argument.String("text").pipe(Argument.withDescription("Follow-up prompt sent to the agent")),
@@ -1726,6 +1833,7 @@ const cancel = Command.make("cancel", { id: runId }, Effect.fn("runs.cancel")(fu
 })).pipe(Command.withDescription("Stop a run right away, even mid-prompt"));
 const runs = Command.make("runs").pipe(Command.withDescription("Manage runs"), Command.withSubcommands([
 	create,
+	list,
 	get,
 	sendPrompt,
 	prompts,
@@ -1736,12 +1844,12 @@ const runs = Command.make("runs").pipe(Command.withDescription("Manage runs"), C
 	end,
 	cancel
 ]));
-// src/main.ts
+// src/tracks.ts
 const health = Command.make("health", {}, Effect.fn("health")(function* () {
 	yield* (yield* ApiClient).health();
 	yield* Console.log("ok");
 })).pipe(Command.withDescription("Check that the API is reachable"));
-const cli = Command.make("tracks").pipe(Command.withDescription("Run coding agents in sandboxes and compare them on your scenarios"), Command.withSubcommands([
+const tracks = Command.make("tracks").pipe(Command.withDescription("Run coding agents in sandboxes and compare them on your scenarios"), Command.withSubcommands([
 	health,
 	runs,
 	connections,
@@ -1749,15 +1857,23 @@ const cli = Command.make("tracks").pipe(Command.withDescription("Run coding agen
 	executions,
 	library
 ]));
-const ApiClientLayer = Layer.unwrap(Effect.map(readEnv(...CLI_ENV), (env) => ApiClient.layer({
-	baseUrl: env.TRACKS_API_URL,
-	apiKey: env.TRACKS_API_KEY,
-	userAgent: `${CLI_USER_AGENT}/${version}`
-})));
+// src/main.ts
+const apiKey = Effect.option(Effect.gen(function* () {
+	const { TRACKS_API_KEY } = yield* readEnv("TRACKS_API_KEY");
+	return TRACKS_API_KEY;
+}));
+const ApiClientLayer = Layer.unwrap(Effect.gen(function* () {
+	const { TRACKS_API_URL } = yield* readEnv("TRACKS_API_URL");
+	return ApiClient.layer({
+		baseUrl: TRACKS_API_URL,
+		apiKey: yield* apiKey,
+		userAgent: `${CLI_USER_AGENT}/${version}`
+	});
+}));
 const MainLayer = Layer.mergeAll(ApiClientLayer, NodeServices.layer).pipe(Layer.provideMerge(FetchHttpClient.layer));
-Command.run(cli, { version }).pipe(Effect.catch((error) => {
+Command.run(tracks, { version }).pipe(Effect.catch((error) => {
 	const explainedByHelp = error._tag === "ShowHelp";
 	const line = "message" in error ? `${error._tag}: ${error.message}` : error._tag;
-	return Effect.andThen(explainedByHelp ? Effect.void : Console.error(line), Effect.sync(() => void (process.exitCode = 1)));
+	return Effect.andThen(explainedByHelp ? Effect.void : Console.error(line), Effect.sync(() => void (process.exitCode = error._tag === "RunUnfinished" || error._tag === "PassRateTooLow" ? 2 : 1)));
 }), Effect.provide(MainLayer), NodeRuntime.runMain);
 export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@alpic-ai/tracks",
-  "version": "0.7.1",
+  "version": "0.10.0",
   "description": "Run coding agents in sandboxes from the command line",
   "license": "UNLICENSED",
   "repository": {
@@ -24,7 +24,9 @@
     "effect": "^4.0.7"
   },
   "devDependencies": {
+    "@effect/vitest": "^4.0.7",
     "@tracks/contract": "0.0.0",
+    "@tracks/docs": "0.0.0",
     "@tracks/env": "0.0.0",
     "rolldown": "^1.2.15"
   },
@@ -33,6 +35,7 @@
   },
   "scripts": {
     "start": "node --import @tracks/env/preload src/main.ts",
-    "build": "rolldown --config rolldown.config.ts"
+    "build": "rolldown --config rolldown.config.ts",
+    "docs:generate": "node scripts/docs.ts"
   }
 }