@bugmole/cli 0.4.0

package/src/registry-worker/runner/routes.ts

@@ -0,0 +1,171 @@
+// Routes for hosted runs: the runner's bundle, writing to Bugmole storage,
+// and a project's default place to run.
+import type { Env, Principal } from "../core.js";
+import { memberOfProject, response, requestJson } from "../core.js";
+import { aiAccess } from "../ai/routes.js";
+import { isProjectArtifactKey, ARTIFACT_STORAGE_UNCONFIGURED } from "../artifacts.js";
+import { auditProject } from "../org/audit.js";
+import { matchPath, registerRoutes, type AuthedRouteContext } from "../routes/index.js";
+import { cloudRunMaxMinutes } from "./policy.js";
+import { cloudWorkerId } from "./jobs.js";
+import "./dispatch.js";
+import "./metering.js";
+
+/** A Worker request body is capped at 100 MB; stay under it. */
+export const MAX_UPLOAD_BYTES = 95 * 1024 * 1024;
+
+// A runner may ask for AI work on its own run only.
+const memberCheck = aiAccess.check;
+aiAccess.check = async (env, principal, projectId, work) => {
+  const runner = principal.runner;
+  if (!runner) return memberCheck(env, principal, projectId, work);
+  if (runner.projectId !== projectId) return false;
+  return runner.kind === "run" ? work.runId === runner.id : !work.runId && work.requestId === runner.id;
+};
+
+function parseJson(value: string | null): unknown {
+  if (!value) return null;
+  try {
+    return JSON.parse(value);
+  } catch {
+    return null;
+  }
+}
+
+async function bundle(env: Env, principal: Principal, runId: string): Promise<Response> {
+  if (principal.runner?.kind !== "run" || principal.runner.id !== runId) return response({ error: "Run not found" }, 404);
+  const run = await env.DB.prepare(`
+    SELECT r.id, r.project_id AS projectId, r.workspace_id AS workspaceId, r.plan_id AS planId,
+           r.journey_id AS journeyId, r.status, r.worker_id AS workerId, r.execution_mode AS executionMode,
+           r.targets_json AS targetsJson, r.environment_base_url AS environmentBaseUrl, r.fix_from_json AS fixFromJson,
+           p.name AS projectName
+    FROM runs r JOIN projects p ON p.id = r.project_id
+    WHERE r.id = ?
+  `).bind(runId).first<Record<string, string | null>>();
+  if (!run || run.projectId !== principal.runner.projectId) return response({ error: "Run not found" }, 404);
+  if (run.status !== "running" || run.workerId !== `cloud:${runId}`) {
+    return response({ error: "This run isn't assigned to a cloud runner", status: run.status }, 409);
+  }
+  return response({
+    run: {
+      id: run.id,
+      projectId: run.projectId,
+      planId: run.planId,
+      journeyId: run.journeyId,
+      status: run.status,
+      executionMode: run.executionMode,
+      environmentBaseUrl: run.environmentBaseUrl,
+      targets: parseJson(run.targetsJson),
+      fixFrom: parseJson(run.fixFromJson),
+    },
+    project: { id: run.projectId, name: run.projectName, workspaceId: run.workspaceId },
+    workerId: run.workerId,
+    maxMinutes: cloudRunMaxMinutes(env),
+  });
+}
+
+async function taskBundle(env: Env, principal: Principal, taskId: string): Promise<Response> {
+  if (principal.runner?.kind !== "task" || principal.runner.id !== taskId) return response({ error: "Task not found" }, 404);
+  const task = await env.DB.prepare(`
+    SELECT t.id, t.project_id AS projectId, t.workspace_id AS workspaceId, t.title, t.description,
+           t.task_type AS taskType, t.status, t.worker_id AS workerId, t.attempts, p.name AS projectName
+    FROM agent_tasks t JOIN projects p ON p.id = t.project_id
+    WHERE t.id = ?
+  `).bind(taskId).first<Record<string, string | number | null>>();
+  if (!task || task.projectId !== principal.runner.projectId) return response({ error: "Task not found" }, 404);
+  if (task.status !== "running" || task.workerId !== cloudWorkerId({ kind: "task", id: taskId })) {
+    return response({ error: "This task isn't assigned to a cloud runner", status: task.status }, 409);
+  }
+  return response({
+    task: {
+      id: task.id,
+      projectId: task.projectId,
+      title: task.title,
+      description: task.description,
+      taskType: task.taskType,
+      status: task.status,
+      attempts: task.attempts,
+      executionMode: "cloud",
+    },
+    project: { id: task.projectId, name: task.projectName, workspaceId: task.workspaceId },
+    workerId: task.workerId,
+    maxMinutes: cloudRunMaxMinutes(env),
+  });
+}
+
+async function managedProject(env: Env, projectId: string): Promise<boolean> {
+  const row = await env.DB.prepare("SELECT storage_mode AS storageMode FROM projects WHERE id = ? AND deleted_at IS NULL")
+    .bind(projectId).first<{ storageMode: string }>();
+  return row?.storageMode === "managed";
+}
+
+async function putObject({ env, principal, request, url }: AuthedRouteContext, projectId: string): Promise<Response> {
+  if (!env.ARTIFACTS) return response({ error: ARTIFACT_STORAGE_UNCONFIGURED }, 503);
+  if (!(await memberOfProject(env, principal, projectId, "project.write"))) return response({ error: "Project scope is not allowed" }, 403);
+  if (!(await managedProject(env, projectId))) {
+    return response({ error: "This project keeps its files in its own storage, not Bugmole's.", code: "storage_not_managed" }, 409);
+  }
+  const key = url.searchParams.get("key") ?? "";
+  if (!isProjectArtifactKey(projectId, key)) return response({ error: "key must stay under this project's artifacts" }, 400);
+  const declared = request.headers.get("content-length");
+  if (declared !== null && !(Number(declared) <= MAX_UPLOAD_BYTES)) return response({ error: "The file is larger than 95 MB" }, 413);
+  const contentType = request.headers.get("content-type") || "application/octet-stream";
+  // R2 needs the length up front: stream when the client sent it, otherwise buffer.
+  let body: ReadableStream | ArrayBuffer | null = request.body;
+  if (declared === null) {
+    body = await request.arrayBuffer();
+    if (body.byteLength > MAX_UPLOAD_BYTES) return response({ error: "The file is larger than 95 MB" }, 413);
+  }
+  const stored = await env.ARTIFACTS.put(key, body, { httpMetadata: { contentType } });
+  return response({ key, size: stored?.size ?? Number(declared), contentType }, 201);
+}
+
+async function deletePrefix({ env, principal, url, request }: AuthedRouteContext, projectId: string): Promise<Response> {
+  if (!env.ARTIFACTS) return response({ error: ARTIFACT_STORAGE_UNCONFIGURED }, 503);
+  // Runners are limited to their own run's prefix before routing.
+  const permission = principal.runner ? "project.write" : "project.manage";
+  if (!(await memberOfProject(env, principal, projectId, permission))) return response({ error: "Project scope is not allowed" }, 403);
+  if (!(await managedProject(env, projectId))) return response({ error: "This project keeps its files in its own storage, not Bugmole's.", code: "storage_not_managed" }, 409);
+  const prefix = url.searchParams.get("prefix") ?? "";
+  if (!prefix.endsWith("/") || !isProjectArtifactKey(projectId, prefix)) {
+    return response({ error: "prefix must be a folder under this project's artifacts" }, 400);
+  }
+  let deleted = 0;
+  let cursor: string | undefined;
+  do {
+    const page = await env.ARTIFACTS.list({ prefix, cursor, limit: 1000 });
+    const keys = page.objects.map((object) => object.key);
+    if (keys.length) await env.ARTIFACTS.delete(keys);
+    deleted += keys.length;
+    cursor = page.truncated ? page.cursor : undefined;
+  } while (cursor);
+  if (!principal.runner) {
+    await auditProject(env, principal, projectId, { action: "storage.deleted", targetType: "prefix", targetId: prefix, data: { deleted } }, request);
+  }
+  return response({ deleted });
+}
+
+async function updateSettings({ env, principal, request }: AuthedRouteContext, projectId: string): Promise<Response> {
+  if (!(await memberOfProject(env, principal, projectId, "project.manage"))) return response({ error: "Project scope is not allowed" }, 403);
+  const body = await requestJson(request);
+  const mode = body?.defaultExecutionMode;
+  if (mode !== "self" && mode !== "cloud") return response({ error: "defaultExecutionMode must be self or cloud" }, 422);
+  await env.DB.prepare("UPDATE projects SET default_execution_mode = ? WHERE id = ?").bind(mode, projectId).run();
+  await auditProject(env, principal, projectId, { action: "project.settings_updated", targetType: "project", targetId: projectId, data: { defaultExecutionMode: mode } }, request);
+  return response({ project: { id: projectId, defaultExecutionMode: mode } });
+}
+
+registerRoutes(async (context) => {
+  const { request, url, env, principal } = context;
+  const bundleMatch = matchPath(url.pathname, "/api/runner/runs/:runId/bundle");
+  if (bundleMatch && request.method === "GET") return bundle(env, principal, bundleMatch.runId);
+  const taskBundleMatch = matchPath(url.pathname, "/api/runner/tasks/:taskId/bundle");
+  if (taskBundleMatch && request.method === "GET") return taskBundle(env, principal, taskBundleMatch.taskId);
+  const objectMatch = matchPath(url.pathname, "/api/projects/:projectId/artifacts/object");
+  if (objectMatch && request.method === "PUT") return putObject(context, objectMatch.projectId);
+  const prefixMatch = matchPath(url.pathname, "/api/projects/:projectId/artifacts");
+  if (prefixMatch && request.method === "DELETE") return deletePrefix(context, prefixMatch.projectId);
+  const settingsMatch = matchPath(url.pathname, "/api/projects/:projectId/settings");
+  if (settingsMatch && request.method === "PATCH") return updateSettings(context, settingsMatch.projectId);
+  return null;
+});

package/src/registry-worker/runner/runner.test.ts

@@ -0,0 +1,358 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { call, createTestEnv, runScheduled, seedPlan, seedProject, seedVerifiedDomain } from "../testing/harness.js";
+import { recordUsage } from "../billing/ledger.js";
+import { runnerMayCall, signRunnerToken, verifyRunnerToken } from "./tokens.js";
+import { dispatchCloudRuns, pickFairly, runnerLauncher, type LaunchRequest } from "./dispatch.js";
+import { billableMinutes } from "./metering.js";
+
+const SECRET = { RUNNER_TOKEN_SECRET: "runner-test-secret", FEATURE_CLOUD_RUNNERS: "on", REGISTRY_PUBLIC_URL: "https://registry.test" };
+
+function fakeLauncher(fail = 0) {
+  const launches: LaunchRequest[] = [];
+  let failures = fail;
+  const original = runnerLauncher.start;
+  runnerLauncher.start = async (_env, launch) => {
+    if (failures > 0) {
+      failures -= 1;
+      throw new Error("no capacity");
+    }
+    launches.push(launch);
+  };
+  return { launches, restore: () => { runnerLauncher.start = original; } };
+}
+
+async function cloudProject(overrides: Record<string, string> = {}, seed: Parameters<typeof seedProject>[1] = {}) {
+  const setup = createTestEnv({ ...SECRET, ...overrides });
+  const project = await seedProject(setup.env, { storageProvider: "bugmole", ...seed });
+  await seedPlan(setup.env, project.projectId, project.owner);
+  // Several of this file's tests explore or run against https://shop.test.
+  seedVerifiedDomain(setup.db, project.workspaceId, "shop.test");
+  const queue = (key: string, projectId = project.projectId, owner = project.owner) =>
+    call(setup.env, "POST", `/api/projects/${projectId}/runs`, {
+      as: owner,
+      headers: { "x-aegis-email": "owner@acme.test", "x-aegis-email-verified": "true" },
+      body: { planId: "checkout", idempotencyKey: key, executionMode: "cloud" },
+    });
+  return { ...setup, ...project, queue };
+}
+
+test("runner tokens name one run, can't be altered and expire", async () => {
+  const { env } = createTestEnv(SECRET);
+  const identity = { kind: "run" as const, id: "run_1", projectId: "shop", workspaceId: "acme" };
+  const token = await signRunnerToken(env, identity, 1_000);
+  assert.deepEqual(await verifyRunnerToken(env, token, 2_000), identity);
+  assert.equal(await verifyRunnerToken(env, token, 1_000 + 2 * 3_600_000), null, "expired");
+  const [payload, signature] = token.slice(4).split(".");
+  const forged = Buffer.from(JSON.stringify({ ...identity, id: "run_2", exp: 9e15 })).toString("base64url");
+  assert.equal(await verifyRunnerToken(env, `vrt_${forged}.${signature}`, 2_000), null);
+  assert.equal(await verifyRunnerToken({ ...env, RUNNER_TOKEN_SECRET: "other" }, token, 2_000), null);
+  assert.ok(payload);
+
+  const allowed = (method: string, path: string) => runnerMayCall(identity, method, new URL(`https://r${path}`));
+  assert.equal(allowed("GET", "/api/runner/runs/run_1/bundle"), true);
+  assert.equal(allowed("GET", "/api/runner/runs/run_2/bundle"), false);
+  assert.equal(allowed("POST", "/api/runs/run_1"), true);
+  assert.equal(allowed("POST", "/api/runs/run_1/claim"), false);
+  assert.equal(allowed("GET", "/api/projects/shop/artifacts/object?key=projects/shop/plans/a.yaml"), true);
+  assert.equal(allowed("PUT", "/api/projects/shop/artifacts/object?key=projects/shop/plans/a.yaml"), false);
+  assert.equal(allowed("PUT", "/api/projects/shop/artifacts/object?key=projects/shop/runs/run_1/video.webm"), true);
+  assert.equal(allowed("PUT", "/api/projects/shop/artifacts/object?key=projects/shop/runs/run_10/video.webm"), false);
+  assert.equal(allowed("GET", "/api/projects/other/artifacts"), false);
+  assert.equal(allowed("GET", "/api/projects"), false);
+  assert.equal(allowed("POST", "/api/tasks/run_1"), false);
+
+  const task = { kind: "task" as const, id: "task_1", projectId: "shop", workspaceId: "acme" };
+  const taskAllowed = (method: string, path: string) => runnerMayCall(task, method, new URL(`https://r${path}`));
+  assert.equal(taskAllowed("GET", "/api/runner/tasks/task_1/bundle"), true);
+  assert.equal(taskAllowed("GET", "/api/runner/runs/task_1/bundle"), false);
+  assert.equal(taskAllowed("POST", "/api/tasks/task_1"), true);
+  assert.equal(taskAllowed("POST", "/api/tasks/task_1/claim"), false);
+  assert.equal(taskAllowed("POST", "/api/projects/shop/plans"), true);
+  assert.equal(taskAllowed("PUT", "/api/projects/shop/artifacts/object?key=projects/shop/spec/flows/a.yaml"), true);
+  assert.equal(taskAllowed("PUT", "/api/projects/shop/artifacts/object?key=projects/shop/journeys/graph.json"), true);
+  assert.equal(taskAllowed("PUT", "/api/projects/shop/artifacts/object?key=projects/shop/runs/run_1/x.png"), false);
+  assert.equal(taskAllowed("DELETE", "/api/projects/shop/artifacts?prefix=projects/shop/spec/"), false);
+  assert.equal(taskAllowed("POST", "/api/runs/task_1"), false);
+});
+
+test("a cloud run is started, reads its plan, uploads evidence, and bills registry-measured minutes", async () => {
+  const launcher = fakeLauncher();
+  try {
+    const { env, db, artifacts, projectId, workspaceId, owner, queue } = await cloudProject();
+    await artifacts.put(`projects/${projectId}/plans/checkout.flow.yaml`, "suites: []\n");
+    const created = await queue("c1");
+    assert.equal(created.status, 201);
+    const runId = created.json.run.id;
+    assert.equal(launcher.launches.length, 1, "queuing starts a runner straight away");
+    assert.equal(launcher.launches[0].kind, "run");
+    const { token, registryUrl } = launcher.launches[0];
+    assert.equal(registryUrl, "https://registry.test");
+
+    const bundle = await call(env, "GET", `/api/runner/runs/${runId}/bundle`, { token });
+    assert.equal(bundle.status, 200);
+    assert.equal(bundle.json.workerId, `cloud:${runId}`);
+    assert.equal(bundle.json.run.planId, "checkout");
+    const plan = await call(env, "GET", `/api/projects/${projectId}/artifacts/object?key=projects/${projectId}/plans/checkout.flow.yaml`, { token });
+    assert.equal(plan.json, "suites: []\n");
+
+    const evidenceKey = `projects/${projectId}/runs/${runId}/artifacts/chromium/screen.png`;
+    const upload = await call(env, "PUT", `/api/projects/${projectId}/artifacts/object?key=${evidenceKey}`, {
+      token, body: "png-bytes", headers: { "content-type": "image/png", "content-length": "9" },
+    });
+    assert.equal(upload.status, 201);
+    assert.equal(artifacts.objects.get(evidenceKey)?.contentType, "image/png");
+    const outside = await call(env, "PUT", `/api/projects/${projectId}/artifacts/object?key=projects/${projectId}/plans/checkout.flow.yaml`, {
+      token, body: "x", headers: { "content-length": "1" },
+    });
+    assert.equal(outside.status, 403, "a runner can't rewrite plans");
+    assert.equal((await call(env, "GET", "/api/projects", { token })).status, 403);
+    assert.equal((await call(env, "GET", `/api/runner/runs/${runId}/bundle`, { as: owner })).status, 404, "bundles are for runners only");
+
+    const heartbeat = await call(env, "POST", `/api/runs/${runId}`, { token, body: { workerId: `cloud:${runId}`, status: "running", eventType: "heartbeat" } });
+    assert.deepEqual(heartbeat.json, { updated: true });
+    db.db.prepare("UPDATE runs SET claimed_at = ? WHERE id = ?").run(new Date(Date.now() - 150_000).toISOString(), runId);
+    const done = await call(env, "POST", `/api/runs/${runId}`, { token, body: { workerId: `cloud:${runId}`, status: "success", eventType: "completed" } });
+    assert.equal(done.status, 200);
+    const run = await call(env, "GET", `/api/runs/${runId}`, { as: owner });
+    assert.equal(run.json.run.billedMinutes, 3, "2.5 minutes rounds up");
+    const usage = await call(env, "GET", `/api/workspaces/${workspaceId}/usage`, { as: owner });
+    assert.equal(usage.json.usage.lines.browser_minute.used, 3);
+
+    // Finishing again (a retried request) doesn't bill twice.
+    await call(env, "POST", `/api/runs/${runId}`, { token, body: { workerId: `cloud:${runId}`, status: "success", eventType: "completed" } });
+    const again = await call(env, "GET", `/api/workspaces/${workspaceId}/usage`, { as: owner });
+    assert.equal(again.json.usage.lines.browser_minute.used, 3);
+  } finally {
+    launcher.restore();
+  }
+});
+
+test("discovery runs in the cloud too, publishes what it finds, and is billed", async () => {
+  const launcher = fakeLauncher();
+  try {
+    const { env, db, artifacts, projectId, owner, workspaceId } = await cloudProject();
+    const explore = (body: Record<string, unknown>) => call(env, "POST", `/api/projects/${projectId}/tasks`, {
+      as: owner, body: { title: "Explore with AI https://shop.test", description: "Explore https://shop.test", taskType: "discovery", idempotencyKey: "d1", ...body },
+    });
+    assert.equal((await explore({ taskType: "code", idempotencyKey: "c1", executionMode: "cloud" })).status, 422, "coding tasks need your own agent");
+    const created = await explore({ executionMode: "cloud" });
+    assert.equal(created.status, 201);
+    const taskId = created.json.task.id;
+    assert.equal(created.json.task.executionMode, "cloud");
+    assert.deepEqual(launcher.launches.map((item) => [item.kind, item.id]), [["task", taskId]]);
+    const { token } = launcher.launches[0];
+
+    assert.equal((await call(env, "POST", `/api/tasks/${taskId}/claim`, { as: owner, body: { workerId: "laptop" } })).status, 409);
+    const bundle = await call(env, "GET", `/api/runner/tasks/${taskId}/bundle`, { token });
+    assert.equal(bundle.status, 200);
+    assert.equal(bundle.json.task.description, "Explore https://shop.test");
+    assert.equal(bundle.json.workerId, `cloud:task:${taskId}`);
+    assert.equal((await call(env, "GET", `/api/runner/runs/${taskId}/bundle`, { token })).status, 403);
+
+    const flowKey = `projects/${projectId}/spec/flows/discovered/home.yaml`;
+    assert.equal((await call(env, "PUT", `/api/projects/${projectId}/artifacts/object?key=${flowKey}`, { token, body: "- launchApp\n", headers: { "content-type": "text/yaml" } })).status, 201);
+    assert.ok(artifacts.objects.has(flowKey));
+    const published = await call(env, "POST", `/api/projects/${projectId}/plans`, {
+      token, body: { planId: "discovered-home", journeyId: "discovered-home", objectKey: `projects/${projectId}/plans/discovered-home.flow.yaml` },
+    });
+    assert.equal(published.status, 201);
+
+    const workerId = `cloud:task:${taskId}`;
+    const running = await call(env, "POST", `/api/tasks/${taskId}`, { token, body: { workerId, status: "running", progressMessage: "Opened the home page" } });
+    assert.equal(running.json.stop, undefined);
+    db.db.prepare("UPDATE agent_tasks SET claimed_at = ? WHERE id = ?").run(new Date(Date.now() - 90_000).toISOString(), taskId);
+    await call(env, "POST", `/api/tasks/${taskId}`, { token, body: { workerId, status: "completed", completionSummary: "Explored 3 pages" } });
+    const task = db.db.prepare("SELECT status, billed_minutes AS minutes FROM agent_tasks WHERE id = ?").get(taskId) as any;
+    assert.deepEqual({ ...task }, { status: "completed", minutes: 2 });
+    const usage = await call(env, "GET", `/api/workspaces/${workspaceId}/usage`, { as: owner });
+    assert.equal(usage.json.usage.lines.browser_minute.used, 2);
+
+    // The project had no environment at all (createManagedProject doesn't
+    // create one) — a completed discovery run is the first time anyone
+    // knows the project's URL, so it fills the gap instead of leaving it.
+    const env_ = db.db.prepare("SELECT base_url AS baseUrl, is_default AS isDefault FROM project_environments WHERE project_id = ?").get(projectId) as any;
+    assert.equal(env_.baseUrl, "https://shop.test");
+    assert.equal(env_.isDefault, 1);
+  } finally {
+    launcher.restore();
+  }
+});
+
+test("a discovered URL fills a project's empty default environment but never overwrites one a human set", async () => {
+  const launcher = fakeLauncher();
+  try {
+    const { env, db, projectId, owner } = await cloudProject();
+    db.db.prepare(`
+      INSERT INTO project_environments (id, project_id, app_id, name, base_url, status, is_default)
+      VALUES ('default', ?, 'default', 'Production', '', 'active', 1)
+    `).run(projectId);
+    const explore = (idempotencyKey: string, url: string) => call(env, "POST", `/api/projects/${projectId}/tasks`, {
+      as: owner,
+      body: { title: `Explore ${url}`, description: `Explore ${url}`, taskType: "discovery", idempotencyKey, executionMode: "cloud" },
+    });
+    const complete = async (launchIndex: number, taskId: string) => {
+      const { token } = launcher.launches[launchIndex];
+      await call(env, "POST", `/api/tasks/${taskId}`, {
+        token, body: { workerId: `cloud:task:${taskId}`, status: "completed", completionSummary: "Explored 1 page" },
+      });
+    };
+    const readBaseUrl = () => (db.db.prepare("SELECT base_url AS baseUrl FROM project_environments WHERE project_id = ?").get(projectId) as any).baseUrl;
+
+    const first = await explore("d1", "https://shop.test");
+    await complete(0, first.json.task.id);
+    assert.equal(readBaseUrl(), "https://shop.test", "filled the empty default environment");
+
+    // A second, different discovery task completing must not clobber what's
+    // now a human-editable (and possibly since hand-changed) base_url.
+    db.db.prepare("UPDATE project_environments SET base_url = 'https://staging.shop.test' WHERE project_id = ?").run(projectId);
+    const second = await explore("d2", "https://shop.test");
+    await complete(1, second.json.task.id);
+    assert.equal(readBaseUrl(), "https://staging.shop.test", "an environment a human already pointed somewhere is left alone");
+  } finally {
+    launcher.restore();
+  }
+});
+
+test("runs and discovery share the workspace's parallel slots", async () => {
+  const launcher = fakeLauncher();
+  try {
+    const { env, projectId, owner, queue } = await cloudProject();
+    const run = (await queue("r1")).json.run.id;
+    const task = (await call(env, "POST", `/api/projects/${projectId}/tasks`, {
+      as: owner, body: { title: "Explore https://shop.test", description: "https://shop.test", taskType: "discovery", idempotencyKey: "d1", executionMode: "cloud" },
+    })).json.task.id;
+    const waiting = (await queue("r2")).json.run.id;
+    assert.deepEqual(launcher.launches.map((item) => item.id), [run, task]);
+    const token = launcher.launches[1].token;
+    await call(env, "POST", `/api/tasks/${task}`, { token, body: { workerId: `cloud:task:${task}`, status: "error", errorMessage: "site down" } });
+    assert.deepEqual(launcher.launches.map((item) => item.id), [run, task, waiting]);
+  } finally {
+    launcher.restore();
+  }
+});
+
+test("cloud runs need Bugmole storage", async () => {
+  const launcher = fakeLauncher();
+  try {
+    const { env, owner, projectId } = await cloudProject({}, { storageProvider: "r2" });
+    const created = await call(env, "POST", `/api/projects/${projectId}/runs`, {
+      as: owner, body: { planId: "checkout", idempotencyKey: "b1", executionMode: "cloud" },
+    });
+    assert.equal(created.status, 422);
+    assert.equal(created.json.code, "cloud_needs_managed_storage");
+    const upload = await call(env, "PUT", `/api/projects/${projectId}/artifacts/object?key=projects/${projectId}/plans/a.yaml`, {
+      as: owner, body: "x", headers: { "content-length": "1" },
+    });
+    assert.equal(upload.status, 409, "own-storage projects don't write to Bugmole's bucket");
+  } finally {
+    launcher.restore();
+  }
+});
+
+test("workspaces take turns within their parallel limit and the pool size", () => {
+  const run = (id: string, workspaceId: string) => ({ id, projectId: "p", workspaceId, attempts: 0 });
+  const queued = [run("a1", "a"), run("a2", "a"), run("a3", "a"), run("a4", "a"), run("b1", "b"), run("c1", "c")];
+  const free = new Map([["a", 2], ["b", 1], ["c", 0]]);
+  assert.deepEqual(pickFairly(queued, free, 10).map((item) => item.id), ["a1", "b1", "a2"]);
+  assert.deepEqual(pickFairly(queued, free, 2).map((item) => item.id), ["a1", "b1"]);
+  assert.deepEqual(pickFairly(queued, free, 0), []);
+});
+
+test("the plan's parallel limit holds runs in the queue until a slot frees up", async () => {
+  const launcher = fakeLauncher();
+  try {
+    const { env, queue } = await cloudProject();
+    const ids = [];
+    for (const key of ["p1", "p2", "p3"]) ids.push((await queue(key)).json.run.id);
+    assert.deepEqual(launcher.launches.map((item) => item.id), ids.slice(0, 2), "Free runs two at a time");
+    const token = launcher.launches[0].token;
+    await call(env, "POST", `/api/runs/${ids[0]}`, { token, body: { workerId: `cloud:${ids[0]}`, status: "error", eventType: "failed" } });
+    assert.deepEqual(launcher.launches.map((item) => item.id), ids, "a finished run starts the next");
+  } finally {
+    launcher.restore();
+  }
+});
+
+test("a runner that can't start is retried, then the run fails without a charge", async () => {
+  const launcher = fakeLauncher(10);
+  try {
+    const { env, db, owner, workspaceId, queue } = await cloudProject();
+    const runId = (await queue("f1")).json.run.id;
+    let run = db.db.prepare("SELECT status, dispatch_attempts AS attempts, next_dispatch_at AS next FROM runs WHERE id = ?").get(runId) as any;
+    assert.equal(run.status, "queued");
+    assert.equal(run.attempts, 1);
+    assert.ok(run.next);
+    const later = (ms: number) => new Date(Date.now() + ms);
+    await dispatchCloudRuns(env, later(15_000));
+    assert.equal((db.db.prepare("SELECT dispatch_attempts AS attempts FROM runs WHERE id = ?").get(runId) as any).attempts, 1, "waits out the retry delay");
+    await dispatchCloudRuns(env, later(31_000));
+    await dispatchCloudRuns(env, later(200_000));
+    run = db.db.prepare("SELECT status, failure_code AS code, billed_at AS billed FROM runs WHERE id = ?").get(runId);
+    assert.deepEqual({ ...run }, { status: "error", code: "runner_unavailable", billed: null });
+    const usage = await call(env, "GET", `/api/workspaces/${workspaceId}/usage`, { as: owner });
+    assert.equal(usage.json.usage.lines.browser_minute.used, 0);
+  } finally {
+    launcher.restore();
+  }
+});
+
+test("a runner that goes silent is failed and billed up to its last heartbeat", async () => {
+  const launcher = fakeLauncher();
+  try {
+    const { env, db, queue } = await cloudProject();
+    const runId = (await queue("l1")).json.run.id;
+    const minutesAgo = (minutes: number) => new Date(Date.now() - minutes * 60_000).toISOString();
+    db.db.prepare("UPDATE runs SET claimed_at = ?, heartbeat_at = ?, lease_expires_at = ? WHERE id = ?")
+      .run(minutesAgo(30), minutesAgo(25).replace("T", " ").slice(0, 19), minutesAgo(24), runId);
+    await runScheduled(env, "* * * * *");
+    const run = db.db.prepare("SELECT status, failure_code AS code, billed_minutes AS minutes FROM runs WHERE id = ?").get(runId) as any;
+    assert.equal(run.status, "error");
+    assert.equal(run.code, "runner_lost");
+    assert.equal(run.minutes, 6, "5 minutes of heartbeats plus the one-minute grace");
+  } finally {
+    launcher.restore();
+  }
+});
+
+test("a running cloud run is told to stop when the minutes or the time limit run out", async () => {
+  const launcher = fakeLauncher();
+  try {
+    const { env, db, workspaceId, queue } = await cloudProject({ FEATURE_ENTITLEMENTS: "on", CLOUD_RUN_MAX_MINUTES: "30" });
+    const runId = (await queue("s1")).json.run.id;
+    const token = launcher.launches[0].token;
+    const beat = () => call(env, "POST", `/api/runs/${runId}`, { token, body: { workerId: `cloud:${runId}`, status: "running", eventType: "heartbeat" } });
+    await recordUsage(env, { workspaceId, kind: "browser_minute", quantity: 296, source: "cloud_run", idempotencyKey: "earlier" });
+    db.db.prepare("UPDATE runs SET claimed_at = ? WHERE id = ?").run(new Date(Date.now() - 3 * 60_000).toISOString(), runId);
+    assert.deepEqual((await beat()).json, { updated: true });
+    db.db.prepare("UPDATE runs SET claimed_at = ? WHERE id = ?").run(new Date(Date.now() - 5 * 60_000).toISOString(), runId);
+    const stopped = await beat();
+    assert.equal(stopped.json.stop, true);
+    assert.equal(stopped.json.reason, "quota");
+    db.db.prepare("DELETE FROM usage_events").run();
+    db.db.prepare("UPDATE runs SET claimed_at = ? WHERE id = ?").run(new Date(Date.now() - 31 * 60_000).toISOString(), runId);
+    assert.equal((await beat()).json.reason, "time_limit");
+  } finally {
+    launcher.restore();
+  }
+});
+
+test("projects show their storage and default place to run, which managers can change", async () => {
+  const { env, owner, projectId } = await cloudProject();
+  const listed = await call(env, "GET", "/api/projects", { as: owner });
+  const project = listed.json.projects.find((item: { id: string }) => item.id === projectId);
+  assert.equal(project.storageMode, "managed");
+  assert.equal(project.defaultExecutionMode, "self");
+  assert.equal((await call(env, "PATCH", `/api/projects/${projectId}/settings`, { as: owner, body: { defaultExecutionMode: "moon" } })).status, 422);
+  const changed = await call(env, "PATCH", `/api/projects/${projectId}/settings`, { as: owner, body: { defaultExecutionMode: "cloud" } });
+  assert.equal(changed.status, 200);
+  assert.equal((await call(env, "PATCH", `/api/projects/${projectId}/settings`, { as: "user_stranger", body: { defaultExecutionMode: "self" } })).status, 403);
+});
+
+test("billable minutes round up, start at one, and stop a minute after the last heartbeat", () => {
+  const at = (seconds: number) => new Date(Date.UTC(2026, 8, 17, 10, 0, seconds)).toISOString();
+  assert.equal(billableMinutes({ claimedAt: at(0), completedAt: at(5), heartbeatAt: at(5) }), 1);
+  assert.equal(billableMinutes({ claimedAt: at(0), completedAt: at(61), heartbeatAt: at(61) }), 2);
+  assert.equal(billableMinutes({ claimedAt: at(0), completedAt: at(3000), heartbeatAt: "2026-09-17 10:01:00" }), 2);
+  assert.equal(billableMinutes({ claimedAt: at(0), completedAt: null, heartbeatAt: null }, Date.parse(at(125))), 3);
+});

package/src/registry-worker/runner/tokens.ts

@@ -0,0 +1,93 @@
+// Short-lived credentials for hosted runners. A token names one run and is
+// signed with RUNNER_TOKEN_SECRET, so the registry needs no table for them and
+// a leaked token expires with the run (two hours at most).
+import type { Env } from "../core.js";
+
+export const RUNNER_TOKEN_PREFIX = "vrt_";
+export const RUNNER_TOKEN_TTL_MS = 2 * 60 * 60 * 1000;
+
+/** A runner acts for exactly one run or one discovery task. */
+export type RunnerIdentity = { kind: "run" | "task"; id: string; projectId: string; workspaceId: string | null };
+type Claims = RunnerIdentity & { exp: number };
+
+const encoder = new TextEncoder();
+
+function base64Url(bytes: Uint8Array): string {
+  let binary = "";
+  for (const byte of bytes) binary += String.fromCharCode(byte);
+  return btoa(binary).replaceAll("+", "-").replaceAll("/", "_").replace(/=+$/, "");
+}
+
+function fromBase64Url(value: string): Uint8Array {
+  const padded = value.replaceAll("-", "+").replaceAll("_", "/") + "=".repeat((4 - (value.length % 4)) % 4);
+  return Uint8Array.from(atob(padded), (char) => char.charCodeAt(0));
+}
+
+async function hmacKey(secret: string, usage: "sign" | "verify"): Promise<CryptoKey> {
+  return crypto.subtle.importKey("raw", encoder.encode(secret), { name: "HMAC", hash: "SHA-256" }, false, [usage]);
+}
+
+export async function signRunnerToken(env: Env, identity: RunnerIdentity, now = Date.now()): Promise<string> {
+  if (!env.RUNNER_TOKEN_SECRET) throw new Error("RUNNER_TOKEN_SECRET is not configured");
+  const claims: Claims = { ...identity, exp: now + RUNNER_TOKEN_TTL_MS };
+  const payload = base64Url(encoder.encode(JSON.stringify(claims)));
+  const signature = await crypto.subtle.sign("HMAC", await hmacKey(env.RUNNER_TOKEN_SECRET, "sign"), encoder.encode(payload));
+  return `${RUNNER_TOKEN_PREFIX}${payload}.${base64Url(new Uint8Array(signature))}`;
+}
+
+export async function verifyRunnerToken(env: Env, token: string, now = Date.now()): Promise<RunnerIdentity | null> {
+  if (!env.RUNNER_TOKEN_SECRET || !token.startsWith(RUNNER_TOKEN_PREFIX)) return null;
+  const [payload, signature, extra] = token.slice(RUNNER_TOKEN_PREFIX.length).split(".");
+  if (!payload || !signature || extra !== undefined) return null;
+  try {
+    const valid = await crypto.subtle.verify(
+      "HMAC",
+      await hmacKey(env.RUNNER_TOKEN_SECRET, "verify"),
+      fromBase64Url(signature),
+      encoder.encode(payload),
+    );
+    if (!valid) return null;
+    const claims = JSON.parse(new TextDecoder().decode(fromBase64Url(payload))) as Partial<Claims>;
+    if (typeof claims.id !== "string" || typeof claims.projectId !== "string" || typeof claims.exp !== "number") return null;
+    if (claims.kind !== "run" && claims.kind !== "task") return null;
+    if (claims.exp <= now) return null;
+    return { kind: claims.kind, id: claims.id, projectId: claims.projectId, workspaceId: typeof claims.workspaceId === "string" ? claims.workspaceId : null };
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * The only calls a runner makes. A run reads its bundle and the project's
+ * stored plans, writes evidence under its own run, and reports. A discovery
+ * task writes the flows, plans and journey graph it found, and publishes the
+ * plans. Everything else is refused before routing.
+ */
+export function runnerMayCall(identity: RunnerIdentity, method: string, url: URL): boolean {
+  const id = encodeURIComponent(identity.id);
+  const project = encodeURIComponent(identity.projectId);
+  const path = url.pathname;
+  if (path === "/api/ai/complete") return method === "POST";
+  const artifacts = path === `/api/projects/${project}/artifacts` || path === `/api/projects/${project}/artifacts/object`;
+  const key = url.searchParams.get(path.endsWith("/object") ? "key" : "prefix") ?? "";
+  if (identity.kind === "run") {
+    if (path === `/api/runner/runs/${id}/bundle`) return method === "GET";
+    if (path === `/api/runs/${id}`) return method === "GET" || method === "POST";
+    if (path === `/api/runs/${id}/device-access`) return method === "GET";
+    if (path === `/api/runs/${id}/device-sessions`) return method === "POST";
+    if (path.startsWith(`/api/runs/${id}/device-sessions/`)) return method === "GET" || method === "POST";
+    if (artifacts) {
+      if (method === "GET") return true;
+      return (method === "PUT" || method === "DELETE") && key.startsWith(`projects/${project}/runs/${id}/`);
+    }
+    return false;
+  }
+  if (path === `/api/runner/tasks/${id}/bundle`) return method === "GET";
+  if (path === `/api/tasks/${id}`) return method === "POST";
+  if (path === `/api/projects/${project}/plans`) return method === "POST";
+  if (artifacts) {
+    if (method === "GET") return true;
+    return method === "PUT" && ["spec", "plans", "journeys"].some((folder) => key.startsWith(`projects/${project}/${folder}/`));
+  }
+  return false;
+}