@bugmole/cli 0.4.0

package/src/registry-worker/billing/thresholds.ts

@@ -0,0 +1,107 @@
+// Emails owners and billing managers once per period when usage crosses 80%
+// and 100% of an allowance, or of the spending limit.
+import type { UsageKind } from "../../billing/plan-catalog.js";
+import type { Env } from "../core.js";
+import { emailSender, escapeHtml } from "../email/sender.js";
+import { usageSummary } from "./ledger.js";
+
+const LABELS: Record<UsageKind, string> = {
+  browser_minute: "cloud minutes",
+  ai_credit: "AI credits",
+  device_minute: "real-device minutes",
+};
+
+export type ThresholdAlert = { key: string; subject: string; body: string };
+
+/** Alerts due for the current usage, highest threshold first per meter. */
+export function dueAlerts(summary: Awaited<ReturnType<typeof usageSummary>>["summary"]): ThresholdAlert[] {
+  const alerts: ThresholdAlert[] = [];
+  for (const kind of ["browser_minute", "ai_credit"] as const) {
+    const line = summary.lines[kind];
+    if (!line.included) continue;
+    const share = line.used / line.included;
+    for (const threshold of [100, 80]) {
+      if (share * 100 >= threshold) {
+        alerts.push({
+          key: `${kind}:${threshold}`,
+          subject: threshold === 100 ? `You've used all of this month's ${LABELS[kind]}` : `You've used ${threshold}% of this month's ${LABELS[kind]}`,
+          body: `${line.used.toLocaleString("en-US")} of ${line.included.toLocaleString("en-US")} ${LABELS[kind]} used this period.` +
+            (threshold === 100
+              ? line.unitCents === null
+                ? " Cloud usage is paused until the next period; runs on your own machines and CI keep working."
+                : ` Additional usage is billed at $${(line.unitCents / 100).toFixed(2)} each.`
+              : ""),
+        });
+        break;
+      }
+    }
+  }
+  if (summary.spendingLimitCents !== null && summary.spendingLimitCents > 0) {
+    const share = summary.overageCents / summary.spendingLimitCents;
+    for (const threshold of [100, 80]) {
+      if (share * 100 >= threshold) {
+        alerts.push({
+          key: `spending_limit:${threshold}`,
+          subject: threshold === 100 ? "Your spending limit has been reached" : `You've used ${threshold}% of your spending limit`,
+          body: `Additional usage this period is $${(summary.overageCents / 100).toFixed(2)} of your $${(summary.spendingLimitCents / 100).toFixed(2)} limit.` +
+            (threshold === 100 ? " Cloud runs are paused until you raise the limit or the next period starts." : ""),
+        });
+        break;
+      }
+    }
+  }
+  return alerts;
+}
+
+export async function billingRecipients(env: Env, workspaceId: string): Promise<Array<{ email: string }>> {
+  const rows = await env.DB.prepare(`
+    SELECT DISTINCT u.email FROM workspace_memberships m
+    JOIN aegis_users u ON u.id = m.aegis_id
+    WHERE m.workspace_id = ? AND m.status = 'active' AND m.role IN ('owner', 'billing') AND u.email IS NOT NULL
+  `).bind(workspaceId).all<{ email: string }>();
+  return rows.results.map(({ email }) => ({ email }));
+}
+
+export type AlertHook = (env: Env, workspaceId: string, alert: ThresholdAlert) => Promise<void>;
+const hooks: AlertHook[] = [];
+/** Lets other channels (Slack) hear about the same alerts. */
+export function onUsageAlert(hook: AlertHook): void {
+  hooks.push(hook);
+}
+
+export async function checkUsageThresholds(env: Env, workspaceId: string, now = new Date()): Promise<void> {
+  const { summary } = await usageSummary(env, workspaceId, now);
+  const due = dueAlerts(summary);
+  if (!due.length) return;
+  const workspace = await env.DB.prepare("SELECT name FROM workspaces WHERE id = ?").bind(workspaceId).first<{ name: string }>();
+  const billingUrl = `${(env.DASHBOARD_URL ?? "https://app.bugmole.com").replace(/\/$/, "")}/billing?workspace=${encodeURIComponent(workspaceId)}`;
+  for (const alert of due) {
+    const claimed = await env.DB.prepare(`
+      INSERT INTO usage_alerts_sent (workspace_id, period_start, alert) VALUES (?, ?, ?)
+      ON CONFLICT DO NOTHING
+    `).bind(workspaceId, summary.period.start, alert.key).run();
+    if (!claimed.meta?.changes) continue;
+    const recipients = await billingRecipients(env, workspaceId);
+    if (recipients.length) {
+      const name = workspace?.name ?? workspaceId;
+      try {
+        await emailSender(env).send({
+          to: recipients,
+          subject: `${alert.subject} — ${name}`,
+          tag: "usage-alert",
+          text: `${alert.body}\n\nManage your plan and spending limit: ${billingUrl}`,
+          html: `<p>${escapeHtml(alert.body)}</p><p><a href="${billingUrl}">Manage your plan and spending limit</a></p>`,
+        });
+      } catch (error) {
+        console.error("[usage] alert email failed", error);
+      }
+    }
+    for (const hook of hooks) {
+      try {
+        await hook(env, workspaceId, alert);
+      } catch (error) {
+        console.error("[usage] alert hook failed", error);
+      }
+    }
+  }
+}

package/src/registry-worker/core.ts

@@ -0,0 +1,308 @@
+// Shared registry primitives: environment, principals, responses, crypto.
+// Route modules import these instead of reaching into index.ts.
+import type { EmailEnv } from "./email/sender.js";
+import { RUNNER_TOKEN_PREFIX, verifyRunnerToken, type RunnerIdentity } from "./runner/tokens.js";
+export interface Env extends EmailEnv {
+  DB: D1Database;
+  AEGIS_CLIENT_ID: string;
+  AEGIS_CLIENT_SECRET: string;
+  RUNS_COORDINATOR?: DurableObjectNamespace;
+  /** Bucket the cloud worker writes run evidence to. The artifact routes return 503 without it. */
+  ARTIFACTS?: R2Bucket;
+  REGISTRY_REALM_OWNER_TENANT_ID?: string;
+  /** Public dashboard origin, used in emails and links (e.g. https://app.bugmole.com). */
+  DASHBOARD_URL?: string;
+  /** Signs the short-lived tokens hosted runners use; see runner/tokens.ts. */
+  RUNNER_TOKEN_SECRET?: string;
+  /** Service binding to the valkyrie-runner Worker that starts cloud-run containers. */
+  RUNNER?: Fetcher;
+  /** Cloud runs executing at once across every workspace (default 20). */
+  CLOUD_POOL_MAX?: string;
+  /** Longest a cloud run may take before it is stopped, in minutes (default 60). */
+  CLOUD_RUN_MAX_MINUTES?: string;
+  /** This registry's public origin, handed to runners (e.g. https://api.bugmole.com). */
+  REGISTRY_PUBLIC_URL?: string;
+  /** Rollout flags; see flags.ts. */
+  [key: `FEATURE_${string}`]: string | undefined;
+}
+
+export type Principal = {
+  aegisId: string;
+  /** Sign-in email passed along by the dashboard, when known. */
+  email?: string;
+  /** Aegis confirmed that email (only asserted by the dashboard). */
+  emailVerified?: boolean;
+  projectIds: string[];
+  scopes: string[];
+  apiKeyId?: string;
+  /** Set for a hosted runner: it may act on this one run and nothing else. */
+  runner?: RunnerIdentity;
+  /**
+   * Enterprise sign-in: the Aegis organization (only asserted by the dashboard).
+   * Groups are sent at sign-in only; null on other requests.
+   */
+  org?: { id: string; groups: string[] | null };
+};
+
+/** Reads the dashboard's `x-aegis-org` / `x-aegis-groups` headers (groups as URI-encoded JSON). */
+export function orgFromHeaders(headers: Headers): Principal["org"] {
+  const id = headers.get("x-aegis-org")?.trim();
+  if (!id || !/^[0-9a-f-]{36}$/i.test(id)) return undefined;
+  const raw = headers.get("x-aegis-groups");
+  if (raw === null) return { id, groups: null };
+  let groups: string[] = [];
+  try {
+    const parsed = JSON.parse(decodeURIComponent(raw)) as unknown;
+    if (Array.isArray(parsed)) groups = parsed.filter((g): g is string => typeof g === "string").slice(0, 500);
+  } catch {
+    groups = [];
+  }
+  return { id, groups };
+}
+
+
+
+export const jsonHeaders = { "content-type": "application/json; charset=utf-8" };
+
+export function response(body: unknown, status = 200): Response {
+  return new Response(JSON.stringify(body), { status, headers: jsonHeaders });
+}
+
+export function bearer(request: Request): string | null {
+  const value = request.headers.get("authorization")?.trim();
+  return value?.toLowerCase().startsWith("bearer ") ? value.slice(7).trim() || null : null;
+}
+
+export async function sha256(value: string): Promise<string> {
+  const bytes = new TextEncoder().encode(value);
+  const digest = await crypto.subtle.digest("SHA-256", bytes);
+  return [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, "0")).join("");
+}
+
+export async function subscriptionPrincipal(request: Request, env: Env): Promise<Principal | null> {
+  const token = new URL(request.url).searchParams.get("token");
+  if (!token || !env.AEGIS_CLIENT_SECRET) return null;
+  try {
+    const decoded = new TextDecoder().decode(fromBase64(token));
+    const [aegisId, expires, signature] = decoded.split(".");
+    if (!aegisId || !expires || !signature || Number(expires) <= Date.now()) return null;
+    const key = await crypto.subtle.importKey(
+      "raw",
+      new TextEncoder().encode(env.AEGIS_CLIENT_SECRET),
+      { name: "HMAC", hash: "SHA-256" },
+      false,
+      ["sign"],
+    );
+    const digest = await crypto.subtle.sign(
+      "HMAC",
+      key,
+      new TextEncoder().encode(`${aegisId}.${expires}`),
+    );
+    const expected = [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, "0")).join("");
+    if (!constantTimeEqual(signature, expected)) return null;
+    return { aegisId, projectIds: [], scopes: ["runs:read"] };
+  } catch {
+    return null;
+  }
+}
+
+export async function apiPrincipal(request: Request, env: Env): Promise<Principal | null> {
+  const token = bearer(request);
+  if (token?.startsWith(RUNNER_TOKEN_PREFIX)) {
+    const runner = await verifyRunnerToken(env, token);
+    if (!runner) return null;
+    return { aegisId: `runner:${runner.kind}:${runner.id}`, projectIds: [runner.projectId], scopes: ["runs:write", "tasks:write"], runner };
+  }
+  if (token) {
+    const hash = await sha256(token);
+    const row = await env.DB.prepare(`
+      SELECT id, aegis_id, project_ids_json, scopes_json, expires_at
+      FROM api_keys
+      WHERE token_hash = ? AND revoked_at IS NULL
+    `).bind(hash).first<{
+      id: string;
+      aegis_id: string;
+      project_ids_json: string;
+      scopes_json: string;
+      expires_at: string | null;
+    }>();
+    if (!row || (row.expires_at && Date.parse(row.expires_at) <= Date.now())) return null;
+    return {
+      aegisId: row.aegis_id,
+      projectIds: JSON.parse(row.project_ids_json || "[]"),
+      scopes: JSON.parse(row.scopes_json || "[]"),
+      apiKeyId: row.id,
+    };
+  }
+
+  const aegisId = request.headers.get("x-aegis-id")?.trim();
+  const secret = request.headers.get("x-aegis-secret");
+  if (!aegisId || !secret || !env.AEGIS_CLIENT_SECRET) return null;
+  if (!constantTimeEqual(secret, env.AEGIS_CLIENT_SECRET)) return null;
+  const email = request.headers.get("x-aegis-email")?.trim().toLowerCase();
+  const validEmail = email && /^[^\s@]+@[^\s@]+$/.test(email) ? email : undefined;
+  const org = orgFromHeaders(request.headers);
+  return {
+    aegisId,
+    projectIds: [],
+    scopes: ["*"],
+    ...(validEmail ? { email: validEmail, emailVerified: request.headers.get("x-aegis-email-verified") === "true" } : {}),
+    ...(org ? { org } : {}),
+  };
+}
+
+export function constantTimeEqual(left: string, right: string): boolean {
+  const leftBytes = new TextEncoder().encode(left);
+  const rightBytes = new TextEncoder().encode(right);
+  if (leftBytes.length !== rightBytes.length) return false;
+  let result = 0;
+  for (let index = 0; index < leftBytes.length; index++) result |= leftBytes[index] ^ rightBytes[index];
+  return result === 0;
+}
+
+export function can(principal: Principal, scope: string): boolean {
+  return principal.scopes.includes("*") || principal.scopes.includes(scope);
+}
+
+export function projectAllowed(principal: Principal, projectId: string): boolean {
+  return principal.projectIds.length === 0 || principal.projectIds.includes(projectId);
+}
+
+export const WORKSPACE_ROLES = ["owner", "admin", "member", "billing"] as const;
+export type WorkspaceRole = (typeof WORKSPACE_ROLES)[number];
+
+/**
+ * What each role may do. Billing managers see everything but only change
+ * billing; members run tests and edit plans; admins also manage the workspace
+ * and its people; owners do all of it, including billing.
+ */
+export const PERMISSIONS = {
+  read: ["owner", "admin", "member", "billing"],
+  "project.write": ["owner", "admin", "member"],
+  "project.manage": ["owner", "admin"],
+  "workspace.write": ["owner", "admin", "member"],
+  "workspace.manage": ["owner", "admin"],
+  "members.manage": ["owner", "admin"],
+  "billing.manage": ["owner", "billing"],
+  "audit.read": ["owner", "admin"],
+} as const satisfies Record<string, readonly WorkspaceRole[]>;
+export type Permission = keyof typeof PERMISSIONS;
+
+export function isWorkspaceRole(value: unknown): value is WorkspaceRole {
+  return typeof value === "string" && (WORKSPACE_ROLES as readonly string[]).includes(value);
+}
+
+export function roleAllows(role: string | null | undefined, permission: Permission): boolean {
+  return isWorkspaceRole(role) && (PERMISSIONS[permission] as readonly string[]).includes(role);
+}
+
+/** The caller's role in a workspace, or null when they aren't an active member. */
+export async function workspaceRole(env: Env, aegisId: string, workspaceId: string): Promise<WorkspaceRole | null> {
+  const row = await env.DB.prepare(`
+    SELECT role FROM workspace_memberships
+    WHERE workspace_id = ? AND aegis_id = ? AND status = 'active'
+  `).bind(workspaceId, aegisId).first<{ role: string }>();
+  return isWorkspaceRole(row?.role) ? row.role : null;
+}
+
+/**
+ * The caller's role for a project: their workspace role, or for projects
+ * registered before workspaces existed, their direct project membership.
+ */
+export async function projectRole(env: Env, aegisId: string, projectId: string): Promise<WorkspaceRole | null> {
+  const row = await env.DB.prepare(`
+    SELECT wm.role AS workspaceRole, pm.role AS projectRole
+    FROM projects p
+    LEFT JOIN workspace_memberships wm
+      ON wm.workspace_id = p.workspace_id AND wm.aegis_id = ? AND wm.status = 'active'
+    LEFT JOIN project_memberships pm
+      ON pm.project_id = p.id AND pm.aegis_id = ? AND pm.status = 'active'
+    WHERE p.id = ? AND p.deleted_at IS NULL
+  `).bind(aegisId, aegisId, projectId).first<{ workspaceRole: string | null; projectRole: string | null }>();
+  if (!row) return null;
+  if (isWorkspaceRole(row.workspaceRole)) return row.workspaceRole;
+  return isWorkspaceRole(row.projectRole) ? row.projectRole : null;
+}
+
+/**
+ * Whether the caller may act on a project. API keys are limited to the
+ * projects they were issued for *and* to what their creator can still do, so
+ * removing someone from a workspace also disables their keys.
+ */
+export async function memberOfProject(
+  env: Env,
+  principal: Principal,
+  projectId: string,
+  permission: Permission = "read",
+): Promise<boolean> {
+  if (principal.projectIds.length > 0 && !principal.projectIds.includes(projectId)) return false;
+  // A runner reads its project's plans and writes its own run, nothing more.
+  if (principal.runner) return principal.runner.projectId === projectId && (permission === "read" || permission === "project.write");
+  if (!roleAllows(await projectRole(env, principal.aegisId, projectId), permission)) return false;
+  // Projects beyond a downgraded plan's limit are kept, read-only.
+  if (permission !== "read" && (await projectLocked(env, projectId))) return false;
+  return true;
+}
+
+export async function projectLocked(env: Env, projectId: string): Promise<boolean> {
+  const row = await env.DB.prepare("SELECT locked_at AS lockedAt FROM projects WHERE id = ?").bind(projectId).first<{ lockedAt: string | null }>();
+  return Boolean(row?.lockedAt);
+}
+
+export async function memberOfWorkspace(
+  env: Env,
+  principal: Principal,
+  workspaceId: string,
+  permission: Permission = "read",
+): Promise<boolean> {
+  if (principal.runner) return false;
+  return roleAllows(await workspaceRole(env, principal.aegisId, workspaceId), permission);
+}
+
+export function randomToken(bytes = 32): string {
+  const data = new Uint8Array(bytes);
+  crypto.getRandomValues(data);
+  return btoa(String.fromCharCode(...data)).replaceAll("+", "-").replaceAll("/", "_").replaceAll("=", "");
+}
+
+export function base64(value: Uint8Array): string {
+  return btoa(String.fromCharCode(...value)).replaceAll("+", "-").replaceAll("/", "_").replaceAll("=", "");
+}
+
+export function fromBase64(value: string): Uint8Array {
+  const padded = value.replaceAll("-", "+").replaceAll("_", "/") + "=".repeat((4 - value.length % 4) % 4);
+  return Uint8Array.from(atob(padded), (character) => character.charCodeAt(0));
+}
+
+export async function encryptForSecret(value: string, secret: string): Promise<{ ciphertext: string; iv: string }> {
+  const secretHash = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(secret));
+  const key = await crypto.subtle.importKey("raw", secretHash, "AES-GCM", false, ["encrypt"]);
+  const iv = crypto.getRandomValues(new Uint8Array(12));
+  const encrypted = await crypto.subtle.encrypt(
+    { name: "AES-GCM", iv },
+    key,
+    new TextEncoder().encode(value),
+  );
+  return { ciphertext: base64(new Uint8Array(encrypted)), iv: base64(iv) };
+}
+
+export async function decryptForSecret(ciphertext: string, iv: string, secret: string): Promise<string> {
+  const secretHash = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(secret));
+  const key = await crypto.subtle.importKey("raw", secretHash, "AES-GCM", false, ["decrypt"]);
+  const decrypted = await crypto.subtle.decrypt(
+    { name: "AES-GCM", iv: fromBase64(iv) },
+    key,
+    fromBase64(ciphertext),
+  );
+  return new TextDecoder().decode(decrypted);
+}
+
+
+export function newId(prefix: string): string {
+  return `${prefix}_${randomToken(12)}`;
+}
+
+export async function requestJson(request: Request): Promise<Record<string, unknown> | null> {
+  const body = await request.json().catch(() => null);
+  return body && typeof body === "object" && !Array.isArray(body) ? body as Record<string, unknown> : null;
+}

package/src/registry-worker/devices/devices.test.ts

@@ -0,0 +1,185 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import type { Env } from "../core.js";
+import { call, createTestEnv, seedPlan, seedProject } from "../testing/harness.js";
+import { runnerLauncher, type LaunchRequest } from "../runner/dispatch.js";
+import { sweepDeviceSessions } from "./routes.js";
+
+const ZIP = new Uint8Array([0x50, 0x4b, 0x03, 0x04, 1, 2, 3]);
+const PROJECT_ARN = "arn:aws:devicefarm:us-west-2:123456789012:project:abc-1";
+
+function fakeClouds() {
+  const state = { deviceFarm: [] as string[], sessionStatus: "RUNNING", metered: 3, stopped: false };
+  const doFetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
+    const url = String(input);
+    const auth = new Headers(init?.headers).get("authorization") ?? "";
+    if (url.includes("browserstack.com/app-automate/devices.json")) {
+      if (auth !== `Basic ${btoa("ada:good-key")}`) return new Response("", { status: 401 });
+      return Response.json([{ os: "ios", os_version: "17", device: "iPhone 15", realMobile: true }, { os: "android", os_version: "14.0", device: "Google Pixel 8", realMobile: true }]);
+    }
+    if (url.includes("devicefarm")) {
+      const action = new Headers(init?.headers).get("x-amz-target")!.split(".")[1];
+      state.deviceFarm.push(action);
+      if (action === "ListDevices") return Response.json({ devices: [{ arn: "arn:aws:devicefarm:us-west-2::device:PIXEL8", name: "Google Pixel 8", platform: "ANDROID", os: "14" }] });
+      if (action === "CreateRemoteAccessSession") return Response.json({ remoteAccessSession: { arn: "arn:session:1", status: "PENDING" } });
+      if (action === "GetRemoteAccessSession") {
+        return Response.json({ remoteAccessSession: state.stopped
+          ? { arn: "arn:session:1", status: "COMPLETED", deviceMinutes: { metered: state.metered } }
+          : { arn: "arn:session:1", status: state.sessionStatus, endpoints: { remoteDriverEndpoint: "https://appium.test/wd" } } });
+      }
+      if (action === "StopRemoteAccessSession") {
+        state.stopped = true;
+        return Response.json({});
+      }
+      if (action === "ListArtifacts") return Response.json({ artifacts: [{ name: "Video", type: "VIDEO", extension: "mp4", url: "https://s3.test/video.mp4" }] });
+    }
+    return new Response("not found", { status: 404 });
+  }) as unknown as typeof fetch;
+  return { state, doFetch };
+}
+
+async function business(clouds: ReturnType<typeof fakeClouds>, plan = "business") {
+  const setup = createTestEnv({
+    FEATURE_DEVICE_CLOUDS: "on",
+    FEATURE_CLOUD_RUNNERS: "on",
+    SECRETS_ENCRYPTION_KEY: "k",
+    RUNNER_TOKEN_SECRET: "r",
+    DEVICE_FARM_FETCH: clouds.doFetch,
+    BUGMOLE_DEVICE_FARM_ACCESS_KEY_ID: "AKIABUGMOLE",
+    BUGMOLE_DEVICE_FARM_SECRET_ACCESS_KEY: "bugmole-secret",
+    BUGMOLE_DEVICE_FARM_PROJECT_ARN: PROJECT_ARN,
+  } as Partial<Env>);
+  const project = await seedProject(setup.env, { storageProvider: "bugmole" });
+  setup.db.db.prepare("INSERT INTO workspace_plans (workspace_id, plan) VALUES (?, ?)").run(project.workspaceId, plan);
+  await seedPlan(setup.env, project.projectId, project.owner);
+  return { ...setup, ...project };
+}
+
+test("a device cloud is checked before it's saved, and its key never comes back", async () => {
+  const clouds = fakeClouds();
+  const { env, db, workspaceId, owner } = await business(clouds);
+  const add = (body: Record<string, unknown>) => call(env, "POST", `/api/workspaces/${workspaceId}/device-clouds`, { as: owner, body });
+  const rejected = await add({ provider: "browserstack", username: "ada", secret: "wrong" });
+  assert.equal(rejected.status, 422);
+  assert.match(rejected.json.error, /didn't accept/);
+  assert.equal((await add({ provider: "aws_device_farm", username: "AKIA", secret: "s", projectArn: "arn:aws:s3:::x" })).status, 422);
+  const added = await add({ provider: "browserstack", username: "ada", secret: "good-key", label: "Main" });
+  assert.equal(added.status, 201);
+  const stored = db.db.prepare("SELECT secret_ciphertext AS c FROM device_cloud_credentials").get() as any;
+  assert.ok(!String(stored.c).includes("good-key"));
+  const listed = await call(env, "GET", `/api/workspaces/${workspaceId}/device-clouds`, { as: owner });
+  assert.ok(!JSON.stringify(listed.json).includes("good-key"));
+  assert.equal(listed.json.bugmole.available, true);
+  const devices = await call(env, "GET", `/api/workspaces/${workspaceId}/device-clouds/${added.json.connection.id}/devices`, { as: owner });
+  assert.deepEqual(devices.json.devices.map((device: { name: string }) => device.name), ["Google Pixel 8", "iPhone 15"]);
+  const own = await call(env, "GET", `/api/workspaces/${workspaceId}/device-clouds/bugmole/devices`, { as: owner });
+  assert.equal(own.json.devices[0].deviceArn, "arn:aws:devicefarm:us-west-2::device:PIXEL8");
+  assert.equal((await call(env, "DELETE", `/api/workspaces/${workspaceId}/device-clouds/${added.json.connection.id}`, { as: owner })).status, 200);
+  assert.equal(db.db.prepare("SELECT secret_ciphertext AS c FROM device_cloud_credentials").get()?.c, "", "the key is wiped when disconnected");
+});
+
+test("device clouds are a Business feature", async () => {
+  const clouds = fakeClouds();
+  const { env, workspaceId, owner, projectId } = await business(clouds, "team");
+  assert.equal((await call(env, "POST", `/api/workspaces/${workspaceId}/device-clouds`, { as: owner, body: { provider: "browserstack", username: "ada", secret: "good-key" } })).status, 402);
+  const run = await call(env, "POST", `/api/projects/${projectId}/runs`, {
+    as: owner, body: { planId: "checkout", idempotencyKey: "t", targets: [{ kind: "device", provider: "browserstack", credentialId: "x", platform: "ios", deviceName: "iPhone 15", browserName: "safari" }] },
+  });
+  assert.equal(run.status, 402);
+});
+
+test("app builds are checked, stored, and only usable in their own project", async () => {
+  const clouds = fakeClouds();
+  const { env, artifacts, projectId, owner } = await business(clouds);
+  const upload = (query: string, body: BodyInit) => call(env, "PUT", `/api/projects/${projectId}/app-builds?${query}`, { as: owner, body: body as string, headers: { "content-type": "application/octet-stream" } });
+  assert.equal((await upload("platform=android&filename=shop.ipa", Buffer.from(ZIP) as unknown as string)).status, 422);
+  assert.equal((await upload("platform=android&filename=shop.apk", "not a zip")).status, 422);
+  const ok = await call(env, "PUT", `/api/projects/${projectId}/app-builds?platform=android&filename=shop.apk`, {
+    as: owner, body: new TextDecoder("latin1").decode(ZIP), headers: { "content-type": "application/octet-stream" },
+  });
+  assert.equal(ok.status, 201, JSON.stringify(ok.json));
+  assert.ok([...artifacts.objects.keys()].some((key) => key.startsWith(`projects/${projectId}/app-builds/`)));
+  const listed = await call(env, "GET", `/api/projects/${projectId}/app-builds`, { as: owner });
+  assert.equal(listed.json.builds[0].filename, "shop.apk");
+
+  const wrongPlatform = await call(env, "POST", `/api/projects/${projectId}/runs`, {
+    as: owner, body: { planId: "checkout", idempotencyKey: "w", targets: [{ kind: "device", provider: "bugmole", platform: "ios", deviceName: "iPhone 15", appBuildId: ok.json.build.id }], executionMode: "cloud" },
+  });
+  assert.equal(wrongPlatform.json.code, "app_build_platform");
+});
+
+test("device targets are checked when a run is created", async () => {
+  const clouds = fakeClouds();
+  const { env, projectId, owner } = await business(clouds);
+  const create = (target: Record<string, unknown>, executionMode = "self") => call(env, "POST", `/api/projects/${projectId}/runs`, {
+    as: owner, headers: { "x-aegis-email": "o@acme.test", "x-aegis-email-verified": "true" },
+    body: { planId: "checkout", idempotencyKey: crypto.randomUUID(), executionMode, targets: [target] },
+  });
+  const bugmoleWeb = { kind: "device", provider: "bugmole", platform: "android", deviceName: "Google Pixel 8", browserName: "chrome" };
+  assert.equal((await create(bugmoleWeb)).json.code, "bugmole_devices_need_cloud");
+  assert.equal((await create({ ...bugmoleWeb, real: false }, "cloud")).json.code, "bugmole_real_only");
+  assert.equal((await create({ kind: "device", provider: "kobiton", credentialId: "nope", platform: "android", deviceName: "Pixel", browserName: "chrome" })).json.code, "device_credential_missing");
+  assert.equal((await create({ kind: "device", provider: "kobiton", platform: "android", deviceName: "Pixel", browserName: "chrome" })).status, 422);
+  const ok = await create(bugmoleWeb, "cloud");
+  assert.equal(ok.status, 201, JSON.stringify(ok.json));
+  assert.equal(ok.json.run.targets[0].provider, "bugmole");
+});
+
+test("a cloud run gets Bugmole's devices through the registry, and pays for the minutes it used", async () => {
+  const clouds = fakeClouds();
+  const { env, db, projectId, owner, workspaceId } = await business(clouds);
+  const launches: LaunchRequest[] = [];
+  const original = runnerLauncher.start;
+  runnerLauncher.start = async (_env, launch) => {
+    launches.push(launch);
+  };
+  try {
+    const target = { kind: "device", provider: "bugmole", platform: "android", deviceName: "Google Pixel 8", browserName: "chrome" };
+    const created = await call(env, "POST", `/api/projects/${projectId}/runs`, {
+      as: owner, headers: { "x-aegis-email": "o@acme.test", "x-aegis-email-verified": "true" },
+      body: { planId: "checkout", idempotencyKey: "d1", executionMode: "cloud", targets: [target] },
+    });
+    const runId = created.json.run.id;
+    const token = launches[0].token;
+
+    const access = await call(env, "GET", `/api/runs/${runId}/device-access`, { token });
+    assert.deepEqual(access.json, { credentials: {}, appBuilds: [], bugmole: true });
+    const asOwner = await call(env, "GET", `/api/runs/${runId}/device-access`, { as: owner });
+    assert.equal(asOwner.json.bugmole, false, "only the cloud runner opens Bugmole's devices");
+
+    assert.equal((await call(env, "POST", `/api/runs/${runId}/device-sessions`, { token, body: { device: { ...target, deviceName: "iPhone 15" } } })).status, 422);
+    const opened = await call(env, "POST", `/api/runs/${runId}/device-sessions`, { token, body: { device: target, name: "checkout" } });
+    assert.equal(opened.status, 201, JSON.stringify(opened.json));
+    assert.ok(!JSON.stringify(opened.json).includes("bugmole-secret"), "AWS keys stay in the registry");
+    const status = await call(env, "GET", `/api/runs/${runId}/device-sessions/${opened.json.sessionId}`, { token });
+    assert.deepEqual(status.json, { status: "running", hubUrl: "https://appium.test/wd" });
+    assert.equal((await call(env, "POST", `/api/runs/${runId}/device-sessions`, { as: owner, body: { device: target } })).status, 403);
+
+    const finished = await call(env, "POST", `/api/runs/${runId}/device-sessions/${opened.json.sessionId}/finish`, { token, body: { passed: true } });
+    assert.deepEqual(finished.json, { links: [], videoUrl: "https://s3.test/video.mp4", meteredMinutes: 3 });
+    await call(env, "POST", `/api/runs/${runId}/device-sessions/${opened.json.sessionId}/finish`, { token, body: { passed: true } });
+    const usage = await call(env, "GET", `/api/workspaces/${workspaceId}/usage`, { as: owner });
+    assert.equal(usage.json.usage.lines.device_minute.used, 3, "billed once");
+    assert.equal(usage.json.usage.overageCents, 75, "3 minutes × $0.25");
+
+    // Once the run is over, credentials are no longer handed out.
+    db.db.prepare("UPDATE runs SET status = 'success' WHERE id = ?").run(runId);
+    assert.equal((await call(env, "GET", `/api/runs/${runId}/device-access`, { token })).status, 409);
+  } finally {
+    runnerLauncher.start = original;
+  }
+});
+
+test("a session its runner abandoned is stopped and billed by the sweep", async () => {
+  const clouds = fakeClouds();
+  const { env, db, projectId, workspaceId } = await business(clouds);
+  db.db.prepare("INSERT INTO runs (id, project_id, tenant_id, plan_id, journey_id, requested_by, idempotency_key, status, workspace_id) VALUES ('run_x', ?, ?, 'checkout', 'checkout', 'u', 'k', 'error', ?)")
+    .run(projectId, workspaceId, workspaceId);
+  db.db.prepare("INSERT INTO bugmole_device_sessions (id, run_id, project_id, workspace_id, device_name, session_arn, status) VALUES ('vds_1', 'run_x', ?, ?, 'Pixel', 'arn:session:1', 'running')")
+    .run(projectId, workspaceId);
+  assert.equal(await sweepDeviceSessions(env), 1);
+  const row = db.db.prepare("SELECT status, metered_minutes AS minutes FROM bugmole_device_sessions WHERE id = 'vds_1'").get() as any;
+  assert.deepEqual({ ...row }, { status: "finished", minutes: 3 });
+  assert.ok(clouds.state.deviceFarm.includes("StopRemoteAccessSession"));
+  assert.equal(await sweepDeviceSessions(env), 0);
+});