@caelo-cms/provisioning 0.10.55 → 0.10.57

package/src/upgrader-assess.ts

@@ -27,6 +27,8 @@
  * | target not newer than installed (downgrade/same)    | refused           |
  * | installed below the release's minFromVersion        | refused           |
  * | release raises the infrastructure contract          | needs_cloud_shell |
+ * |   … on the converge route (#819, the converge job)  | proceed           |
+ * |   … converge route, automatic, autoConverge off     | refused           |
  * | contract (destructive) migrations, automatic run    | refused           |
  * | contract migrations, Owner's click                  | proceed + warning |
  * | automatic run the update policy does not allow      | refused           |
@@ -38,6 +40,13 @@
  * that rolls the images. So for them `installed.infraContractVersion` is
  * null and the infra-contract row never applies; every "needs_cloud_shell"
  * stop names the upgrade command on the server instead of Cloud Shell.
+ *
+ * The converge job (#819, converge-job.ts) asks with `route: "converge"`:
+ * it runs as the install's provisioner, so an infrastructure change is what
+ * it is for — every other row still applies. An automatic run on the
+ * converge route needs the Owner's `autoConverge` (#864) on top of
+ * everything an automatic install needs; the scheduler queues one only
+ * then, and the job re-checks it here against the policy it claimed.
  */
 
 import {
@@ -60,6 +69,7 @@ import { needsConverge } from "./upgrade/converge.js";
 /** Machine-readable reason codes (`platform_update_requests.refusal_codes`). */
 export type RefusalCode =
   | "registry"
+  | "manifest-missing"
   | "signature"
   | "manifest-unreadable"
   | "install-version-unknown"
@@ -134,6 +144,11 @@ export function assessUpgradeRequest(input: {
   readonly now: Date;
   /** Default `cloud-shell` (the GCP upgrader job). */
   readonly manualPath?: ManualUpgradePath;
+  /**
+   * Which job asks (#819). `converge` = the converge job: an infrastructure
+   * contract bump is installed, not handed to Cloud Shell. Default `app_roll`.
+   */
+  readonly route?: "app_roll" | "converge";
 }): RequestDecision {
   const { request, installed } = input;
   const by = manualRun(input.manualPath ?? "cloud-shell");
@@ -142,10 +157,20 @@ export function assessUpgradeRequest(input: {
     const e = input.manifest.error;
     switch (e.code) {
       case "registry":
+        // Only an outage is worth retrying: a manifest the registry says
+        // does not exist (404 / other 4xx) never appears, so retrying would
+        // loop on "try again later" forever.
+        if (e.transient) {
+          return stop(
+            "failed",
+            "registry",
+            `Could not reach the Caelo release registry to verify v${target} (${e.message}). Nothing was changed; try again later.`,
+          );
+        }
         return stop(
-          "failed",
-          "registry",
-          `Could not reach the Caelo release registry to verify v${target} (${e.message}). Nothing was changed; try again later.`,
+          "refused",
+          "manifest-missing",
+          `v${target} has no signed release manifest in the Caelo release registry, so it cannot be verified and will not be installed. Nothing was changed; retrying will not help. (${e.message})`,
         );
       case "schema":
         return stop(
@@ -170,7 +195,18 @@ export function assessUpgradeRequest(input: {
       manifest,
     );
   }
-  if (compareSemver(target, installed.version) <= 0) {
+  const order = compareSemver(target, installed.version);
+  if (order === 0) {
+    // Reached after a run that rolled but could not record it (#860): the
+    // next run re-records the serving release, which then is the target.
+    return stop(
+      "refused",
+      "not-newer",
+      `v${target} already serves on this install, so there is nothing to install. Nothing was changed.`,
+      manifest,
+    );
+  }
+  if (order < 0) {
     return stop(
       "refused",
       "not-newer",
@@ -188,20 +224,33 @@ export function assessUpgradeRequest(input: {
   }
   // Self-hosted: the target's own CLI renders the host setup in this run.
   const infraContractVersion = installed.infraContractVersion ?? manifest.infraContractVersion;
-  if (
+  const converge = input.route === "converge";
+  const infraChange =
     installed.infraContractVersion !== null &&
-    needsConverge({ infraContractVersion }, manifest.infraContractVersion)
-  ) {
+    needsConverge({ infraContractVersion }, manifest.infraContractVersion);
+  if (infraChange && !converge) {
     return stop(
       "needs_cloud_shell",
       "infra-contract",
-      `v${target} changes this install's cloud setup (infrastructure contract ${infraContractVersion} → ${manifest.infraContractVersion}: ${infraStepsText(infraContractVersion, manifest.infraContractVersion)}). The admin may not change cloud permissions, so this release is installed once with your own Google account in Cloud Shell.`,
+      `v${target} changes this install's cloud setup (infrastructure contract ${infraContractVersion} → ${manifest.infraContractVersion}: ${infraStepsText(infraContractVersion, manifest.infraContractVersion)}). The upgrader may not change cloud permissions, so Caelo installs this release with its converge job: start it again from Settings → Updates. On an install without the converge job, install it once with your own Google account in Cloud Shell.`,
+      manifest,
+    );
+  }
+  if (converge && request.trigger === "auto" && !input.policy.autoConverge) {
+    return stop(
+      "refused",
+      "policy",
+      `Not installed automatically: v${target} changes this install's cloud setup, and the update policy does not install those automatically. Install it from Settings → Updates.`,
       manifest,
     );
   }
+  // On the converge route the contract bump is the job's work, not a
+  // reason for a manual install: only the migrations decide below.
   const assessment = assessAutoInstall(manifest, {
     version: installed.version,
-    infraContractVersion,
+    infraContractVersion: converge
+      ? Math.max(infraContractVersion, manifest.infraContractVersion)
+      : infraContractVersion,
   });
   const destructive = assessment.mode === "manual";
   const warnings: string[] = [];

package/src/upgrader-gcp.ts

@@ -3,8 +3,8 @@
 /**
  * The real side effects of the upgrader job on Google Cloud (epic #696):
  * gcloud as the job's own service account, the install's ops bucket for
- * install.json, the fixed release registry, the migration job and the IAP
- * health probe. upgrader-job.ts takes them as {@link UpgraderJobDeps}; the
+ * install.json, the fixed release registry, the migration job, the IAP
+ * health probe and the IAM self-check (#818). upgrader-job.ts takes them as {@link UpgraderJobDeps}; the
  * database queue and the job log come from `@caelo-cms/platform-updater`,
  * which owns the Query API connection.
  *
@@ -24,12 +24,14 @@ import { readAdminConfig, runMigrationsViaCloudRunJob } from "./migration-runner
 import { fetchVerifiedReleaseManifest } from "./release-manifest.js";
 import type { HttpFetch } from "./runtime-secrets.js";
 import { opsBucketName } from "./state-backend.js";
-import { acquireBucketStateLock, GcsStateStore } from "./state-store.js";
+import { acquireBucketStateLock, GcsStateStore, realSyncGcloud } from "./state-store.js";
 import type { AppRollPlan } from "./upgrade/app-roll.js";
 import { findServiceUrl } from "./upgrade/gcp-lookup.js";
 import { createIapHealthProbe } from "./upgrade/health.js";
+import { restIamTester } from "./upgrader-iam-preflight.js";
 import type { UpdateQueue, UpgraderJobDeps } from "./upgrader-job.js";
 import type { JobLog } from "./upgrader-reporter.js";
+import { UPGRADER_TASK_TIMEOUT_MINUTES } from "./upgrader-setup.js";
 
 /** Where the job runs, from the metadata server. */
 export interface UpgraderPlacement {
@@ -55,8 +57,11 @@ export async function upgraderDatabaseHost(
   return cfg.databaseHost;
 }
 
-/** The IAP-backed deep health probe for `plan`'s install. */
-async function healthProbeFor(run: GcloudRunner, http: HttpFetch, plan: AppRollPlan) {
+/**
+ * The IAP-backed deep health probe for `plan`'s install (the upgrader job;
+ * the converge job, #819, too).
+ */
+export async function healthProbeFor(run: GcloudRunner, http: HttpFetch, plan: AppRollPlan) {
   const { install } = plan;
   const urls: Partial<Record<"admin" | "gateway", string>> = {};
   for (const s of plan.services) {
@@ -102,12 +107,46 @@ export function gcpUpgraderDeps(
     region: placement.region,
     env: GCP_STACK_ENV,
     execution: placement.execution,
+    // #818 — the IAM self-check asks as the job's own account (gcloud's
+    // token from the metadata server), so it sees exactly the job's grants.
+    iam: restIamTester({
+      http,
+      token: async () => {
+        const t = await run(["auth", "print-access-token"]);
+        if (!t.ok) throw new Error(`gcloud auth print-access-token: ${t.stderr.trim()}`);
+        return t.stdout.trim();
+      },
+    }),
     // The default deps: the fixed registry and signer, never overridden.
     fetchManifest: (version) => fetchVerifiedReleaseManifest(version),
-    lockState: () =>
-      acquireBucketStateLock(opsBucketName(placement.projectId), "caelo upgrader job"),
+    // #815 — the lock names this execution, so a later run can break it
+    // once Cloud Run says this one has ended (killed at its task timeout,
+    // OOM). Outside Cloud Run there is no execution to vouch for it.
+    lockState: (onBreak) =>
+      acquireBucketStateLock(
+        opsBucketName(placement.projectId),
+        "caelo upgrader job",
+        realSyncGcloud,
+        {
+          onBreak,
+          ...(placement.execution
+            ? {
+                holder: {
+                  execution: {
+                    projectId: placement.projectId,
+                    region: placement.region,
+                    name: placement.execution,
+                  },
+                  maxRuntimeMs: UPGRADER_TASK_TIMEOUT_MINUTES * 60_000,
+                },
+              }
+            : {}),
+        },
+      ),
+    // Uncached on purpose: the job re-reads install.json once it holds the
+    // lock to see whether another writer changed it meanwhile (#815).
     readInstall: () => {
-      const text = store.read("install.json");
+      const text = store.peek("install.json");
       return text === null ? null : (JSON.parse(text) as InstallMetadata);
     },
     writeInstall: (meta) => store.write("install.json", meta),

package/src/upgrader-iam-preflight.test.ts

@@ -0,0 +1,393 @@
+// SPDX-License-Identifier: MPL-2.0
+
+/**
+ * The upgrader's IAM self-check (#818): which permissions it asks for, how
+ * it groups and judges IAM's answers, the exact REST calls it makes, and
+ * that it never asks for a permission upgrader-setup.ts does not grant (a
+ * check the setup cannot pass would stop every update).
+ */
+
+import { describe, expect, it } from "bun:test";
+import type { HttpFetch } from "./runtime-secrets.js";
+import {
+  type IamResource,
+  type IamTester,
+  objectProbeCall,
+  PREFLIGHT_PROBE_OBJECT,
+  preflightMissingReason,
+  resourceName,
+  restIamTester,
+  runIamPreflight,
+  testIamCall,
+  upgraderAccessChecks,
+} from "./upgrader-iam-preflight.js";
+import {
+  UPGRADER_FORBIDDEN_ROLES,
+  UPGRADER_OPS_LIST_ROLE,
+  UPGRADER_PROJECT_ROLES,
+  UPGRADER_RUN_ROLE,
+  UPGRADER_SQL_BACKUP_ROLE,
+} from "./upgrader-setup.js";
+
+const TARGET = { projectId: "acme", region: "europe-west1", env: "production" };
+
+/**
+ * The permissions of the predefined roles upgrader-setup.ts grants, as far
+ * as the self-check needs them (cloud.google.com/run/docs/reference/iam/roles,
+ * cloud.google.com/iam/docs/understanding-roles). Only the ones asked for
+ * are listed; a new check must find its permission here or in a custom role.
+ */
+const PREDEFINED: Record<string, readonly string[]> = {
+  "roles/cloudsql.instanceUser": ["cloudsql.instances.get", "cloudsql.instances.login"],
+  "roles/logging.logWriter": ["logging.logEntries.create"],
+  "roles/iam.serviceAccountUser": ["iam.serviceAccounts.actAs"],
+  "roles/iam.serviceAccountTokenCreator": ["iam.serviceAccounts.signJwt"],
+  "roles/storage.objectUser": [
+    "storage.objects.get",
+    "storage.objects.create",
+    "storage.objects.delete",
+    "storage.objects.list",
+  ],
+  // The public release mirror grants allUsers roles/artifactregistry.reader.
+  "roles/artifactregistry.reader": ["artifactregistry.repositories.downloadArtifacts"],
+};
+
+/** What upgrader-setup.ts grants the upgrader's account, per resource name. */
+function setupGrants(): Map<string, Set<string>> {
+  const p = TARGET.projectId;
+  const of = (...roles: string[]) => new Set(roles.flatMap((r) => PREDEFINED[r] ?? []));
+  const project = of(...UPGRADER_PROJECT_ROLES);
+  for (const perm of UPGRADER_SQL_BACKUP_ROLE.permissions) project.add(perm);
+  // #883 — the Cloud Run rights come from the custom caeloUpgraderRun role
+  // (no run.developer: it would hold runWithOverrides on the provisioner's jobs).
+  for (const perm of UPGRADER_RUN_ROLE.permissions) project.add(perm);
+  const bucketList = new Set(UPGRADER_OPS_LIST_ROLE.permissions);
+  return new Map([
+    [`projects/${p}`, project],
+    [
+      `projects/${p}/serviceAccounts/caelo-production-run-sa@${p}.iam.gserviceaccount.com`,
+      of("roles/iam.serviceAccountUser"),
+    ],
+    [
+      `projects/${p}/serviceAccounts/caelo-production-gateway-sa@${p}.iam.gserviceaccount.com`,
+      of("roles/iam.serviceAccountUser"),
+    ],
+    [
+      `projects/${p}/serviceAccounts/caelo-production-upgrader@${p}.iam.gserviceaccount.com`,
+      of("roles/iam.serviceAccountUser"),
+    ],
+    [
+      `projects/${p}/serviceAccounts/caelo-mcp@${p}.iam.gserviceaccount.com`,
+      of("roles/iam.serviceAccountTokenCreator"),
+    ],
+    [
+      `projects/${p}/serviceAccounts/caelo-production-audit@${p}.iam.gserviceaccount.com`,
+      of("roles/iam.serviceAccountUser"),
+    ],
+    // The project-level caeloUpgraderRun reaches the job too.
+    [`projects/${p}/locations/europe-west1/jobs/caelo-production-upgrader`, project],
+    [`gs://${p}-caelo-ops`, bucketList],
+    // The install/-conditioned roles/storage.objectUser grant.
+    [`gs://${p}-caelo-ops/install/`, of("roles/storage.objectUser")],
+    [
+      "projects/caelo-website/locations/europe-west1/repositories/caelo-cms-images",
+      of("roles/artifactregistry.reader"),
+    ],
+  ]);
+}
+
+/** A tester answering from `grants`, recording every question. */
+function tester(grants: Map<string, Set<string>>, asked: string[] = []): IamTester {
+  return {
+    granted: async (resource, permissions) => {
+      asked.push(`${resourceName(resource)} ${permissions.join(",")}`);
+      const held = grants.get(resourceName(resource)) ?? new Set<string>();
+      return { ok: true, granted: permissions.filter((x) => held.has(x)) };
+    },
+  };
+}
+
+describe("upgraderAccessChecks", () => {
+  it("asks only for what upgrader-setup.ts grants: a fresh setup passes the self-check", async () => {
+    const r = await runIamPreflight(upgraderAccessChecks(TARGET), tester(setupGrants()));
+    expect(r).toEqual({ kind: "ok" });
+  });
+
+  it("#879 review: the audit job's account is checked, since every app-roll moves that job (#875)", async () => {
+    const grants = setupGrants();
+    grants.delete(
+      "projects/acme/serviceAccounts/caelo-production-audit@acme.iam.gserviceaccount.com",
+    );
+    const r = await runIamPreflight(upgraderAccessChecks(TARGET), tester(grants));
+    expect(r.kind).toBe("missing");
+    if (r.kind !== "missing") return;
+    expect(r.missing.map((m) => [resourceName(m.check.resource), m.missing])).toEqual([
+      [
+        "projects/acme/serviceAccounts/caelo-production-audit@acme.iam.gserviceaccount.com",
+        ["iam.serviceAccounts.actAs"],
+      ],
+    ]);
+    expect(preflightMissingReason(r.missing)).toContain("quality-audit job");
+  });
+
+  it("never needs a permission only a forbidden role would give (no compute, no IAM admin)", () => {
+    const all = upgraderAccessChecks(TARGET).flatMap((c) => c.permissions);
+    for (const p of all) {
+      expect(p.startsWith("compute.")).toBe(false);
+      expect(p.endsWith(".setIamPolicy")).toBe(false);
+    }
+    // Sanity: the forbidden list still bans compute (the Direct VPC decision).
+    expect(UPGRADER_FORBIDDEN_ROLES).toContain("roles/compute.");
+  });
+
+  it("covers every step the run takes after its decision", () => {
+    const all = new Set(upgraderAccessChecks(TARGET).flatMap((c) => c.permissions));
+    for (const p of [
+      "run.services.update", // the roll + traffic
+      "run.jobs.create", // the migration job
+      "run.jobs.run",
+      "run.executions.get", // execute --wait, the lock lease
+      "run.jobs.update", // the self-update
+      "cloudsql.backupRuns.create", // the backup
+      "iam.serviceAccounts.actAs",
+      "iam.serviceAccounts.signJwt", // the IAP health check
+      "storage.objects.list", // gcloud storage cp lists first (#799)
+      "storage.objects.create",
+      "storage.objects.delete", // overwrite + lock release
+      "artifactregistry.repositories.downloadArtifacts",
+    ]) {
+      expect(all.has(p)).toBe(true);
+    }
+  });
+});
+
+describe("runIamPreflight", () => {
+  it("asks once per resource, with every permission the checks on it need", async () => {
+    const asked: string[] = [];
+    await runIamPreflight(upgraderAccessChecks(TARGET), tester(setupGrants(), asked));
+    const names = asked.map((a) => a.split(" ")[0]);
+    expect(new Set(names).size).toBe(names.length);
+    const project = asked.find((a) => a.startsWith("projects/acme "));
+    expect(project).toContain("run.services.update");
+    expect(project).toContain("cloudsql.backupRuns.create");
+  });
+
+  it("names every missing permission, per need, in one outcome", async () => {
+    const grants = setupGrants();
+    grants.get("projects/acme")?.delete("run.jobs.run");
+    grants.get("projects/acme")?.delete("cloudsql.backupRuns.create");
+    grants.get("gs://acme-caelo-ops")?.delete("storage.objects.list");
+    const r = await runIamPreflight(upgraderAccessChecks(TARGET), tester(grants));
+    expect(r.kind).toBe("missing");
+    if (r.kind !== "missing") return;
+    expect(r.missing.flatMap((m) => m.missing).sort()).toEqual([
+      "cloudsql.backupRuns.create",
+      "run.jobs.run",
+      "storage.objects.list",
+    ]);
+    const reason = preflightMissingReason(r.missing);
+    expect(reason).toContain("run the database migrations");
+    expect(reason).toContain("missing storage.objects.list on gs://acme-caelo-ops");
+    expect(reason).toContain("Run the upgrade once with Cloud Shell");
+  });
+
+  it("probes objects.get only once objects.list is confirmed; otherwise names it unchecked, not missing", async () => {
+    const withList: string[] = [];
+    await runIamPreflight(upgraderAccessChecks(TARGET), tester(setupGrants(), withList));
+    expect(withList.find((a) => a.startsWith("gs://acme-caelo-ops/install/"))).toContain(
+      "storage.objects.get",
+    );
+
+    const grants = setupGrants();
+    grants.get("gs://acme-caelo-ops")?.delete("storage.objects.list");
+    const asked: string[] = [];
+    const r = await runIamPreflight(upgraderAccessChecks(TARGET), tester(grants, asked));
+    const objects = asked.find((a) => a.startsWith("gs://acme-caelo-ops/install/"));
+    expect(objects).toBeDefined();
+    expect(objects).not.toContain("storage.objects.get");
+    expect(r.kind).toBe("missing");
+    if (r.kind !== "missing") return;
+    expect(r.missing.flatMap((m) => m.missing)).not.toContain("storage.objects.get");
+    expect(r.missing.flatMap((m) => m.unchecked ?? [])).toEqual(["storage.objects.get"]);
+    expect(preflightMissingReason(r.missing)).toContain(
+      "storage.objects.get on gs://acme-caelo-ops/install/ could not be checked",
+    );
+  });
+
+  it("an unanswerable resource is an error, never 'nothing missing'", async () => {
+    const failing: IamTester = {
+      granted: async (resource) =>
+        resource.kind === "artifact-repository"
+          ? { ok: false, error: "HTTP 403 caller has no access" }
+          : { ok: true, granted: [] },
+    };
+    const r = await runIamPreflight(upgraderAccessChecks(TARGET), failing);
+    expect(r.kind).toBe("error");
+    if (r.kind === "error") expect(r.errors[0]).toContain("HTTP 403");
+  });
+
+  it("a tester that throws is an error too", async () => {
+    const throwing: IamTester = {
+      granted: async () => {
+        throw new Error("socket hang up");
+      },
+    };
+    const r = await runIamPreflight(
+      [{ need: "x", resource: { kind: "project", projectId: "p" }, permissions: ["a.b.c"] }],
+      throwing,
+    );
+    expect(r).toEqual({ kind: "error", errors: ["projects/p: socket hang up"] });
+  });
+});
+
+type Call = { url: string; method: string; headers: Record<string, string>; body?: string };
+
+/** An HTTP client answering by method + URL prefix. */
+function http(calls: Call[], answer: (c: Call) => { status: number; body: string }): HttpFetch {
+  return async (url, init) => {
+    const call = { url, ...init };
+    calls.push(call);
+    const a = answer(call);
+    return { ok: a.status >= 200 && a.status < 300, status: a.status, text: async () => a.body };
+  };
+}
+
+describe("restIamTester", () => {
+  it("calls each API's testIamPermissions with the job's token", async () => {
+    const calls: Call[] = [];
+    const t = restIamTester({
+      http: http(calls, () => ({ status: 200, body: JSON.stringify({ permissions: ["a.b.c"] }) })),
+      token: async () => "tok",
+    });
+    const resources: IamResource[] = [
+      { kind: "project", projectId: "acme" },
+      { kind: "service-account", projectId: "acme", email: "x@acme.iam.gserviceaccount.com" },
+      {
+        kind: "run-job",
+        projectId: "acme",
+        region: "europe-west1",
+        name: "caelo-production-upgrader",
+      },
+      { kind: "bucket", bucket: "acme-caelo-ops" },
+      {
+        kind: "artifact-repository",
+        projectId: "caelo-website",
+        location: "europe-west1",
+        repository: "caelo-cms-images",
+      },
+    ];
+    for (const res of resources) {
+      expect(await t.granted(res, ["a.b.c", "d.e.f"])).toEqual({ ok: true, granted: ["a.b.c"] });
+    }
+    expect(calls.map((c) => `${c.method} ${c.url}`)).toEqual([
+      "POST https://cloudresourcemanager.googleapis.com/v1/projects/acme:testIamPermissions",
+      "POST https://iam.googleapis.com/v1/projects/acme/serviceAccounts/x%40acme.iam.gserviceaccount.com:testIamPermissions",
+      "POST https://run.googleapis.com/v2/projects/acme/locations/europe-west1/jobs/caelo-production-upgrader:testIamPermissions",
+      "GET https://storage.googleapis.com/storage/v1/b/acme-caelo-ops/iam/testPermissions?permissions=a.b.c&permissions=d.e.f",
+      "POST https://artifactregistry.googleapis.com/v1/projects/caelo-website/locations/europe-west1/repositories/caelo-cms-images:testIamPermissions",
+    ]);
+    for (const c of calls) expect(c.headers.Authorization).toBe("Bearer tok");
+    expect(JSON.parse(calls[0]?.body ?? "{}")).toEqual({ permissions: ["a.b.c", "d.e.f"] });
+    expect(calls[3]?.body).toBeUndefined();
+  });
+
+  it("an answer without `permissions` means none is held", async () => {
+    const t = restIamTester({
+      http: http([], () => ({ status: 200, body: "{}" })),
+      token: async () => "tok",
+    });
+    expect(await t.granted({ kind: "project", projectId: "acme" }, ["a.b.c"])).toEqual({
+      ok: true,
+      granted: [],
+    });
+  });
+
+  it("an HTTP error or non-JSON answer is an error, not 'missing'", async () => {
+    const denied = restIamTester({
+      http: http([], () => ({ status: 403, body: "PERMISSION_DENIED" })),
+      token: async () => "tok",
+    });
+    const r = await denied.granted({ kind: "project", projectId: "acme" }, ["a.b.c"]);
+    expect(r).toEqual({ ok: false, error: "HTTP 403 PERMISSION_DENIED" });
+    const garbled = restIamTester({
+      http: http([], () => ({ status: 200, body: "<html>" })),
+      token: async () => "tok",
+    });
+    expect((await garbled.granted({ kind: "project", projectId: "acme" }, ["a"])).ok).toBe(false);
+  });
+
+  it("an empty token is an error before any call", async () => {
+    const calls: Call[] = [];
+    const t = restIamTester({
+      http: http(calls, () => ({ status: 200, body: "{}" })),
+      token: async () => "",
+    });
+    expect((await t.granted({ kind: "project", projectId: "acme" }, ["a"])).ok).toBe(false);
+    expect(calls).toEqual([]);
+  });
+});
+
+describe("install/ object probes", () => {
+  const objects: Extract<IamResource, { kind: "bucket-objects" }> = {
+    kind: "bucket-objects",
+    bucket: "acme-caelo-ops",
+    prefix: "install",
+  };
+  const PERMS = ["storage.objects.get", "storage.objects.create", "storage.objects.delete"];
+
+  it("every write probe is preconditioned on generation 1, so it can never change anything", () => {
+    const create = objectProbeCall(objects, "storage.objects.create");
+    const del = objectProbeCall(objects, "storage.objects.delete");
+    expect(create?.url).toContain("ifGenerationMatch=1");
+    expect(del?.url).toContain("ifGenerationMatch=1");
+    for (const c of [create, del, objectProbeCall(objects, "storage.objects.get")]) {
+      expect(c?.url).toContain(encodeURIComponent(`install/${PREFLIGHT_PROBE_OBJECT}`));
+    }
+    expect(objectProbeCall(objects, "storage.objects.list")).toBeNull();
+  });
+
+  it("404/412 is held, 403 is missing", async () => {
+    const calls: Call[] = [];
+    const t = restIamTester({
+      http: http(calls, (c) =>
+        c.method === "GET"
+          ? { status: 404, body: "No such object" }
+          : c.method === "POST"
+            ? { status: 412, body: "conditionNotMet" }
+            : { status: 403, body: "does not have storage.objects.delete access" },
+      ),
+      token: async () => "tok",
+    });
+    expect(await t.granted(objects, PERMS)).toEqual({
+      ok: true,
+      granted: ["storage.objects.get", "storage.objects.create"],
+    });
+    expect(calls.map((c) => c.method)).toEqual(["GET", "POST", "DELETE"]);
+  });
+
+  it("an upload that succeeded (never expected) is an error, loudly", async () => {
+    const t = restIamTester({
+      http: http([], (c) =>
+        c.method === "POST" ? { status: 200, body: "{}" } : { status: 404, body: "" },
+      ),
+      token: async () => "tok",
+    });
+    const r = await t.granted(objects, PERMS);
+    expect(r.ok).toBe(false);
+  });
+
+  it("a permission without a probe is an error", async () => {
+    const t = restIamTester({
+      http: http([], () => ({ status: 404, body: "" })),
+      token: async () => "tok",
+    });
+    expect((await t.granted(objects, ["storage.objects.update"])).ok).toBe(false);
+  });
+});
+
+describe("testIamCall", () => {
+  it("URL-encodes path parts", () => {
+    const c = testIamCall({ kind: "bucket", bucket: "a b" }, ["x"]);
+    expect(c.url).toContain("/b/a%20b/");
+  });
+});