@intentius/chant-lexicon-terraform 0.124.52 → 0.124.54

package/src/state/exec.ts

@@ -0,0 +1,111 @@
+/**
+ * The binary a state is read and written through: `state pull` and
+ * `state push`. The exec is injectable, so a test runs none.
+ *
+ * Lifted from terragucci's `migrate.ts` (c493e0e): `Ran`, `BinaryIo`,
+ * `BinaryExec`, `runBinary`, `StateFile`, `StateResource` and `pullState`.
+ */
+import { spawn } from "node:child_process";
+
+export interface Ran {
+  code: number;
+  stdout: string;
+  /** stdout and stderr together, as they came. */
+  out: string;
+}
+
+/** What a run of the binary hands back while it runs: `stderr` sees each line of stderr as it arrives, and a line it takes (true) is left out of `out`. */
+export interface BinaryIo {
+  stderr?: (line: string) => boolean;
+}
+
+/** How the binary runs: in `dir`, with `env`. Injectable for tests. */
+export type BinaryExec = (binary: string, args: string[], dir: string, env: NodeJS.ProcessEnv, io?: BinaryIo) => Promise<Ran>;
+
+/** The binary run with `-chdir=<dir>` and `TF_IN_AUTOMATION=1`. */
+export const runBinary: BinaryExec = (binary, args, dir, env, io) =>
+  new Promise((done) => {
+    const child = spawn(binary, [`-chdir=${dir}`, ...args], { env: { ...env, TF_IN_AUTOMATION: "1" }, stdio: ["ignore", "pipe", "pipe"] });
+    let stdout = "";
+    let out = "";
+    let partial = "";
+    const stderrLine = (line: string): void => {
+      let taken = false;
+      try {
+        taken = io!.stderr!(line);
+      } catch {
+        taken = false;
+      }
+      if (!taken) out += `${line}\n`;
+    };
+    child.stdout.on("data", (d) => {
+      stdout += d;
+      out += d;
+    });
+    // A line split across two chunks, a character too, is put back together before it is read.
+    if (io?.stderr) child.stderr.setEncoding("utf8");
+    child.stderr.on("data", (d) => {
+      if (!io?.stderr) {
+        out += d;
+        return;
+      }
+      partial += d;
+      const lines = partial.split("\n");
+      partial = lines.pop()!;
+      for (const l of lines) stderrLine(l);
+    });
+    const rest = (): void => {
+      if (io?.stderr && partial) stderrLine(partial);
+      partial = "";
+    };
+    child.on("error", (e) => {
+      rest();
+      done({ code: 127, stdout, out: `${out}${e.message}` });
+    });
+    child.on("close", (code) => {
+      rest();
+      done({ code: code ?? 1, stdout, out });
+    });
+  });
+
+/** A state file as `state pull` prints it (format version 4); only what a run touches is named. */
+export interface StateFile {
+  version: number;
+  terraform_version?: string;
+  serial: number;
+  lineage: string;
+  outputs?: Record<string, unknown>;
+  resources: StateResource[];
+  check_results?: unknown;
+  [k: string]: unknown;
+}
+
+export interface StateResource {
+  module?: string;
+  mode: string;
+  type: string;
+  name: string;
+  provider?: string;
+  instances?: unknown[];
+  [k: string]: unknown;
+}
+
+/** The last lines of the binary's output, on one line, for an error. */
+export const lastLines = (s: string, n = 4): string => s.trim().split("\n").slice(-n).join(" ").slice(0, 400);
+
+/** A root's state as `state pull` gives it, or null when it has none. */
+export async function pullState(exec: BinaryExec, binary: string, root: string, dir: string, env: NodeJS.ProcessEnv): Promise<StateFile | null> {
+  const pulled = await exec(binary, ["state", "pull"], dir, env);
+  if (pulled.code !== 0) throw new Error(`state pull in ${root} failed: ${lastLines(pulled.out)}`);
+  if (!pulled.stdout.trim()) return null;
+  let s: StateFile;
+  try {
+    s = JSON.parse(pulled.stdout) as StateFile;
+  } catch {
+    throw new Error(`state pull in ${root} printed something that is not a state`);
+  }
+  if (!Array.isArray(s.resources)) s.resources = [];
+  // A backend with no state object yet prints an empty state with no lineage: that is no state.
+  if (!s.lineage && !s.serial && s.resources.length === 0) return null;
+  return s;
+}

package/src/state/gcs.ts

@@ -0,0 +1,254 @@
+/**
+ * Google Cloud Storage for a gcs backend's state, over its JSON API with
+ * `fetch` and `node:crypto`: an object's text and generation, a write, a
+ * write only when there is no object (the lock), and a delete.
+ *
+ * The identity, first match wins: a bearer token (`access_token`, or
+ * GOOGLE_OAUTH_ACCESS_TOKEN), else a credentials file or its JSON:
+ *
+ *   external_account  the job's OIDC token, traded at Google's STS, then the
+ *                     service account it impersonates.
+ *   service_account   a key: requests carry a self-signed JWT.
+ *
+ * With `impersonate_service_account`, every request carries that service
+ * account's token, which the identity asks IAM for.
+ *
+ * Lifted from terragucci's `report/gcs.ts` (c493e0e), without signed URLs,
+ * version listing and the re-plan's traded tokens.
+ */
+import { createSign } from "node:crypto";
+import { readFileSync } from "node:fs";
+import { defaultFetch, encodeStrict, StoreConflict, StoreError, type StoreFetch } from "./object-store";
+
+export const GCS_ENDPOINT = "https://storage.googleapis.com";
+/** IAM Credentials, where a token for an impersonated service account comes from. */
+export const IAM_CREDENTIALS = "https://iamcredentials.googleapis.com";
+const CLOUD_PLATFORM = "https://www.googleapis.com/auth/cloud-platform";
+const READ_WRITE = "https://www.googleapis.com/auth/devstorage.read_write";
+
+export interface GcsTarget {
+  bucket: string;
+  /** The JSON API's address, no trailing slash. Default https://storage.googleapis.com. */
+  endpoint: string;
+  /** The credentials file, read at the first request. */
+  credentialsFile?: string;
+  /** The credentials as JSON: a gcs backend's `credentials` given inline. */
+  credentialsJson?: string;
+  /** A bearer token used as it is. */
+  accessToken?: string;
+  /** The service account every request acts as, through IAM's generateAccessToken. */
+  impersonate?: string;
+  /** IAM Credentials' address, no trailing slash. */
+  iamEndpoint?: string;
+}
+
+interface ExternalAccount {
+  type: "external_account";
+  audience: string;
+  subject_token_type: string;
+  token_url: string;
+  service_account_impersonation_url?: string;
+  credential_source: { file?: string; url?: string; headers?: Record<string, string>; format?: { type?: string; subject_token_field_name?: string } };
+}
+
+interface ServiceAccountKey {
+  type: "service_account";
+  client_email: string;
+  private_key: string;
+  private_key_id?: string;
+}
+
+type Credentials = ExternalAccount | ServiceAccountKey;
+
+const b64url = (b: string | Buffer): string => Buffer.from(b).toString("base64url");
+
+/** A JWT the key signs for itself, which Google's APIs take as an access token for the scope it names. */
+export function selfSignedJwt(key: ServiceAccountKey, now: Date): string {
+  const iat = Math.floor(now.getTime() / 1000);
+  const head = b64url(JSON.stringify({ alg: "RS256", typ: "JWT", ...(key.private_key_id ? { kid: key.private_key_id } : {}) }));
+  const body = b64url(JSON.stringify({ iss: key.client_email, sub: key.client_email, scope: READ_WRITE, iat, exp: iat + 3600 }));
+  return `${head}.${body}.${createSign("RSA-SHA256").update(`${head}.${body}`).sign(key.private_key).toString("base64url")}`;
+}
+
+/** Google's error message, or the start of the body. */
+const why = (text: string): string => {
+  try {
+    const j = JSON.parse(text) as { error?: string | { message?: string }; error_description?: string };
+    if (typeof j.error === "object" && j.error?.message) return j.error.message;
+    if (typeof j.error === "string") return [j.error, j.error_description].filter(Boolean).join(": ");
+  } catch {
+    // Not JSON.
+  }
+  return text.slice(0, 300);
+};
+
+type Token = { token: string; expires: number };
+
+export class GcsClient {
+  private creds?: Credentials;
+  private token?: Promise<Token>;
+
+  constructor(readonly target: GcsTarget, private readonly fetchFn: StoreFetch = defaultFetch, private readonly readFile: (file: string) => string = (f) => readFileSync(f, "utf-8")) {}
+
+  private where(key: string): string {
+    return `gs://${this.target.bucket}/${key}`;
+  }
+
+  private credentials(): Credentials {
+    if (this.creds) return this.creds;
+    const source = this.target.credentialsFile ?? "the backend's credentials";
+    let c: Partial<Credentials>;
+    try {
+      c = JSON.parse(this.target.credentialsJson ?? this.readFile(this.target.credentialsFile ?? "")) as Partial<Credentials>;
+    } catch (e) {
+      throw new StoreError(`cannot read the credentials in ${source}: ${(e as Error).message}`);
+    }
+    if (c.type === "external_account" && c.token_url && c.audience && c.credential_source && (c.credential_source.file || c.credential_source.url)) return (this.creds = c as ExternalAccount);
+    if (c.type === "service_account" && c.client_email && c.private_key) return (this.creds = c as ServiceAccountKey);
+    throw new StoreError(`${source} is not an external_account file with a credential_source file or url, nor a service_account key`);
+  }
+
+  private async post(url: string, headers: Record<string, string>, body: string, what: string): Promise<Record<string, unknown>> {
+    const res = await this.fetchFn(url, { method: "POST", headers, body });
+    const text = await res.text();
+    if (!res.ok) throw new StoreError(`${what}: ${res.status} ${why(text)}`);
+    try {
+      return JSON.parse(text) as Record<string, unknown>;
+    } catch {
+      throw new StoreError(`${what}: the answer is not JSON`);
+    }
+  }
+
+  /** The job's OIDC token, from the file or URL the credentials name. */
+  private async subjectToken(c: ExternalAccount): Promise<string> {
+    const src = c.credential_source;
+    let text: string;
+    if (src.file) {
+      try {
+        text = this.readFile(src.file);
+      } catch (e) {
+        throw new StoreError(`cannot read the OIDC token in ${src.file}: ${(e as Error).message}`);
+      }
+    } else {
+      const res = await this.fetchFn(src.url!, { method: "GET", headers: src.headers ?? {} });
+      text = await res.text();
+      if (!res.ok) throw new StoreError(`the OIDC token at ${src.url!.split("?")[0]}: ${res.status} ${why(text)}`);
+    }
+    const field = src.format?.type === "json" ? src.format.subject_token_field_name : undefined;
+    let token = text.trim();
+    if (field) {
+      try {
+        token = String((JSON.parse(text) as Record<string, unknown>)[field] ?? "");
+      } catch {
+        throw new StoreError(`the job's OIDC token is not JSON with ${field}`);
+      }
+    }
+    if (!token) throw new StoreError("the job's OIDC token is empty");
+    return token;
+  }
+
+  private async impersonateWith(url: string, bearer: string, sa: string): Promise<Token> {
+    const r = await this.post(url, { "content-type": "application/json", authorization: `Bearer ${bearer}` }, JSON.stringify({ scope: [CLOUD_PLATFORM], lifetime: "3600s" }), `impersonating ${sa}`);
+    if (typeof r.accessToken !== "string") throw new StoreError(`IAM answered the impersonation of ${sa} without a token`);
+    const at = typeof r.expireTime === "string" ? Date.parse(r.expireTime) : NaN;
+    return { token: r.accessToken, expires: Number.isNaN(at) ? Date.now() + 3600_000 : at };
+  }
+
+  /** STS's federated token, then the impersonated service account's, when the credentials name one. */
+  private async exchange(c: ExternalAccount): Promise<Token> {
+    const sts = await this.post(
+      c.token_url,
+      { "content-type": "application/x-www-form-urlencoded" },
+      new URLSearchParams({
+        grant_type: "urn:ietf:params:oauth:grant-type:token-exchange",
+        audience: c.audience,
+        scope: CLOUD_PLATFORM,
+        requested_token_type: "urn:ietf:params:oauth:token-type:access_token",
+        subject_token: await this.subjectToken(c),
+        subject_token_type: c.subject_token_type,
+      }).toString(),
+      `Google STS refused the OIDC token for ${c.audience}`,
+    );
+    if (typeof sts.access_token !== "string") throw new StoreError("Google STS answered without a token");
+    const fed = { token: sts.access_token, expires: Date.now() + Number(sts.expires_in ?? 3600) * 1000 };
+    if (!c.service_account_impersonation_url) return fed;
+    return this.impersonateWith(c.service_account_impersonation_url, fed.token, "the service account");
+  }
+
+  /** The identity's own token: the one the target gives, a key's self-signed JWT, or STS's for the OIDC token. */
+  private async own(): Promise<Token> {
+    if (this.target.accessToken) return { token: this.target.accessToken, expires: Date.now() + 3600_000 };
+    const c = this.credentials();
+    return c.type === "service_account" ? { token: selfSignedJwt(c, new Date()), expires: Date.now() + 3600_000 } : this.exchange(c);
+  }
+
+  /** An access token: once per client, again a minute before it expires. */
+  private async bearer(): Promise<string> {
+    const sa = this.target.impersonate;
+    const fresh = async (): Promise<Token> => {
+      if (!sa) return this.own();
+      const base = await this.own();
+      return this.impersonateWith(`${(this.target.iamEndpoint ?? IAM_CREDENTIALS).replace(/\/+$/, "")}/v1/projects/-/serviceAccounts/${encodeURIComponent(sa)}:generateAccessToken`, base.token, sa);
+    };
+    const again = (): Promise<Token> =>
+      (this.token = fresh().catch((e: unknown) => {
+        this.token = undefined;
+        throw e;
+      }));
+    let held = await (this.token ?? again());
+    if (held.expires - Date.now() < 60_000) held = await again();
+    return held.token;
+  }
+
+  private objectUrl(key: string): string {
+    return `${this.target.endpoint}/storage/v1/b/${encodeURIComponent(this.target.bucket)}/o/${encodeStrict(key)}`;
+  }
+
+  /** Write an object; with `ifGenerationMatch`, only when the object still has that generation (`0`: there is none), else StoreConflict. */
+  async put(key: string, body: string, contentType: string, ifGenerationMatch?: string): Promise<void> {
+    const t = this.target;
+    const url = `${t.endpoint}/upload/storage/v1/b/${encodeURIComponent(t.bucket)}/o?uploadType=media&name=${encodeStrict(key)}${ifGenerationMatch !== undefined ? `&ifGenerationMatch=${encodeStrict(ifGenerationMatch)}` : ""}`;
+    const res = await this.fetchFn(url, { method: "POST", headers: { "content-type": contentType, authorization: `Bearer ${await this.bearer()}` }, body });
+    if (res.status === 412) throw new StoreConflict(`PUT ${this.where(key)}: 412, it changed since it was read`);
+    if (!res.ok) throw new StoreError(`PUT ${this.where(key)}: ${res.status} ${why(await res.text())}`);
+  }
+
+  async get(key: string): Promise<string | undefined> {
+    const res = await this.fetchFn(`${this.objectUrl(key)}?alt=media`, { method: "GET", headers: { authorization: `Bearer ${await this.bearer()}` } });
+    if (res.status === 404) return undefined;
+    if (!res.ok) throw new StoreError(`GET ${this.where(key)}: ${res.status} ${why(await res.text())}`);
+    return res.text();
+  }
+
+  /** An object's metadata and never its body: whether it is there, and its generation. */
+  async head(key: string): Promise<{ exists: boolean; generation?: string }> {
+    const res = await this.fetchFn(this.objectUrl(key), { method: "GET", headers: { authorization: `Bearer ${await this.bearer()}` } });
+    const text = await res.text();
+    if (res.status === 404) return { exists: false };
+    if (!res.ok) throw new StoreError(`GET ${this.where(key)} metadata: ${res.status} ${why(text)}`);
+    let g: unknown;
+    try {
+      g = (JSON.parse(text) as { generation?: unknown }).generation;
+    } catch {
+      // Metadata that is not JSON: no generation.
+    }
+    return { exists: true, ...(g !== undefined && g !== null && g !== "" ? { generation: String(g) } : {}) };
+  }
+
+  /** Write an object only when there is none (ifGenerationMatch=0): true when this write made it, false when one was there. */
+  async putIfAbsent(key: string, body: string, contentType: string): Promise<boolean> {
+    try {
+      await this.put(key, body, contentType, "0");
+      return true;
+    } catch (e) {
+      if (e instanceof StoreConflict) return false;
+      throw e;
+    }
+  }
+
+  /** Delete an object's live generation. One that is not there is deleted already. */
+  async remove(key: string): Promise<void> {
+    const res = await this.fetchFn(this.objectUrl(key), { method: "DELETE", headers: { authorization: `Bearer ${await this.bearer()}` } });
+    if (!res.ok && res.status !== 404) throw new StoreError(`DELETE ${this.where(key)}: ${res.status} ${why(await res.text())}`);
+  }
+}

package/src/state/index.ts

@@ -0,0 +1,15 @@
+/**
+ * A root's state for a composed run (#4437): where it is, its lock, its pull
+ * and its write, and the unwritten keep. Plain lexicon code, lifted from
+ * terragucci at c493e0e.
+ */
+
+export * from "./object-store";
+export * from "./backend";
+export * from "./store";
+export * from "./exec";
+export * from "./locks";
+export * from "./root-states";
+export { S3Client, s3FromEnv, type S3Target } from "./s3";
+export { GcsClient, type GcsTarget } from "./gcs";
+export { AzureBlobClient, type AzureTarget } from "./azure-blob";

package/src/state/locks.test.ts

@@ -0,0 +1,84 @@
+// The source-workspace lock a state migration takes (terragucci#1274): a workspace a state is read from is locked
+// through its own API before any state lock, and released when a later lock cannot be taken.
+import { describe, expect, it } from "vitest";
+import { SourceLockedError, StateLockedError, takeLocksWithSources, takeSourceLocks, type LockStore, type SourceLock } from "./locks";
+
+const T = "2026-10-25T00:00:00Z";
+const op = { operation: "terragucci migrate", info: "a state migration is writing this state" };
+
+function store(location: string, held?: string): LockStore & { taken: string[]; released: number } {
+  const s = {
+    location,
+    lockLocation: `${location}.tflock`,
+    taken: [] as string[],
+    released: 0,
+    heldLock: async () => (held ? { id: "other", info: held } : undefined),
+    lock: async (info: string) => {
+      if (held) return false;
+      s.taken.push(info);
+      return true;
+    },
+    unlock: async () => {
+      s.released++;
+    },
+    adopt: () => {},
+  };
+  return s;
+}
+
+function source(root: string, location: string, refuse?: string): SourceLock & { reasons: string[]; released: number } {
+  const s = {
+    root,
+    location,
+    reasons: [] as string[],
+    released: 0,
+    lock: async (reason: string) => {
+      if (refuse) throw new Error(refuse);
+      s.reasons.push(reason);
+      return async () => {
+        s.released++;
+      };
+    },
+  };
+  return s;
+}
+
+describe("source-workspace locks", () => {
+  it("takes the source workspaces first, naming what reads them, then the states", async () => {
+    const src = source("envs/dev", "app.terraform.io/acme/dev");
+    const st = store("s3://b/envs/dev.tfstate");
+    const got = await takeLocksWithSources({ sources: [src], stores: [{ root: "envs/dev", store: st }], now: T, op, sourceOp: { operation: "terragucci migrate", info: "a state migration is reading this state" } });
+    expect(src.reasons).toEqual(["terragucci migrate: a state migration is reading this state"]);
+    expect(st.taken).toHaveLength(1);
+    expect(JSON.parse(st.taken[0]!)).toMatchObject({ Operation: "terragucci migrate", Created: T, Path: "b/envs/dev.tfstate" });
+    expect(got.sources.map((s) => s.root)).toEqual(["envs/dev"]);
+    expect(got.locks.map((l) => l.root)).toEqual(["envs/dev"]);
+    await got.sources[0]!.release();
+    expect(src.released).toBe(1);
+  });
+
+  it("names the operation's own info on a source lock without sourceOp", async () => {
+    const src = source("a", "ws");
+    await takeSourceLocks([src], op);
+    expect(src.reasons).toEqual(["terragucci migrate: a state migration is writing this state"]);
+  });
+
+  it("releases the source locks when a state is locked by another run", async () => {
+    const src = source("envs/dev", "ws");
+    const st = store("s3://b/k", JSON.stringify({ ID: "x", Operation: "OperationTypeApply", Who: "someone" }));
+    await expect(takeLocksWithSources({ sources: [src], stores: [{ root: "envs/dev", store: st }], now: T, op })).rejects.toBeInstanceOf(StateLockedError);
+    expect(src.released).toBe(1);
+  });
+
+  it("releases what it took and takes no state lock when a source workspace is locked", async () => {
+    const first = source("a", "ws-a");
+    const second = source("b", "ws-b", "its workspace ws-b is locked");
+    const st = store("s3://b/k");
+    const err = await takeLocksWithSources({ sources: [first, second], stores: [{ root: "a", store: st }], now: T, op }).catch((e: unknown) => e);
+    expect(err).toBeInstanceOf(SourceLockedError);
+    expect((err as SourceLockedError).root).toBe("b");
+    expect((err as Error).message).toContain("ws-b is locked");
+    expect(first.released).toBe(1);
+    expect(st.taken).toEqual([]);
+  });
+});

package/src/state/locks.ts

@@ -0,0 +1,201 @@
+/**
+ * Taking a set of state locks in one order, each the way its backend takes
+ * it, with lock info that names the operation and the run. A lock another
+ * run holds stops the take: every lock taken so far is released, and the
+ * error names the holder from the lock info it left.
+ *
+ * A run can also hold the lock of a workspace a state is read from, which
+ * no backend block of the root names (a Terraform Cloud or Enterprise
+ * workspace a state migration moves from, terragucci#1274): the workspace's
+ * own lock, taken through its API before any state lock (`takeSourceLocks`,
+ * `takeLocksWithSources`).
+ *
+ * Lifted from terragucci's `migrate.ts` (c493e0e): `lockInfo`,
+ * `LockOperation`, `HeldLock` and `takeLocks`, and the source-workspace lock
+ * of its `takeLocks` (lines 1045 to 1056).
+ */
+import { randomUUID } from "node:crypto";
+import { hostname } from "node:os";
+import type { LockHolder, StateStore } from "./store";
+
+/** What a lock says holds it: the operation and a line of info, as the binary prints them when it finds the lock held. */
+export interface LockOperation {
+  operation: string;
+  info: string;
+}
+
+/** The lock info a run writes, in the shape the binary writes and prints when it finds a lock held. */
+export function lockInfo(path: string, now: string, op: LockOperation, who = `chant@${hostname()}`): string {
+  return JSON.stringify({ ID: randomUUID(), Operation: op.operation, Info: op.info, Who: who, Version: "", Created: now, Path: path });
+}
+
+/** Who holds a lock, from the lock info it left: what is there of it. */
+export interface LockHolderInfo {
+  id: string;
+  who?: string;
+  operation?: string;
+  info?: string;
+  created?: string;
+}
+
+/** The holder a lock info names. */
+export function holderOf(held: LockHolder): LockHolderInfo {
+  let v: Record<string, unknown> = {};
+  try {
+    v = JSON.parse(held.info) as Record<string, unknown>;
+  } catch {
+    // Lock info that is not JSON names no one.
+  }
+  const s = (k: string): string | undefined => (typeof v[k] === "string" && v[k] !== "" ? (v[k] as string) : undefined);
+  return { id: held.id, ...(s("Who") ? { who: s("Who") } : {}), ...(s("Operation") ? { operation: s("Operation") } : {}), ...(s("Info") ? { info: s("Info") } : {}), ...(s("Created") ? { created: s("Created") } : {}) };
+}
+
+/** A lock held by another run stopped the take; nothing was pulled. */
+export class StateLockedError extends Error {
+  constructor(
+    readonly root: string,
+    readonly lockLocation: string,
+    readonly holder?: LockHolderInfo,
+  ) {
+    const by = holder ? [holder.who && `by ${holder.who}`, holder.operation, holder.info && `(${holder.info})`, holder.created && `since ${holder.created}`].filter(Boolean).join(" ") : "";
+    super(`${root}: its state is locked at ${lockLocation}${by ? ` ${by}` : ""}, so nothing was pulled`);
+    this.name = "StateLockedError";
+  }
+}
+
+/** What taking a lock needs of a store: a caller with a store of its own (terragucci's, which also reads versions) passes that. */
+export type LockStore = Pick<StateStore, "location" | "lockLocation" | "heldLock" | "lock" | "unlock" | "adopt">;
+
+/** A lock taken, on one root's state. */
+export interface HeldLock<S extends LockStore = StateStore> {
+  root: string;
+  store: S;
+  /** The lock was another run's, kept on purpose with a state it could not write, and this take took it over. */
+  adopted: boolean;
+  release: () => Promise<void>;
+}
+
+/**
+ * Take the lock of each state, in the order given. Releases what it took and
+ * throws StateLockedError when one is held and `adopt` does not take it over.
+ * `adopt` sees a held lock and says whether this take takes it over (a lock
+ * an earlier run of the same operation left held on purpose).
+ */
+export async function takeLocks<S extends LockStore = StateStore>(
+  stores: readonly { root: string; store: S }[],
+  now: string,
+  op: LockOperation,
+  adopt?: (root: string, store: S, held: LockHolder) => Promise<boolean>,
+  who?: string,
+): Promise<HeldLock<S>[]> {
+  const held: HeldLock<S>[] = [];
+  const releaseAll = async (): Promise<void> => {
+    for (const h of held.reverse()) await h.release().catch(() => {});
+  };
+  for (const { root, store } of stores) {
+    let got: boolean;
+    try {
+      got = await store.lock(lockInfo(store.location.replace(/^[a-z0-9]+:\/\//, ""), now, op, who));
+    } catch (e) {
+      await releaseAll();
+      throw new Error(`${root}: the lock ${store.lockLocation} could not be taken: ${(e as Error).message}`);
+    }
+    if (got) {
+      held.push({ root, store, adopted: false, release: () => store.unlock() });
+      continue;
+    }
+    const info = await store.heldLock().catch(() => undefined);
+    if (info && adopt && (await adopt(root, store, info).catch(() => false))) {
+      store.adopt(info);
+      held.push({ root, store, adopted: true, release: () => store.unlock() });
+      continue;
+    }
+    await releaseAll();
+    throw new StateLockedError(root, store.lockLocation, info ? holderOf(info) : undefined);
+  }
+  return held;
+}
+
+/**
+ * The lock of a workspace a state is read from, taken through the
+ * workspace's own API (Terraform Cloud's `actions/lock`), not a lock object
+ * beside a state. `lock` takes it with a reason the workspace shows, and
+ * returns its release; it throws when another run holds it.
+ */
+export interface SourceLock {
+  root: string;
+  /** The workspace, for messages. */
+  location: string;
+  lock(reason: string): Promise<() => Promise<void>>;
+}
+
+/** A source workspace's lock, taken. */
+export interface HeldSourceLock {
+  root: string;
+  location: string;
+  release: () => Promise<void>;
+}
+
+/** A source workspace's lock could not be taken: another run holds it, or the API refused. Nothing was read. */
+export class SourceLockedError extends Error {
+  constructor(
+    readonly root: string,
+    readonly location: string,
+    readonly reason: string,
+  ) {
+    super(`${root}: the lock of the workspace it is read from (${location}) could not be taken: ${reason}`);
+    this.name = "SourceLockedError";
+  }
+}
+
+/** The reason a source workspace's lock shows: the operation and its info, as a state lock's info names them. */
+export const sourceLockReason = (op: LockOperation): string => `${op.operation}: ${op.info}`;
+
+/**
+ * Take each source workspace's lock, in the order given, so no run writes a
+ * state while it is read. Releases what it took and throws SourceLockedError
+ * when one cannot be taken.
+ */
+export async function takeSourceLocks(sources: readonly SourceLock[], op: LockOperation): Promise<HeldSourceLock[]> {
+  const held: HeldSourceLock[] = [];
+  for (const s of sources) {
+    try {
+      held.push({ root: s.root, location: s.location, release: await s.lock(sourceLockReason(op)) });
+    } catch (e) {
+      for (const h of held.reverse()) await h.release().catch(() => {});
+      throw new SourceLockedError(s.root, s.location, (e as Error).message);
+    }
+  }
+  return held;
+}
+
+export interface LocksWithSourcesInput<S extends LockStore> {
+  /** The workspaces states are read from, locked first. */
+  sources: readonly SourceLock[];
+  /** The states written, locked after the sources, in the order given. */
+  stores: readonly { root: string; store: S }[];
+  now: string;
+  /** What the state locks name. */
+  op: LockOperation;
+  /** What a source workspace's lock names; default `op`. A run that only reads the source says so here. */
+  sourceOp?: LockOperation;
+  adopt?: (root: string, store: S, held: LockHolder) => Promise<boolean>;
+  who?: string;
+}
+
+/**
+ * Every lock a run that reads from source workspaces and writes states
+ * holds: the source workspaces' first, then the states'. When any cannot be
+ * taken, every lock taken so far is released, and the error is
+ * SourceLockedError or StateLockedError.
+ */
+export async function takeLocksWithSources<S extends LockStore = StateStore>(input: LocksWithSourcesInput<S>): Promise<{ sources: HeldSourceLock[]; locks: HeldLock<S>[] }> {
+  const sources = await takeSourceLocks(input.sources, input.sourceOp ?? input.op);
+  try {
+    const locks = await takeLocks(input.stores, input.now, input.op, input.adopt, input.who);
+    return { sources, locks };
+  } catch (e) {
+    for (const h of sources.reverse()) await h.release().catch(() => {});
+    throw e;
+  }
+}