@intentius/chant-lexicon-terraform 0.124.52 → 0.124.55

package/src/state/root-states.ts

@@ -0,0 +1,309 @@
+/**
+ * The states of a set of roots, held for one run: take every lock (in the
+ * order given), pull each state, write each state back with retries, and
+ * release each lock.
+ *
+ * A state whose write still fails after the retries is kept beside the state
+ * in its own backend as `<key>.chant-unwritten`, and its lock stays held,
+ * naming the run. The next take of that lock finds the kept state, takes
+ * the lock over, writes the kept state, removes it, and only then returns.
+ * When the backend cannot take the kept copy either, it goes under the job's
+ * report directory (`<reportDir>/unwritten/<root>.tfstate`), never the
+ * reports bucket.
+ *
+ * A take also reads a state terragucci kept as `<key>.terragucci-unwritten`
+ * under a terragucci composed run's lock, so a repo that moves to chant
+ * mid-incident keeps it. That name is read and removed, never written.
+ *
+ * Lifted from terragucci's `compose.ts` (c493e0e): `ComposedRun.plan`'s lock
+ * take and kept write, `writeState`, `writeWithRetries`, `keep`,
+ * `releaseOne` and `release`.
+ */
+import { mkdirSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { isRefused, stateObject, type LockableState, type StateObject, type StateRefusalKind } from "./backend";
+import { lastLines, pullState, runBinary, type BinaryExec, type StateFile } from "./exec";
+import { takeLocks, type HeldLock, type LockOperation } from "./locks";
+import type { StoreFetch } from "./object-store";
+import { stateStore, type StateStore } from "./store";
+
+/** The suffix of the object beside a root's state that holds a state a run could not write. */
+export const UNWRITTEN_SUFFIX = ".chant-unwritten";
+/** The suffix terragucci keeps an unwritten state under: read and removed, never written. */
+export const LEGACY_UNWRITTEN_SUFFIX = ".terragucci-unwritten";
+/** The operation terragucci's composed run names in its locks. */
+export const LEGACY_COMPOSE_OPERATION = "terragucci compose";
+/** Where a kept state goes under the job's report directory when its own backend could not take it. */
+export const UNWRITTEN_DIR = "unwritten";
+
+/** The operation a composed run's locks name. */
+export const COMPOSE_LOCK: LockOperation = { operation: "chant terraform.compose", info: "a composed run holds this state while it plans, applies and writes it" };
+
+/** How long to wait before each retry of a state's write: five retries, inside the lock timeout of 5 minutes. */
+export const DEFAULT_RETRY_DELAYS = [5_000, 15_000, 30_000, 60_000, 120_000];
+
+/** One root whose state the run holds. */
+export interface RootState {
+  root: string;
+  /** The root's directory, where the binary runs. */
+  dir: string;
+  /** The root's environment for the binary. */
+  env: NodeJS.ProcessEnv;
+  /** Where its state is; read from what `init` recorded, compared with the root's code, when absent. */
+  object?: StateObject;
+  /** `terraform.roots.<name>.backendConfig`, which the comparison with what init recorded reads (chant#4478). */
+  backendConfig?: Readonly<Record<string, string>>;
+  /** `terraform.roots.<name>.backendConfigFiles`, whose keys the same comparison reads (chant#4459). */
+  backendConfigFiles?: readonly string[];
+}
+
+export interface RootStatesOptions {
+  /** The roots, in the order their locks are taken. */
+  roots: RootState[];
+  /** The binary's path. */
+  binary: string;
+  exec?: BinaryExec;
+  /** The operation the locks name. Default COMPOSE_LOCK. */
+  op?: LockOperation;
+  /** Names the run in the locks it takes. */
+  runId?: string;
+  /** A directory of the job's own for the state files `state push` reads. */
+  work: string;
+  /** The job's report directory, where a kept state goes when its backend cannot take it. */
+  reportDir: string;
+  retryDelays?: number[];
+  sleep?: (ms: number) => Promise<void>;
+  fetch?: StoreFetch;
+  /** Opens a root's store; a test's fake stands in here. */
+  openStore?: (root: RootState, object: LockableState) => StateStore;
+  now?: () => string;
+  /** Who the lock info names. */
+  who?: string;
+  log?: (line: string) => void;
+}
+
+/** Why a root's state is not locked. */
+export interface RootRefusal {
+  root: string;
+  backend: string;
+  refused: StateRefusalKind;
+  reason: string;
+}
+
+/** The roots whose state chant does not lock: the take stops before any lock. */
+export class StateRefusedError extends Error {
+  constructor(readonly refusals: RootRefusal[]) {
+    super(refusals.map((r) => `${r.root}: ${r.reason}`).join("\n"));
+    this.name = "StateRefusedError";
+  }
+}
+
+/** How a write ended: written, or kept with the lock held. */
+export type WriteOutcome = { written: true; attempts: number } | { written: false; attempts: number; error: string; kept: string; lockHeld: true };
+
+const sleepMs = (ms: number): Promise<void> => new Promise((r) => setTimeout(r, ms));
+const safeName = (root: string): string => (root === "." ? "root" : root.replace(/[^\w.-]/g, "_"));
+const body = (s: StateFile): string => JSON.stringify(s, null, 2) + "\n";
+
+export class RootStates {
+  private readonly exec: BinaryExec;
+  private readonly op: LockOperation;
+  private readonly objects = new Map<string, StateObject>();
+  private locks: HeldLock[] = [];
+  private readonly heldOnPurpose = new Set<string>();
+
+  constructor(private readonly o: RootStatesOptions) {
+    this.exec = o.exec ?? runBinary;
+    this.op = o.op ?? COMPOSE_LOCK;
+    if (o.runId) this.op = { operation: this.op.operation, info: `${this.op.info} (run ${o.runId})` };
+  }
+
+  private log(line: string): void {
+    this.o.log?.(line);
+  }
+
+  private rootOf(root: string): RootState {
+    const r = this.o.roots.find((x) => x.root === root);
+    if (!r) throw new Error(`${root} is not one of this run's roots`);
+    return r;
+  }
+
+  /** Where a root's state is, or why it is not locked. */
+  object(root: string): StateObject {
+    let o = this.objects.get(root);
+    if (!o) {
+      const r = this.rootOf(root);
+      o = r.object ?? stateObject(r.dir, r.env, undefined, this.o.binary, {
+        ...(r.backendConfig ? { backendConfig: r.backendConfig } : {}),
+        ...(r.backendConfigFiles ? { backendConfigFiles: r.backendConfigFiles } : {}),
+      });
+      this.objects.set(root, o);
+    }
+    return o;
+  }
+
+  /** Every root whose state chant does not lock, with why. */
+  refusals(): RootRefusal[] {
+    return this.o.roots.flatMap((r) => {
+      const o = this.object(r.root);
+      return isRefused(o) ? [{ root: r.root, backend: o.backend, refused: o.refused, reason: o.reason }] : [];
+    });
+  }
+
+  /** The locks held now, by root. */
+  held(): string[] {
+    return this.locks.map((l) => l.root);
+  }
+
+  private lockOf(root: string): HeldLock | undefined {
+    return this.locks.find((l) => l.root === root);
+  }
+
+  /**
+   * Take every root's lock, in order. A root chant does not lock stops the
+   * take before any lock (StateRefusedError); a lock another run holds
+   * releases what was taken and throws StateLockedError, naming the holder.
+   * A lock an earlier run of this operation left held with a kept state is
+   * taken over, and every kept state is written before this returns.
+   */
+  async take(): Promise<void> {
+    const refused = this.refusals();
+    if (refused.length) throw new StateRefusedError(refused);
+    const stores = this.o.roots.map((r) => {
+      const object = this.object(r.root) as LockableState;
+      return { root: r.root, store: this.o.openStore ? this.o.openStore(r, object) : stateStore(object, r.dir, this.o.fetch) };
+    });
+    const now = this.o.now?.() ?? new Date().toISOString();
+    this.locks = await takeLocks(
+      stores,
+      now,
+      this.op,
+      async (root, store, held) => {
+        // A lock an earlier composed run left held for a state it could not write: this run writes that state first.
+        const ours = held.info.includes(this.op.operation) || held.info.includes(LEGACY_COMPOSE_OPERATION);
+        if (!ours || !(await this.kept(store))) return false;
+        this.log(`${root}: its lock is held by an earlier run that kept a state it could not write; this run takes the lock over and writes that state first`);
+        return true;
+      },
+      this.o.who,
+    );
+    for (const l of this.locks) {
+      try {
+        await this.writeKept(l);
+      } catch (e) {
+        // The kept state is still unwritten: its lock stays held, the others are released.
+        this.heldOnPurpose.add(l.root);
+        await this.release();
+        throw new Error(`${l.root}: the state an earlier run kept for it could not be written: ${(e as Error).message}`);
+      }
+    }
+  }
+
+  /** The state kept beside a store's state and the suffix it is under, chant's name first; undefined when none is kept. */
+  private async kept(store: StateStore): Promise<{ suffix: string; text: string } | undefined> {
+    for (const suffix of [UNWRITTEN_SUFFIX, LEGACY_UNWRITTEN_SUFFIX]) {
+      const text = await store.readBeside(suffix).catch(() => undefined);
+      if (text?.trim()) return { suffix, text };
+    }
+    return undefined;
+  }
+
+  /** A state an earlier run kept beside a root's state: write it, then remove what was kept. */
+  private async writeKept(lock: HeldLock): Promise<void> {
+    const kept = await this.kept(lock.store);
+    if (!kept) return;
+    await this.writeState(lock.root, JSON.parse(kept.text) as StateFile);
+    await lock.store.removeBeside(kept.suffix);
+    this.log(`${lock.root}: wrote the state an earlier run kept at ${lock.store.besideLocation(kept.suffix)}`);
+  }
+
+  /** A root's state as `state pull` gives it, or null when it has none. */
+  pull(root: string): Promise<StateFile | null> {
+    const r = this.rootOf(root);
+    return pullState(this.exec, this.o.binary, root, r.dir, r.env);
+  }
+
+  /** Write one root's state under its lock: the store's own write (azurerm's lease), else `state push -lock=false`. */
+  private async writeState(root: string, state: StateFile): Promise<void> {
+    const lock = this.lockOf(root);
+    if (!lock) throw new Error(`${root}: its state is written under its lock, and none is held`);
+    if (lock.store.write) {
+      await lock.store.write(body(state));
+      return;
+    }
+    const r = this.rootOf(root);
+    const file = join(this.o.work, `${safeName(root)}.tfstate`);
+    mkdirSync(this.o.work, { recursive: true });
+    writeFileSync(file, body(state));
+    const push = await this.exec(this.o.binary, ["state", "push", "-lock=false", file], r.dir, r.env);
+    if (push.code !== 0) throw new Error(`state push failed: ${lastLines(push.out, 3)}`);
+  }
+
+  /**
+   * Write one root's state, retrying after each delay while the lock is
+   * held. A state still unwritten after the last retry is kept, and its lock
+   * stays held: `release` leaves it.
+   */
+  async write(root: string, state: StateFile): Promise<WriteOutcome> {
+    const delays = this.o.retryDelays ?? DEFAULT_RETRY_DELAYS;
+    const sleep = this.o.sleep ?? sleepMs;
+    let last = "";
+    for (let i = 0; i <= delays.length; i++) {
+      try {
+        await this.writeState(root, state);
+        return { written: true, attempts: i + 1 };
+      } catch (e) {
+        last = (e as Error).message;
+        if (i < delays.length) {
+          this.log(`${root}: the write of its state failed (${last}); trying again in ${Math.round(delays[i]! / 1000)}s`);
+          await sleep(delays[i]!);
+        }
+      }
+    }
+    const kept = await this.keep(root, state);
+    this.heldOnPurpose.add(root);
+    this.log(`${root}: its state could not be written (${last}); kept at ${kept}, and its lock stays held until a later run writes it`);
+    return { written: false, attempts: delays.length + 1, error: last, kept, lockHeld: true };
+  }
+
+  /** Keep a state that could not be written: beside it in its own backend, else as a file of the job's report directory. Returns where. */
+  private async keep(root: string, state: StateFile): Promise<string> {
+    const lock = this.lockOf(root);
+    try {
+      if (lock) {
+        await lock.store.writeBeside(UNWRITTEN_SUFFIX, body(state));
+        return lock.store.besideLocation(UNWRITTEN_SUFFIX);
+      }
+    } catch (e) {
+      this.log(`${root}: the state could not be kept beside its backend's state either (${(e as Error).message}); it is kept under the job's report directory`);
+    }
+    const file = join(this.o.reportDir, UNWRITTEN_DIR, `${safeName(root)}.tfstate`);
+    mkdirSync(dirname(file), { recursive: true });
+    writeFileSync(file, body(state));
+    return file;
+  }
+
+  /**
+   * Keep a root's lock held past {@link release}, as a kept state's is: for a
+   * root whose state the run could neither write nor keep, so no other run
+   * plans against a state that may not hold what applied.
+   */
+  hold(root: string): void {
+    if (this.lockOf(root)) this.heldOnPurpose.add(root);
+  }
+
+  /** The roots whose lock stays held past {@link release}. */
+  heldOnPurposeRoots(): string[] {
+    return [...this.heldOnPurpose].filter((r) => this.lockOf(r)).sort();
+  }
+
+  /** Release a root's lock, or every lock but those held on purpose for a kept state. */
+  async release(root?: string): Promise<void> {
+    const which = root ? [root] : [...new Set(this.locks.map((l) => l.root))].filter((r) => !this.heldOnPurpose.has(r));
+    for (const r of which) {
+      const mine = this.locks.filter((l) => l.root === r);
+      this.locks = this.locks.filter((l) => l.root !== r);
+      for (const l of mine) await l.release().catch((e: unknown) => this.log(`${r}: its lock could not be released: ${(e as Error).message}`));
+    }
+  }
+}

package/src/state/s3.ts

@@ -0,0 +1,21 @@
+/**
+ * The S3 client a state's lock and write use: chant core's (#4554,
+ * `@intentius/chant/object-store`), which signs with chant's one AWS
+ * signer. Get, put, put only when absent, head and delete one object; with
+ * no static keys, a role is assumed with the job's OIDC token through STS
+ * AssumeRoleWithWebIdentity.
+ */
+import { s3FromEnv as coreS3FromEnv, type S3Target } from "@intentius/chant/object-store";
+
+export { assumeRoleWithWebIdentity, S3Client, sign, type S3Credentials, type S3Location, type S3Target, type S3WebIdentity } from "@intentius/chant/object-store";
+
+/**
+ * The target for `bucket` from the AWS environment variables. Credentials,
+ * first match wins: static keys in `AWS_ACCESS_KEY_ID` and
+ * `AWS_SECRET_ACCESS_KEY`; `AWS_ROLE_ARN` assumed with the token in
+ * `AWS_WEB_IDENTITY_TOKEN_FILE`, in a session named `chant-state` unless
+ * AWS_ROLE_SESSION_NAME names one.
+ */
+export function s3FromEnv(where: { bucket: string; endpoint?: string }, env: NodeJS.ProcessEnv = process.env): S3Target {
+  return coreS3FromEnv({ ...where, sessionName: "chant-state", setting: "the state bucket" }, env);
+}

package/src/state/store.ts

@@ -0,0 +1,225 @@
+/**
+ * A state where it is, opened for its lock and for what a run keeps beside
+ * it. Each backend locks its own way:
+ *
+ *   s3       the lock file `use_lockfile` takes, `<key>.tflock`, written only
+ *            when absent
+ *   gcs      the lock object `<prefix>/<workspace>.tflock`, written only when
+ *            absent; the ID `force-unlock` takes is its generation
+ *   azurerm  a lease on the state blob, with the lock info base64 in its
+ *            `terraformlockid` metadata; the binary cannot write the blob
+ *            while chant holds the lease, so the store writes it
+ *   local    a lock file `<path>.tflock` beside the state, created only when
+ *            absent. It keeps another chant run out; the binary's own local
+ *            lock is a file lock it takes per command, so the run writes
+ *            with `-lock=false` while it holds this one. A bare
+ *            `terraform apply` on the same checkout at the same time is not
+ *            kept out. Ruled on #4481 (ruling 4, 2026-10-25): chant's own
+ *            lock file, no Terraform lock-info file, local backends allowed.
+ *
+ * Lifted from terragucci's `backend.ts` (c493e0e): `StateStore`, `s3Store`,
+ * `gcsStore` and `azureStore`, plus a local store and `removeBeside`.
+ */
+import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { dirname, isAbsolute, join } from "node:path";
+import { AzureBlobClient, type AzureTarget } from "./azure-blob";
+import type { AzureState, GcsState, LocalState, LockableState, S3State } from "./backend";
+import { GcsClient } from "./gcs";
+import { StoreError, type StoreFetch } from "./object-store";
+import { S3Client } from "./s3";
+
+/** The lock a state's backend holds: the ID `force-unlock` takes, and the lock info the binary (or a run) wrote. */
+export interface LockHolder {
+  id: string;
+  /** The lock info as written, JSON. */
+  info: string;
+}
+
+/** A state where it is, for what locks it and keeps beside it. */
+export interface StateStore {
+  readonly backend: LockableState["backend"];
+  /** Where the state is. */
+  readonly location: string;
+  /** Where its lock is: the lock object, or the blob whose lease is the lock. */
+  readonly lockLocation: string;
+  /** The lock the backend holds now, or undefined. */
+  heldLock(): Promise<LockHolder | undefined>;
+  /** Take the state's lock with `info`, the way the backend takes it: false when it is held. */
+  lock(info: string): Promise<boolean>;
+  /** Release the lock `lock` took, or the one `adopt` took over. */
+  unlock(): Promise<void>;
+  /**
+   * Write a new state under the lock, for a backend whose lock keeps the
+   * binary from writing (azurerm's lease); undefined for a backend the
+   * binary writes with `-lock=false` while the run holds the lock.
+   */
+  write?: (body: string) => Promise<void>;
+  /**
+   * The object beside the state whose key is the state's with `suffix` after
+   * it, in the same bucket, container or directory, under the state's own
+   * access controls: read (undefined when there is none), written, removed.
+   */
+  readBeside(suffix: string): Promise<string | undefined>;
+  writeBeside(suffix: string, body: string): Promise<void>;
+  removeBeside(suffix: string): Promise<void>;
+  /** Where the object beside the state with `suffix` is, for messages. */
+  besideLocation(suffix: string): string;
+  /**
+   * Take over a lock this store finds held, which an earlier run of the same
+   * operation left held on purpose: `unlock` then releases it. A lease
+   * (azurerm) is taken over by its ID, the others need nothing.
+   */
+  adopt(held: LockHolder): void;
+}
+
+/** The ID in a lock info's JSON, or "" when it names none. */
+export function lockId(text: string): string {
+  try {
+    const v = JSON.parse(text) as { ID?: unknown };
+    return typeof v.ID === "string" ? v.ID : "";
+  } catch {
+    return "";
+  }
+}
+
+const JSON_TYPE = "application/json";
+
+/** The S3 state, locked by the lock file `use_lockfile` takes. */
+function s3Store(o: S3State, fetchFn?: StoreFetch): StateStore {
+  const client = new S3Client(o.target, fetchFn);
+  const lockKey = `${o.key}.tflock`;
+  return {
+    backend: "s3",
+    location: `s3://${o.bucket}/${o.key}`,
+    lockLocation: `s3://${o.bucket}/${lockKey}`,
+    heldLock: async () => {
+      const text = await client.get(lockKey);
+      return text === undefined ? undefined : { id: lockId(text), info: text };
+    },
+    lock: (info) => client.putIfAbsent(lockKey, info, JSON_TYPE),
+    unlock: () => client.remove(lockKey),
+    readBeside: (suffix) => client.get(`${o.key}${suffix}`),
+    writeBeside: async (suffix, body) => void (await client.put(`${o.key}${suffix}`, body, JSON_TYPE)),
+    removeBeside: (suffix) => client.remove(`${o.key}${suffix}`),
+    besideLocation: (suffix) => `s3://${o.bucket}/${o.key}${suffix}`,
+    adopt: () => {},
+  };
+}
+
+/** The GCS state, locked by `<prefix>/<workspace>.tflock`, whose generation is the lock's ID. */
+function gcsStore(o: GcsState, fetchFn?: StoreFetch): StateStore {
+  const client = new GcsClient(o.target, fetchFn);
+  return {
+    backend: "gcs",
+    location: `gs://${o.bucket}/${o.key}`,
+    lockLocation: `gs://${o.bucket}/${o.lock}`,
+    heldLock: async () => {
+      const head = await client.head(o.lock);
+      const info = head.exists ? await client.get(o.lock) : undefined;
+      if (!head.exists || info === undefined) return undefined;
+      return { id: head.generation ?? lockId(info), info };
+    },
+    lock: (info) => client.putIfAbsent(o.lock, info, JSON_TYPE),
+    unlock: () => client.remove(o.lock),
+    readBeside: (suffix) => client.get(`${o.key}${suffix}`),
+    writeBeside: async (suffix, body) => void (await client.put(`${o.key}${suffix}`, body, JSON_TYPE)),
+    removeBeside: (suffix) => client.remove(`${o.key}${suffix}`),
+    besideLocation: (suffix) => `gs://${o.bucket}/${o.key}${suffix}`,
+    adopt: () => {},
+  };
+}
+
+/** The metadata an azurerm backend keeps a lock's info in, base64 JSON, on the leased blob. */
+export const LOCK_METADATA = "terraformlockid";
+
+/** The azurerm state blob, locked by a lease on it with the lock info in its metadata. */
+function azureStore(o: AzureState, fetchFn?: StoreFetch): StateStore {
+  const client = new AzureBlobClient({ account: o.account, container: o.container, endpoint: o.endpoint, ...o.auth } as AzureTarget, fetchFn);
+  let leaseId: string | undefined;
+  const at = `az://${o.account}/${o.container}/${o.key}`;
+  return {
+    backend: "azurerm",
+    location: at,
+    lockLocation: at,
+    heldLock: async () => {
+      const head = await client.head(o.key);
+      if (!head.exists || head.leaseState !== "leased") return undefined;
+      const meta = head.metadata[LOCK_METADATA];
+      const info = meta ? Buffer.from(meta, "base64").toString("utf-8") : "";
+      return { id: lockId(info), info };
+    },
+    lock: async (info) => {
+      const id = lockId(info);
+      // The backend leases a blob that is there: a state with none yet gets an empty one first, as the backend makes it.
+      if (!(await client.head(o.key)).exists) await client.putBlob(o.key, "", JSON_TYPE, { ifNoneMatch: true });
+      if (!(await client.lease(o.key, "acquire", id))) return false;
+      leaseId = id;
+      await client.setMetadata(o.key, { [LOCK_METADATA]: Buffer.from(info).toString("base64") }, id);
+      return true;
+    },
+    unlock: async () => {
+      if (!leaseId) return;
+      const id = leaseId;
+      await client.setMetadata(o.key, {}, id);
+      await client.lease(o.key, "release", id);
+      leaseId = undefined;
+    },
+    write: async (body) => {
+      if (!leaseId) throw new StoreError(`${at} is written under its lease, and none is held`);
+      // As the backend writes it: a snapshot first when it takes snapshots, and the lock info kept on the blob.
+      if (o.snapshot && (await client.head(o.key)).exists) await client.snapshot(o.key);
+      const meta = (await client.head(o.key)).metadata;
+      await client.putBlob(o.key, body, JSON_TYPE, { leaseId, metadata: meta });
+    },
+    readBeside: (suffix) => client.get(`${o.key}${suffix}`),
+    writeBeside: async (suffix, body) => void (await client.putBlob(`${o.key}${suffix}`, body, JSON_TYPE)),
+    removeBeside: (suffix) => client.remove(`${o.key}${suffix}`),
+    besideLocation: (suffix) => `${at}${suffix}`,
+    adopt: (held) => {
+      leaseId = held.id;
+    },
+  };
+}
+
+/** A local state, locked by `<path>.tflock` created only when absent. `dir` is the root the path is relative to. */
+function localStore(o: LocalState, dir: string): StateStore {
+  const file = isAbsolute(o.path) ? o.path : join(dir, o.path);
+  const lockFile = `${file}.tflock`;
+  const read = (f: string): string | undefined => (existsSync(f) ? readFileSync(f, "utf-8") : undefined);
+  return {
+    backend: "local",
+    location: file,
+    lockLocation: lockFile,
+    heldLock: async () => {
+      const text = read(lockFile);
+      return text === undefined ? undefined : { id: lockId(text), info: text };
+    },
+    lock: async (info) => {
+      mkdirSync(dirname(lockFile), { recursive: true });
+      try {
+        writeFileSync(lockFile, info, { flag: "wx" });
+        return true;
+      } catch (e) {
+        if ((e as NodeJS.ErrnoException).code === "EEXIST") return false;
+        throw e;
+      }
+    },
+    unlock: async () => rmSync(lockFile, { force: true }),
+    readBeside: async (suffix) => read(`${file}${suffix}`),
+    writeBeside: async (suffix, body) => {
+      mkdirSync(dirname(file), { recursive: true });
+      writeFileSync(`${file}${suffix}`, body);
+    },
+    removeBeside: async (suffix) => rmSync(`${file}${suffix}`, { force: true }),
+    besideLocation: (suffix) => `${file}${suffix}`,
+    adopt: () => {},
+  };
+}
+
+/** The store a state is in, opened with the credentials stateObject chose. `dir` is the root a local path is relative to. */
+export function stateStore(o: LockableState, dir: string, fetchFn?: StoreFetch): StateStore {
+  if (o.backend === "s3") return s3Store(o, fetchFn);
+  if (o.backend === "gcs") return gcsStore(o, fetchFn);
+  if (o.backend === "azurerm") return azureStore(o, fetchFn);
+  return localStore(o, dir);
+}