@intentius/chant-lexicon-terraform 0.124.84 → 0.124.86

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. package/dist/ci-setup.d.ts +11 -2
  2. package/dist/ci-setup.d.ts.map +1 -1
  3. package/dist/ci-setup.js +11 -6
  4. package/dist/components/capability-plugin.d.ts +12 -0
  5. package/dist/components/capability-plugin.d.ts.map +1 -1
  6. package/dist/components/capability-plugin.js +22 -4
  7. package/dist/compose.d.ts.map +1 -1
  8. package/dist/compose.js +1 -1
  9. package/dist/estate-migration.d.ts +101 -0
  10. package/dist/estate-migration.d.ts.map +1 -0
  11. package/dist/estate-migration.js +471 -0
  12. package/dist/integrity.json +2 -2
  13. package/dist/manifest.json +1 -1
  14. package/dist/op/activities/terraform.d.ts +65 -0
  15. package/dist/op/activities/terraform.d.ts.map +1 -1
  16. package/dist/op/activities/terraform.js +94 -1
  17. package/dist/state/backend.d.ts +46 -9
  18. package/dist/state/backend.d.ts.map +1 -1
  19. package/dist/state/backend.js +84 -31
  20. package/dist/state/gitlab.d.ts +1 -1
  21. package/dist/state/gitlab.d.ts.map +1 -1
  22. package/dist/state/gitlab.js +27 -14
  23. package/dist/state/index.d.ts +1 -0
  24. package/dist/state/index.d.ts.map +1 -1
  25. package/dist/state/index.js +1 -0
  26. package/dist/state/s3.d.ts +9 -3
  27. package/dist/state/s3.d.ts.map +1 -1
  28. package/dist/state/s3.js +4 -3
  29. package/dist/state/source.d.ts +92 -0
  30. package/dist/state/source.d.ts.map +1 -0
  31. package/dist/state/source.js +254 -0
  32. package/dist/state/store.d.ts +41 -4
  33. package/dist/state/store.d.ts.map +1 -1
  34. package/dist/state/store.js +125 -10
  35. package/dist/state/versions.d.ts +77 -10
  36. package/dist/state/versions.d.ts.map +1 -1
  37. package/dist/state/versions.js +118 -64
  38. package/dist/state-migration.d.ts +43 -1
  39. package/dist/state-migration.d.ts.map +1 -1
  40. package/dist/state-migration.js +251 -20
  41. package/dist/state-ops.d.ts +2 -3
  42. package/dist/state-ops.d.ts.map +1 -1
  43. package/dist/state-ops.js +4 -7
  44. package/package.json +2 -2
  45. package/src/ci-setup.test.ts +20 -0
  46. package/src/ci-setup.ts +11 -6
  47. package/src/components/capability-plugin.ts +36 -3
  48. package/src/components/terraform-apply-command-hook.test.ts +157 -0
  49. package/src/compose.ts +1 -0
  50. package/src/estate-migration.test.ts +186 -0
  51. package/src/estate-migration.ts +563 -0
  52. package/src/op/activities/terraform.ts +153 -3
  53. package/src/state/backend-verdict.test.ts +4 -3
  54. package/src/state/backend.test.ts +3 -1
  55. package/src/state/backend.ts +108 -33
  56. package/src/state/gitlab.ts +25 -12
  57. package/src/state/index.ts +1 -0
  58. package/src/state/s3.ts +5 -4
  59. package/src/state/source.ts +289 -0
  60. package/src/state/store.ts +145 -11
  61. package/src/state/versions.test.ts +365 -0
  62. package/src/state/versions.ts +148 -68
  63. package/src/state-migration-backends.test.ts +277 -0
  64. package/src/state-migration.ts +265 -23
  65. package/src/state-ops.ts +6 -8
@@ -10,6 +10,12 @@
10
10
  * restores a root's state put back to a version a migration recorded (a
11
11
  * revert), refused when the state moved past what that
12
12
  * migration left
13
+ * backends a root's state moved to the backend its code names now, from
14
+ * the backend it was in (read through the root with an
15
+ * override file naming that backend, in a data dir of its own),
16
+ * from a TFE-API workspace or from an exported state file
17
+ * (`./state/source.ts`), #4806. The backend the code names must
18
+ * hold no state; the state read is left where it was
13
19
  *
14
20
  * Planning (planTerraformMigration) initialises each root in a data dir of
15
21
  * its own, reads its backend version id and pulls its state, keeps only
@@ -23,28 +29,44 @@
23
29
  * a write that stops half way leaves a resource in two states rather than
24
30
  * in none. Then each root plans against its real backend.
25
31
  *
32
+ * A backend that keeps no versions (pg, kubernetes, consul, an http backend
33
+ * other than GitLab's) takes the general path, for each kind (#4806): the
34
+ * binary reads it with `state pull` and writes it with `state push` under
35
+ * the backend's own lock, which refuses a state of another lineage or an
36
+ * older serial, and the digest binds the state's contents, since there is
37
+ * no version id to bind. A backend move into one replaces the empty state a
38
+ * backend writes at init (kubernetes does) with `-force`; the digest bound
39
+ * that it was empty.
40
+ *
41
+ * A backend move's root names its source (`MigrationRootPlan.source`): the
42
+ * request binds it, under the locks the engine reads it again, and the lock
43
+ * of a workspace it is read from is taken first, through the workspace's
44
+ * API.
45
+ *
26
46
  * State contents never leave the work dir. The record holds version ids and
27
47
  * digests.
28
48
  *
29
49
  * Lifted from terragucci's `migrate.ts` (`moveResources`, `addressMatches`,
30
50
  * `derivedLineage`, `stateDigest`, `resourceChanges`, `backendBlock`,
31
- * `proofPlan`, `planMigration`'s moves and reverts, `applyMigration`,
32
- * `movedSince`), so terragucci deletes its copy. A backend move (a state
33
- * read from another backend, a TFE workspace or an exported file) and a
34
- * migration of choudoufu's live estates stay terragucci's for now.
51
+ * `proofPlan`, `planMigration`, `applyMigration`, `movedSince`, `takeLocks`,
52
+ * `readSource`, `withBackend`, `isPulled`, `PULLED_BACKENDS`,
53
+ * `backendRefusal`, `parseBackends`, `pulledProblems`, `locationOf`), so
54
+ * terragucci deletes its copy. A migration of choudoufu's live estates is
55
+ * `./estate-migration.ts`.
35
56
  */
36
57
  import { createHash } from "node:crypto";
37
58
  import { hostname } from "node:os";
38
59
  import { join, resolve } from "node:path";
39
60
  import { mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
40
- import type { MigrationCheck, MigrationRootPlan, MigrationStateAt, StateMigrationEngine } from "@intentius/chant/state-migration";
61
+ import type { MigrationCheck, MigrationRootPlan, MigrationSource, MigrationStateAt, StateMigrationEngine } from "@intentius/chant/state-migration";
41
62
  import { initRoot, planStateCheck, pushStateFile, type RootRun } from "./state-ops";
42
- import { isRefused, isStored, stateLocation, stateObject, type LockableState } from "./state/backend";
63
+ import { initialisedBackend, isRefused, isStored, readableStateObject, stateLocation, stateObject, type InitialisedBackend, type LockableState, type StateObject } from "./state/backend";
43
64
  import { pullState, runBinary, type BinaryExec, type StateFile, type StateResource } from "./state/exec";
44
- import { takeLocksWithSources, type LockOperation } from "./state/locks";
65
+ import { SourceLockedError, StateLockedError, takeLocksWithSources, type LockOperation, type SourceLock } from "./state/locks";
45
66
  import type { StoreFetch } from "./state/object-store";
67
+ import { EXTERNAL_BACKENDS, externalProblems, externalSource, type ExternalSource } from "./state/source";
46
68
  import { stateStore, type StateStore } from "./state/store";
47
- import { stateVersionSource } from "./state/versions";
69
+ import { keepsHistory, NO_HISTORY, recordedAddress, stateVersionSource } from "./state/versions";
48
70
 
49
71
  /** Resources that move from one root's state to another's. */
50
72
  export interface Move {
@@ -64,6 +86,19 @@ export interface Restore {
64
86
  from_version_id: string | null;
65
87
  }
66
88
 
89
+ /** A root's state moving to the backend its code now names, from the backend it was in (#4806). */
90
+ export interface BackendMove {
91
+ root: string;
92
+ /** Where the state is now: a backend's type and configuration, as a backend block gives them; `remote` or `cloud` for a TFE-API workspace; `file` with `config.path` for an exported state file. */
93
+ from: { backend: string; config: Record<string, unknown> };
94
+ }
95
+
96
+ /** The backends a migration reads and writes through the binary, under the backend's own lock: those that keep no versions (#4806). */
97
+ export const PULLED_BACKENDS: readonly string[] = Object.keys(NO_HISTORY);
98
+
99
+ /** The backends a migration locks and writes itself. */
100
+ export const MIGRATED_BACKENDS: readonly string[] = ["s3", "gcs", "azurerm", "local", "http"];
101
+
67
102
  /** A migration that cannot be planned: an address in no state, a root that is refused. Nothing was written. */
68
103
  export class MigrationError extends Error {
69
104
  constructor(message: string) {
@@ -95,6 +130,82 @@ export function derivedLineage(...parts: string[]): string {
95
130
 
96
131
  const emptyState = (lineage: string, serial: number, terraformVersion = "1.0.0"): StateFile => ({ version: 4, terraform_version: terraformVersion, serial, lineage, outputs: {}, resources: [], check_results: null });
97
132
 
133
+ /** A state with no resources and no outputs: what a backend writes at init before anything is applied. */
134
+ const isEmptyState = (s: StateFile): boolean => s.resources.length === 0 && Object.keys(s.outputs ?? {}).length === 0;
135
+
136
+ /** A state the binary reads and writes for a migration, under its backend's own lock: one that keeps no versions (pg, kubernetes, consul, an http backend other than GitLab's). */
137
+ export interface PulledState {
138
+ backend: string;
139
+ pulled: true;
140
+ /** Its address, credentials left out, when the configuration names one. */
141
+ location?: string;
142
+ }
143
+
144
+ /** The backend as a pulled state, or undefined when it is not one that keeps no versions. */
145
+ export function pulledState(b: InitialisedBackend, env: NodeJS.ProcessEnv = {}): PulledState | undefined {
146
+ if (!PULLED_BACKENDS.includes(b.type) || keepsHistory(b, env) !== false) return undefined;
147
+ const location = recordedAddress(b);
148
+ return { backend: b.type, pulled: true, ...(location ? { location } : {}) };
149
+ }
150
+
151
+ /** A state a migration writes or reads: one chant locks, or one the binary reads and writes under its own lock. */
152
+ export type MigratedState = LockableState | PulledState;
153
+
154
+ const isPulledState = (o: MigratedState): o is PulledState => "pulled" in o;
155
+
156
+ /** Where a migrated state is, as the record names it. */
157
+ export const migratedLocation = (o: MigratedState): string | undefined => (isPulledState(o) ? o.location : stateLocation(o));
158
+
159
+ /** Settings of a pulled backend that hold a credential, which the job gives in the backend's own variables instead. */
160
+ const SECRET_KEYS: Record<string, string> = { password: "the backend's password variable", token: "the backend's token variable", client_key: "KUBE_CLIENT_KEY_DATA", access_token: "CONSUL_HTTP_TOKEN" };
161
+
162
+ /** The settings a backend move's `from.config` must give, by backend: where the state is. */
163
+ const FROM_NEEDS: Record<string, string[]> = { s3: ["bucket", "key"], gcs: ["bucket"], azurerm: ["storage_account_name", "container_name", "key"], kubernetes: ["secret_suffix"], consul: ["path"], http: ["address"] };
164
+
165
+ /**
166
+ * Problems with backend moves as a migration file gives them, one line each,
167
+ * headed `backends[<i>]`: a root that moves twice, a source backend chant
168
+ * does not read, a config that does not say where the state is, a
169
+ * credential written in the repo. None: the moves can be planned.
170
+ */
171
+ export function backendMoveProblems(moves: readonly BackendMove[]): string[] {
172
+ const p: string[] = [];
173
+ const seen = new Set<string>();
174
+ const kinds = [...MIGRATED_BACKENDS, ...PULLED_BACKENDS.filter((b) => !MIGRATED_BACKENDS.includes(b)), ...EXTERNAL_BACKENDS];
175
+ for (const [i, b] of moves.entries()) {
176
+ const at = `backends[${i}]`;
177
+ if (seen.has(b.root)) p.push(`${at}: ${b.root} moves twice`);
178
+ seen.add(b.root);
179
+ if (!kinds.includes(b.from.backend)) {
180
+ p.push(`${at}.from.backend must be ${kinds.join(", ")}`);
181
+ continue;
182
+ }
183
+ const config = b.from.config;
184
+ if (EXTERNAL_BACKENDS.includes(b.from.backend)) p.push(...externalProblems(at, b.from.backend, config));
185
+ if (PULLED_BACKENDS.includes(b.from.backend)) {
186
+ for (const k of Object.keys(config)) if (k in SECRET_KEYS) p.push(`${at}.from.config.${k}: a credential does not belong in the repo; the job gives it in ${SECRET_KEYS[k]}`);
187
+ const conn = config.conn_str;
188
+ if (b.from.backend === "pg" && typeof conn === "string" && (/^postgres(ql)?:\/\/[^/@]*:[^/@]*@/i.test(conn) || /(^|\s)password\s*=/.test(conn))) p.push(`${at}.from.config.conn_str holds a password, which does not belong in the repo; the job gives it in PGPASSWORD`);
189
+ }
190
+ const needs = FROM_NEEDS[b.from.backend];
191
+ if (needs && needs.some((k) => typeof config[k] !== "string")) p.push(`${at}.from.config must name the ${needs.join(", ").replace(/, ([^,]*)$/, " and $1")} of the state`);
192
+ }
193
+ return p;
194
+ }
195
+
196
+ /** An HCL value from the JSON of a backend's configuration. */
197
+ function hcl(v: unknown): string {
198
+ if (v && typeof v === "object" && !Array.isArray(v)) return `{ ${Object.entries(v).filter(([, x]) => x !== null && x !== undefined).map(([k, x]) => `${k} = ${hcl(x)}`).join(", ")} }`;
199
+ if (Array.isArray(v)) return `[${v.map(hcl).join(", ")}]`;
200
+ return JSON.stringify(v);
201
+ }
202
+
203
+ /** A terraform block whose backend is `type` with `config`, for an override file. */
204
+ export function backendBlock(type: string, config: Record<string, unknown>): string {
205
+ const body = Object.entries(config).filter(([, v]) => v !== null && v !== undefined).map(([k, v]) => ` ${k} = ${hcl(v)}\n`).join("");
206
+ return `terraform {\n backend ${JSON.stringify(type)} {\n${body} }\n}\n`;
207
+ }
208
+
98
209
  /**
99
210
  * The states after the moves. `before` holds each affected root's state, or
100
211
  * null when it has none. A changed state keeps its lineage and gets the next
@@ -172,14 +283,20 @@ export interface TerraformMigrationInput {
172
283
  moves?: Move[];
173
284
  /** States put back to a version a migration recorded. */
174
285
  restores?: Restore[];
175
- /** Every root the moves or restores name. */
286
+ /** States moved to the backend each root's code names now (#4806). */
287
+ backends?: BackendMove[];
288
+ /** Every root the moves, restores or backend moves name. */
176
289
  roots: MigrationRootRun[];
290
+ /** What a `file` source's relative path is read from. Default the working directory. */
291
+ repo?: string;
177
292
  /** The work dir: new states, plans and data dirs. The caller removes it. */
178
293
  work: string;
179
294
  exec?: BinaryExec;
180
295
  fetch?: StoreFetch;
181
296
  /** What the migration's lock says. */
182
297
  lock?: LockOperation;
298
+ /** What the lock of a workspace a backend move reads from says. Default the migration's operation, reading. */
299
+ sourceLock?: LockOperation;
183
300
  /** Who the lock names. Default `chant@<host>`. */
184
301
  who?: string;
185
302
  /** The proof plan's override file. Default {@link DEFAULT_OVERRIDE_FILE}. */
@@ -193,15 +310,22 @@ export interface TerraformMigrationPlan {
193
310
  engine: StateMigrationEngine;
194
311
  }
195
312
 
313
+ /** Where a backend move's state is now: a backend read through the root with an override file, or a source read without the binary. */
314
+ type Source = { object: MigratedState; block: string; data: string } | { external: ExternalSource };
315
+
196
316
  interface Planned {
197
317
  run: MigrationRootRun & { env: NodeJS.ProcessEnv };
198
- object: LockableState;
318
+ object: MigratedState;
199
319
  file: string;
200
320
  gained: number;
321
+ source?: Source;
322
+ /** The state the root's backend holds is the empty one it wrote at init: the push is forced over it. */
323
+ replacesEmpty?: boolean;
201
324
  }
202
325
 
203
326
  /** A root's version id now, when its store keeps versions. `pin` lets an azurerm blob with snapshots only take one to name it. */
204
- async function versionNow(o: LockableState, fetchFn: StoreFetch | undefined, pin: boolean): Promise<string | undefined> {
327
+ async function versionNow(o: MigratedState, fetchFn: StoreFetch | undefined, pin: boolean): Promise<string | undefined> {
328
+ if (isPulledState(o)) return undefined;
205
329
  const src = stateVersionSource(o, fetchFn, { pin });
206
330
  if ("refused" in src) return undefined;
207
331
  return (await src.current()).versionId;
@@ -217,15 +341,20 @@ export async function planTerraformMigration(input: TerraformMigrationInput): Pr
217
341
  const log = input.log ?? (() => {});
218
342
  const lockOp = input.lock ?? DEFAULT_MIGRATE_LOCK;
219
343
  const override = input.overrideFile ?? DEFAULT_OVERRIDE_FILE;
220
- if (!!input.moves?.length === !!input.restores?.length) throw new MigrationError(`migration ${input.name}: give it moves or restores, one of the two`);
344
+ const kinds = [input.moves?.length && "moves", input.restores?.length && "restores", input.backends?.length && "backends"].filter(Boolean);
345
+ if (kinds.length !== 1) throw new MigrationError(`migration ${input.name}: give it moves, restores or backends, one of the three`);
346
+ if (input.backends) {
347
+ const problems = backendMoveProblems(input.backends);
348
+ if (problems.length > 0) throw new MigrationError(`migration ${input.name}:\n ${problems.join("\n ")}`);
349
+ }
221
350
  mkdirSync(input.work, { recursive: true });
222
- const named = [...new Set(input.moves ? input.moves.flatMap((m) => [m.from, m.to]) : input.restores!.map((r) => r.root))].sort();
351
+ const named = [...new Set(input.moves ? input.moves.flatMap((m) => [m.from, m.to]) : input.restores ? input.restores.map((r) => r.root) : input.backends!.map((b) => b.root))].sort();
223
352
  const byName = new Map(input.roots.map((r) => [r.root, r]));
224
353
  const missing = named.filter((n) => !byName.has(n));
225
354
  if (missing.length > 0) throw new MigrationError(`migration ${input.name}: ${missing.join(", ")} ${missing.length === 1 ? "is" : "are"} not among its roots`);
226
355
 
227
356
  const runs = new Map<string, Planned["run"]>();
228
- const objects = new Map<string, LockableState>();
357
+ const objects = new Map<string, MigratedState>();
229
358
  const versions = new Map<string, string | undefined>();
230
359
  const before = new Map<string, StateFile | null>();
231
360
  for (const [i, name] of named.entries()) {
@@ -235,8 +364,10 @@ export async function planTerraformMigration(input: TerraformMigrationInput): Pr
235
364
  env = { ...env, TF_DATA_DIR: join(input.work, `data-${i}`) };
236
365
  await initRoot({ ...r, env, exec, reconfigure: true });
237
366
  }
238
- const o = stateObject(r.dir, env, undefined, r.binary);
239
- if (isRefused(o)) throw new MigrationError(`migration ${input.name} cannot run: ${name}: ${o.reason}`);
367
+ const found = stateObject(r.dir, env, undefined, r.binary);
368
+ const recorded = initialisedBackend(r.dir, env);
369
+ const o: MigratedState | undefined = !isRefused(found) ? found : found.refused === "backend" && recorded ? pulledState(recorded, env) : undefined;
370
+ if (!o) throw new MigrationError(`migration ${input.name} cannot run: ${name}: ${(found as { reason: string }).reason}`);
240
371
  runs.set(name, { ...r, env });
241
372
  objects.set(name, o);
242
373
  versions.set(name, await versionNow(o, input.fetch, false));
@@ -244,12 +375,29 @@ export async function planTerraformMigration(input: TerraformMigrationInput): Pr
244
375
  }
245
376
 
246
377
  const after = new Map<string, StateFile>();
378
+ const sources = new Map<string, { source: Source; at: MigrationSource }>();
379
+ const replaces = new Set<string>();
247
380
  if (input.moves) {
248
381
  for (const [root, s] of moveResources(input.name, input.moves, before)) after.set(root, s);
382
+ } else if (input.backends) {
383
+ for (const [i, b] of input.backends.entries()) {
384
+ const run = runs.get(b.root)!;
385
+ const target = objects.get(b.root)!;
386
+ const source = sourceOf(input, b, run, target, i);
387
+ // A backend that writes an empty state of its own lineage at init (kubernetes) has it replaced; the digest binds that it was empty.
388
+ const held = before.get(b.root);
389
+ if (held && isPulledState(target) && isEmptyState(held)) replaces.add(b.root);
390
+ else if (held) throw new MigrationError(`migration ${input.name}: the backend ${b.root}'s code names, ${migratedLocation(target) ?? target.backend}, already holds a state, so a move would overwrite it`);
391
+ const read = await readSource(source, run, exec, input.fetch, override);
392
+ const where = "external" in source ? source.external.location : (migratedLocation(source.object) ?? b.from.backend);
393
+ if (!read.state) throw new MigrationError(`migration ${input.name}: ${where} holds no state of ${b.root} to move`);
394
+ sources.set(b.root, { source, at: { ...(where ? { location: where } : {}), ...(read.version ? { version_id: read.version } : {}), digest: stateDigest(read.state) } });
395
+ after.set(b.root, read.state);
396
+ }
249
397
  } else {
250
398
  for (const r of input.restores!) {
251
399
  const o = objects.get(r.root)!;
252
- const location = stateLocation(o);
400
+ const location = migratedLocation(o);
253
401
  if (location !== r.location) throw new MigrationError(`migration ${input.name}: ${r.root}'s state is at ${location}, and the revert restores ${r.location}`);
254
402
  const v = versions.get(r.root);
255
403
  if ((r.from_version_id ?? undefined) !== v) throw new MigrationError(`migration ${input.name}: ${r.root}'s state is at version ${v ?? "none"}, and the migration it reverts left ${r.from_version_id ?? "none"}; it moved since, so putting the older version back would undo that change too`);
@@ -260,6 +408,7 @@ export async function planTerraformMigration(input: TerraformMigrationInput): Pr
260
408
  after.set(r.root, emptyState(lineage, serial, now?.terraform_version));
261
409
  continue;
262
410
  }
411
+ if (isPulledState(o)) throw new MigrationError(`migration ${input.name}: ${r.root}'s ${o.backend} backend keeps no versions to put back`);
263
412
  const src = stateVersionSource(o, input.fetch);
264
413
  if ("refused" in src) throw new MigrationError(`migration ${input.name}: ${r.root}: ${src.refused}`);
265
414
  const text = await src.readVersion(r.version_id);
@@ -277,17 +426,71 @@ export async function planTerraformMigration(input: TerraformMigrationInput): Pr
277
426
  const run = runs.get(name)!;
278
427
  const file = join(input.work, `${i}.tfstate`);
279
428
  writeFileSync(file, `${JSON.stringify(state, null, 2)}\n`);
280
- planned.set(name, { run, object: objects.get(name)!, file, gained: state.resources.length - (before.get(name)?.resources.length ?? 0) });
429
+ const src = sources.get(name);
430
+ planned.set(name, { run, object: objects.get(name)!, file, gained: state.resources.length - (before.get(name)?.resources.length ?? 0), ...(src ? { source: src.source } : {}), ...(replaces.has(name) ? { replacesEmpty: true } : {}) });
281
431
  const proof = await proofPlan(run, exec, file, join(input.work, `proof-${i}`), override);
282
432
  log(`${name}: ${proof.changes.length === 0 ? "no changes against its new state" : `${proof.changes.length} change${proof.changes.length === 1 ? "" : "s"} against its new state: ${proof.changes.join(", ")}`}`);
283
433
  const was = before.get(name) ?? null;
284
434
  const v = versions.get(name);
285
- const location = stateLocation(objects.get(name)!);
286
- roots.push({ root: name, ...(location ? { location } : {}), before: { ...(v ? { version_id: v } : {}), digest: was ? stateDigest(was) : null }, after: { digest: stateDigest(state) }, proof });
435
+ const location = migratedLocation(objects.get(name)!);
436
+ roots.push({ root: name, ...(location ? { location } : {}), before: { ...(v ? { version_id: v } : {}), digest: was ? stateDigest(was) : null }, after: { digest: stateDigest(state) }, proof, ...(src ? { source: src.at } : {}) });
287
437
  }
288
438
  return { roots, engine: migrationEngine(planned, input, exec, lockOp) };
289
439
  }
290
440
 
441
+ /**
442
+ * Where a backend move reads its state from: an external source (a TFE-API
443
+ * workspace, a state file), or the backend `from` names, read through the
444
+ * root. Throws MigrationError for a backend chant does not read, an s3
445
+ * backend that takes no lock file, and a source the root's code names
446
+ * already.
447
+ */
448
+ function sourceOf(input: TerraformMigrationInput, b: BackendMove, run: Planned["run"], target: MigratedState, i: number): Source {
449
+ const external = externalSource(input.repo ?? process.cwd(), b.from.backend, b.from.config, run.env, input.fetch);
450
+ if (external) return { external };
451
+ const from = { type: b.from.backend, config: b.from.config };
452
+ const pulled = pulledState(from, run.env);
453
+ let object: MigratedState;
454
+ if (pulled) object = pulled;
455
+ else {
456
+ const o = readableStateObject(run.dir, run.env, from);
457
+ if (isRefused(o)) throw new MigrationError(`migration ${input.name} cannot run: the backend ${b.root} moves from: ${o.reason}`);
458
+ if (o.backend === "s3" && !o.lockfile) throw new MigrationError(`migration ${input.name} cannot run: the backend ${b.root} moves from: its s3 backend takes no lock file (use_lockfile = true), so a migration could not hold its lock`);
459
+ object = o;
460
+ }
461
+ const at = migratedLocation(object);
462
+ if (at !== undefined && at === migratedLocation(target)) throw new MigrationError(`migration ${input.name}: ${b.root}'s code names the backend it moves from, ${at}; change the backend block in the same change`);
463
+ return { object, block: backendBlock(b.from.backend, b.from.config), data: join(input.work, `source-${i}`) };
464
+ }
465
+
466
+ /** A source's state and version id now: an external source read directly, a backend through the root with an override file naming it, in its own data dir. */
467
+ async function readSource(s: Source, run: Planned["run"], exec: BinaryExec, fetchFn: StoreFetch | undefined, override: string): Promise<{ state: StateFile | null; version?: string }> {
468
+ if ("external" in s) {
469
+ const got = await s.external.read();
470
+ if (!got) return { state: null };
471
+ let state: StateFile;
472
+ try {
473
+ state = JSON.parse(got.text) as StateFile;
474
+ } catch {
475
+ throw new MigrationError(`${s.external.location} holds something that is not a state`);
476
+ }
477
+ if (!Array.isArray(state.resources)) state.resources = [];
478
+ return { state, ...(got.version ? { version: got.version } : {}) };
479
+ }
480
+ const at = join(run.dir, override);
481
+ const env = { ...run.env, TF_DATA_DIR: s.data };
482
+ writeFileSync(at, s.block);
483
+ let state: StateFile | null;
484
+ try {
485
+ await initRoot({ ...run, env, exec, reconfigure: true });
486
+ state = await pullState(exec, run.binary, run.root, run.dir, env);
487
+ } finally {
488
+ rmSync(at, { force: true });
489
+ }
490
+ const version = await versionNow(s.object, fetchFn, false);
491
+ return { state, ...(version ? { version } : {}) };
492
+ }
493
+
291
494
  /**
292
495
  * Plan a root against a state file: an override file switches its backend to
293
496
  * that local file, in a data dir of its own, so the root's `.terraform` and
@@ -307,18 +510,56 @@ async function proofPlan(run: Planned["run"], exec: BinaryExec, file: string, da
307
510
  }
308
511
  }
309
512
 
513
+ const stored = (o: MigratedState): o is Exclude<LockableState, { backend: "local" }> & StateObject => !isPulledState(o) && isStored(o);
514
+
310
515
  function migrationEngine(planned: Map<string, Planned>, input: TerraformMigrationInput, exec: BinaryExec, lockOp: LockOperation): StateMigrationEngine {
311
516
  const log = input.log ?? (() => {});
517
+ const override = input.overrideFile ?? DEFAULT_OVERRIDE_FILE;
312
518
  const stores = new Map<string, StateStore>();
313
- for (const [root, p] of planned) if (isStored(p.object)) stores.set(root, stateStore(p.object, p.run.dir, input.fetch));
519
+ for (const [root, p] of planned) if (stored(p.object)) stores.set(root, stateStore(p.object, p.run.dir, input.fetch));
520
+ // A backend move holds the lock of the state it reads too, after the states it writes, and a source workspace's lock before any.
521
+ const sourceStores: { root: string; store: StateStore }[] = [];
522
+ const sourceLocks: SourceLock[] = [];
523
+ for (const [root, p] of planned) {
524
+ if (!p.source) continue;
525
+ if ("external" in p.source) {
526
+ const ext = p.source.external;
527
+ if (ext.lock) sourceLocks.push({ root, location: ext.location, lock: (reason) => ext.lock!(reason) });
528
+ } else if (stored(p.source.object)) sourceStores.push({ root, store: stateStore(p.source.object, p.run.dir, input.fetch) });
529
+ }
314
530
  const pinned = new Map<string, string>();
315
531
  return {
316
532
  async lock() {
317
- const got = await takeLocksWithSources({ sources: [], stores: [...stores].map(([root, store]) => ({ root, store })), now: new Date().toISOString(), op: lockOp, who: input.who ?? `chant@${hostname()}` });
533
+ let got: Awaited<ReturnType<typeof takeLocksWithSources>>;
534
+ try {
535
+ got = await takeLocksWithSources({
536
+ sources: sourceLocks,
537
+ stores: [...[...stores].map(([root, store]) => ({ root, store })), ...sourceStores],
538
+ now: new Date().toISOString(),
539
+ op: lockOp,
540
+ sourceOp: input.sourceLock ?? { operation: lockOp.operation, info: "a state migration is reading this state" },
541
+ who: input.who ?? `chant@${hostname()}`,
542
+ });
543
+ } catch (e) {
544
+ if (e instanceof SourceLockedError) throw new MigrationError(`${e.root}: ${e.reason}`);
545
+ if (e instanceof StateLockedError) {
546
+ const h = e.holder;
547
+ const by = h ? [h.who && `by ${h.who}`, h.operation, h.info, h.created && `since ${h.created}`].filter(Boolean).join(", ") : "";
548
+ throw new MigrationError(`${e.root}: its state is locked (${e.lockLocation}${by ? `: ${by}` : ""}), so nothing was written`);
549
+ }
550
+ throw e;
551
+ }
318
552
  return async () => {
319
553
  for (const l of got.locks.reverse()) await l.release().catch((e: unknown) => log(`${l.root}: the lock could not be released: ${(e as Error).message}`));
554
+ for (const l of got.sources.reverse()) await l.release().catch((e: unknown) => log(`${l.root}: the lock of ${l.location} could not be released: ${(e as Error).message}`));
320
555
  };
321
556
  },
557
+ async source(root): Promise<MigrationStateAt> {
558
+ const p = planned.get(root)!;
559
+ if (!p.source) return { digest: null };
560
+ const read = await readSource(p.source, p.run, exec, input.fetch, override);
561
+ return { ...(read.version ? { version_id: read.version } : {}), digest: read.state ? stateDigest(read.state) : null };
562
+ },
322
563
  async current(root): Promise<MigrationStateAt> {
323
564
  const p = planned.get(root)!;
324
565
  const v = await versionNow(p.object, input.fetch, false);
@@ -337,8 +578,9 @@ function migrationEngine(planned: Map<string, Planned>, input: TerraformMigratio
337
578
  for (const [root, p] of order) {
338
579
  const store = stores.get(root);
339
580
  // A state whose lock keeps the binary out (azurerm's lease) is written under it; the binary writes the others with the lock held for it.
581
+ // Over the empty state a backend wrote at init, of another lineage, the push is forced; under the locks its digest was found still empty.
340
582
  if (store?.write) await store.write(readFileSync(p.file, "utf-8"));
341
- else await pushStateFile({ ...p.run, exec, file: p.file, ...(store ? { lock: false } : {}) });
583
+ else await pushStateFile({ ...p.run, exec, file: p.file, ...(store ? { lock: false } : p.replacesEmpty ? { force: true } : {}) });
342
584
  log(`${root}: wrote its new state`);
343
585
  }
344
586
  },
package/src/state-ops.ts CHANGED
@@ -46,9 +46,9 @@ import { exportStateVersion, StateExportError, type StateExportInput, type State
46
46
  import { parseLockInfo, releaseStateLock, StateUnlockError, type LockInfo, type StateLockSource, type StateUnlockInput, type StateUnlockOutcome } from "@intentius/chant/state-unlock";
47
47
  import { plannerForBinary, terraformChangeSetPart } from "./change-set";
48
48
  import { DEFAULT_LOCK_TIMEOUT, lockTimeoutArgs } from "./lock-timeout";
49
- import { initialisedBackend, isRefused, stateObject, type StateObject } from "./state/backend";
49
+ import { initialisedBackend, isGitLabState, isRefused, stateObject, type StateObject } from "./state/backend";
50
50
  import { lastLines, runBinary, type BinaryExec } from "./state/exec";
51
- import { gitlabState, gitlabStore, shownUrl, type GitLabFetch, type GitLabState } from "./state/gitlab";
51
+ import { gitlabStore, shownUrl, type GitLabFetch } from "./state/gitlab";
52
52
  import type { StoreFetch } from "./state/object-store";
53
53
  import { stateStore } from "./state/store";
54
54
  import { gitlabVersionSource, stateVersionSource } from "./state/versions";
@@ -243,22 +243,20 @@ async function withBackend<T>(r: RootBackendRun, act: (object: RootStateAt, env:
243
243
  }
244
244
  }
245
245
 
246
- /** Where a root's state is: a state object, or a GitLab-managed state (./state/gitlab.ts), which the lock verdict refuses and export and release read. */
247
- type RootStateAt = StateObject | GitLabState;
246
+ /** Where a root's state is: a state object, a GitLab-managed state among them (./state/gitlab.ts, #4793). */
247
+ type RootStateAt = StateObject;
248
248
 
249
- const isGitLab = (o: RootStateAt): o is GitLabState => "address" in o;
249
+ const isGitLab = isGitLabState;
250
250
 
251
251
  /**
252
252
  * The root's state. A read needs no lock file, so an s3 backend that locks
253
253
  * with DynamoDB is read as one that takes the lock file: the refusal is
254
- * about holding its lock, which a read never does. An http backend whose
255
- * address is GitLab's state API is that state.
254
+ * about holding its lock, which a read never does.
256
255
  */
257
256
  function objectOf(dir: string, env: NodeJS.ProcessEnv, binary: string, readOnly: boolean): RootStateAt {
258
257
  const o = stateObject(dir, env, undefined, binary);
259
258
  if (!isRefused(o)) return o;
260
259
  const recorded = initialisedBackend(dir, env);
261
- if (o.backend === "http" && recorded?.type === "http") return gitlabState(recorded.config, env) ?? o;
262
260
  if (!readOnly || o.refused !== "no-lockfile") return o;
263
261
  return stateObject(dir, env, recorded ? { ...recorded, config: { ...recorded.config, use_lockfile: true } } : undefined, binary, { backendConfig: { use_lockfile: "true" } });
264
262
  }