@intentius/chant-lexicon-terraform 0.124.85 → 0.124.87
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ci-setup.d.ts +11 -2
- package/dist/ci-setup.d.ts.map +1 -1
- package/dist/ci-setup.js +11 -6
- package/dist/components/capability-plugin.d.ts +12 -0
- package/dist/components/capability-plugin.d.ts.map +1 -1
- package/dist/components/capability-plugin.js +22 -4
- package/dist/compose.d.ts.map +1 -1
- package/dist/compose.js +1 -1
- package/dist/estate-migration.d.ts +101 -0
- package/dist/estate-migration.d.ts.map +1 -0
- package/dist/estate-migration.js +471 -0
- package/dist/integrity.json +2 -2
- package/dist/manifest.json +1 -1
- package/dist/op/activities/terraform.d.ts +65 -0
- package/dist/op/activities/terraform.d.ts.map +1 -1
- package/dist/op/activities/terraform.js +94 -1
- package/dist/state/backend.d.ts +46 -9
- package/dist/state/backend.d.ts.map +1 -1
- package/dist/state/backend.js +84 -31
- package/dist/state/gitlab.d.ts +1 -1
- package/dist/state/gitlab.d.ts.map +1 -1
- package/dist/state/gitlab.js +27 -14
- package/dist/state/index.d.ts +1 -0
- package/dist/state/index.d.ts.map +1 -1
- package/dist/state/index.js +1 -0
- package/dist/state/s3.d.ts +9 -3
- package/dist/state/s3.d.ts.map +1 -1
- package/dist/state/s3.js +4 -3
- package/dist/state/source.d.ts +92 -0
- package/dist/state/source.d.ts.map +1 -0
- package/dist/state/source.js +254 -0
- package/dist/state/store.d.ts +41 -4
- package/dist/state/store.d.ts.map +1 -1
- package/dist/state/store.js +125 -10
- package/dist/state/versions.d.ts +77 -10
- package/dist/state/versions.d.ts.map +1 -1
- package/dist/state/versions.js +118 -64
- package/dist/state-migration.d.ts +43 -1
- package/dist/state-migration.d.ts.map +1 -1
- package/dist/state-migration.js +251 -20
- package/dist/state-ops.d.ts +2 -3
- package/dist/state-ops.d.ts.map +1 -1
- package/dist/state-ops.js +4 -7
- package/package.json +2 -2
- package/src/ci-setup.test.ts +20 -0
- package/src/ci-setup.ts +11 -6
- package/src/components/capability-plugin.ts +36 -3
- package/src/components/terraform-apply-command-hook.test.ts +157 -0
- package/src/compose.ts +1 -0
- package/src/estate-migration.test.ts +186 -0
- package/src/estate-migration.ts +563 -0
- package/src/op/activities/terraform.ts +153 -3
- package/src/state/backend-verdict.test.ts +4 -3
- package/src/state/backend.test.ts +3 -1
- package/src/state/backend.ts +108 -33
- package/src/state/gitlab.ts +25 -12
- package/src/state/index.ts +1 -0
- package/src/state/s3.ts +5 -4
- package/src/state/source.ts +289 -0
- package/src/state/store.ts +145 -11
- package/src/state/versions.test.ts +365 -0
- package/src/state/versions.ts +148 -68
- package/src/state-migration-backends.test.ts +277 -0
- package/src/state-migration.ts +265 -23
- package/src/state-ops.ts +6 -8
package/src/state-migration.ts
CHANGED
|
@@ -10,6 +10,12 @@
|
|
|
10
10
|
* restores a root's state put back to a version a migration recorded (a
|
|
11
11
|
* revert), refused when the state moved past what that
|
|
12
12
|
* migration left
|
|
13
|
+
* backends a root's state moved to the backend its code names now, from
|
|
14
|
+
* the backend it was in (read through the root with an
|
|
15
|
+
* override file naming that backend, in a data dir of its own),
|
|
16
|
+
* from a TFE-API workspace or from an exported state file
|
|
17
|
+
* (`./state/source.ts`), #4806. The backend the code names must
|
|
18
|
+
* hold no state; the state read is left where it was
|
|
13
19
|
*
|
|
14
20
|
* Planning (planTerraformMigration) initialises each root in a data dir of
|
|
15
21
|
* its own, reads its backend version id and pulls its state, keeps only
|
|
@@ -23,28 +29,44 @@
|
|
|
23
29
|
* a write that stops half way leaves a resource in two states rather than
|
|
24
30
|
* in none. Then each root plans against its real backend.
|
|
25
31
|
*
|
|
32
|
+
* A backend that keeps no versions (pg, kubernetes, consul, an http backend
|
|
33
|
+
* other than GitLab's) takes the general path, for each kind (#4806): the
|
|
34
|
+
* binary reads it with `state pull` and writes it with `state push` under
|
|
35
|
+
* the backend's own lock, which refuses a state of another lineage or an
|
|
36
|
+
* older serial, and the digest binds the state's contents, since there is
|
|
37
|
+
* no version id to bind. A backend move into one replaces the empty state a
|
|
38
|
+
* backend writes at init (kubernetes does) with `-force`; the digest bound
|
|
39
|
+
* that it was empty.
|
|
40
|
+
*
|
|
41
|
+
* A backend move's root names its source (`MigrationRootPlan.source`): the
|
|
42
|
+
* request binds it, under the locks the engine reads it again, and the lock
|
|
43
|
+
* of a workspace it is read from is taken first, through the workspace's
|
|
44
|
+
* API.
|
|
45
|
+
*
|
|
26
46
|
* State contents never leave the work dir. The record holds version ids and
|
|
27
47
|
* digests.
|
|
28
48
|
*
|
|
29
49
|
* Lifted from terragucci's `migrate.ts` (`moveResources`, `addressMatches`,
|
|
30
50
|
* `derivedLineage`, `stateDigest`, `resourceChanges`, `backendBlock`,
|
|
31
|
-
* `proofPlan`, `planMigration`
|
|
32
|
-
* `
|
|
33
|
-
*
|
|
34
|
-
* migration of choudoufu's live estates
|
|
51
|
+
* `proofPlan`, `planMigration`, `applyMigration`, `movedSince`, `takeLocks`,
|
|
52
|
+
* `readSource`, `withBackend`, `isPulled`, `PULLED_BACKENDS`,
|
|
53
|
+
* `backendRefusal`, `parseBackends`, `pulledProblems`, `locationOf`), so
|
|
54
|
+
* terragucci deletes its copy. A migration of choudoufu's live estates is
|
|
55
|
+
* `./estate-migration.ts`.
|
|
35
56
|
*/
|
|
36
57
|
import { createHash } from "node:crypto";
|
|
37
58
|
import { hostname } from "node:os";
|
|
38
59
|
import { join, resolve } from "node:path";
|
|
39
60
|
import { mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
40
|
-
import type { MigrationCheck, MigrationRootPlan, MigrationStateAt, StateMigrationEngine } from "@intentius/chant/state-migration";
|
|
61
|
+
import type { MigrationCheck, MigrationRootPlan, MigrationSource, MigrationStateAt, StateMigrationEngine } from "@intentius/chant/state-migration";
|
|
41
62
|
import { initRoot, planStateCheck, pushStateFile, type RootRun } from "./state-ops";
|
|
42
|
-
import { isRefused, isStored, stateLocation, stateObject, type LockableState } from "./state/backend";
|
|
63
|
+
import { initialisedBackend, isRefused, isStored, readableStateObject, stateLocation, stateObject, type InitialisedBackend, type LockableState, type StateObject } from "./state/backend";
|
|
43
64
|
import { pullState, runBinary, type BinaryExec, type StateFile, type StateResource } from "./state/exec";
|
|
44
|
-
import { takeLocksWithSources, type LockOperation } from "./state/locks";
|
|
65
|
+
import { SourceLockedError, StateLockedError, takeLocksWithSources, type LockOperation, type SourceLock } from "./state/locks";
|
|
45
66
|
import type { StoreFetch } from "./state/object-store";
|
|
67
|
+
import { EXTERNAL_BACKENDS, externalProblems, externalSource, type ExternalSource } from "./state/source";
|
|
46
68
|
import { stateStore, type StateStore } from "./state/store";
|
|
47
|
-
import { stateVersionSource } from "./state/versions";
|
|
69
|
+
import { keepsHistory, NO_HISTORY, recordedAddress, stateVersionSource } from "./state/versions";
|
|
48
70
|
|
|
49
71
|
/** Resources that move from one root's state to another's. */
|
|
50
72
|
export interface Move {
|
|
@@ -64,6 +86,19 @@ export interface Restore {
|
|
|
64
86
|
from_version_id: string | null;
|
|
65
87
|
}
|
|
66
88
|
|
|
89
|
+
/** A root's state moving to the backend its code now names, from the backend it was in (#4806). */
|
|
90
|
+
export interface BackendMove {
|
|
91
|
+
root: string;
|
|
92
|
+
/** Where the state is now: a backend's type and configuration, as a backend block gives them; `remote` or `cloud` for a TFE-API workspace; `file` with `config.path` for an exported state file. */
|
|
93
|
+
from: { backend: string; config: Record<string, unknown> };
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** The backends a migration reads and writes through the binary, under the backend's own lock: those that keep no versions (#4806). */
|
|
97
|
+
export const PULLED_BACKENDS: readonly string[] = Object.keys(NO_HISTORY);
|
|
98
|
+
|
|
99
|
+
/** The backends a migration locks and writes itself. */
|
|
100
|
+
export const MIGRATED_BACKENDS: readonly string[] = ["s3", "gcs", "azurerm", "local", "http"];
|
|
101
|
+
|
|
67
102
|
/** A migration that cannot be planned: an address in no state, a root that is refused. Nothing was written. */
|
|
68
103
|
export class MigrationError extends Error {
|
|
69
104
|
constructor(message: string) {
|
|
@@ -95,6 +130,82 @@ export function derivedLineage(...parts: string[]): string {
|
|
|
95
130
|
|
|
96
131
|
const emptyState = (lineage: string, serial: number, terraformVersion = "1.0.0"): StateFile => ({ version: 4, terraform_version: terraformVersion, serial, lineage, outputs: {}, resources: [], check_results: null });
|
|
97
132
|
|
|
133
|
+
/** A state with no resources and no outputs: what a backend writes at init before anything is applied. */
|
|
134
|
+
const isEmptyState = (s: StateFile): boolean => s.resources.length === 0 && Object.keys(s.outputs ?? {}).length === 0;
|
|
135
|
+
|
|
136
|
+
/** A state the binary reads and writes for a migration, under its backend's own lock: one that keeps no versions (pg, kubernetes, consul, an http backend other than GitLab's). */
|
|
137
|
+
export interface PulledState {
|
|
138
|
+
backend: string;
|
|
139
|
+
pulled: true;
|
|
140
|
+
/** Its address, credentials left out, when the configuration names one. */
|
|
141
|
+
location?: string;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/** The backend as a pulled state, or undefined when it is not one that keeps no versions. */
|
|
145
|
+
export function pulledState(b: InitialisedBackend, env: NodeJS.ProcessEnv = {}): PulledState | undefined {
|
|
146
|
+
if (!PULLED_BACKENDS.includes(b.type) || keepsHistory(b, env) !== false) return undefined;
|
|
147
|
+
const location = recordedAddress(b);
|
|
148
|
+
return { backend: b.type, pulled: true, ...(location ? { location } : {}) };
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/** A state a migration writes or reads: one chant locks, or one the binary reads and writes under its own lock. */
|
|
152
|
+
export type MigratedState = LockableState | PulledState;
|
|
153
|
+
|
|
154
|
+
const isPulledState = (o: MigratedState): o is PulledState => "pulled" in o;
|
|
155
|
+
|
|
156
|
+
/** Where a migrated state is, as the record names it. */
|
|
157
|
+
export const migratedLocation = (o: MigratedState): string | undefined => (isPulledState(o) ? o.location : stateLocation(o));
|
|
158
|
+
|
|
159
|
+
/** Settings of a pulled backend that hold a credential, which the job gives in the backend's own variables instead. */
|
|
160
|
+
const SECRET_KEYS: Record<string, string> = { password: "the backend's password variable", token: "the backend's token variable", client_key: "KUBE_CLIENT_KEY_DATA", access_token: "CONSUL_HTTP_TOKEN" };
|
|
161
|
+
|
|
162
|
+
/** The settings a backend move's `from.config` must give, by backend: where the state is. */
|
|
163
|
+
const FROM_NEEDS: Record<string, string[]> = { s3: ["bucket", "key"], gcs: ["bucket"], azurerm: ["storage_account_name", "container_name", "key"], kubernetes: ["secret_suffix"], consul: ["path"], http: ["address"] };
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* Problems with backend moves as a migration file gives them, one line each,
|
|
167
|
+
* headed `backends[<i>]`: a root that moves twice, a source backend chant
|
|
168
|
+
* does not read, a config that does not say where the state is, a
|
|
169
|
+
* credential written in the repo. None: the moves can be planned.
|
|
170
|
+
*/
|
|
171
|
+
export function backendMoveProblems(moves: readonly BackendMove[]): string[] {
|
|
172
|
+
const p: string[] = [];
|
|
173
|
+
const seen = new Set<string>();
|
|
174
|
+
const kinds = [...MIGRATED_BACKENDS, ...PULLED_BACKENDS.filter((b) => !MIGRATED_BACKENDS.includes(b)), ...EXTERNAL_BACKENDS];
|
|
175
|
+
for (const [i, b] of moves.entries()) {
|
|
176
|
+
const at = `backends[${i}]`;
|
|
177
|
+
if (seen.has(b.root)) p.push(`${at}: ${b.root} moves twice`);
|
|
178
|
+
seen.add(b.root);
|
|
179
|
+
if (!kinds.includes(b.from.backend)) {
|
|
180
|
+
p.push(`${at}.from.backend must be ${kinds.join(", ")}`);
|
|
181
|
+
continue;
|
|
182
|
+
}
|
|
183
|
+
const config = b.from.config;
|
|
184
|
+
if (EXTERNAL_BACKENDS.includes(b.from.backend)) p.push(...externalProblems(at, b.from.backend, config));
|
|
185
|
+
if (PULLED_BACKENDS.includes(b.from.backend)) {
|
|
186
|
+
for (const k of Object.keys(config)) if (k in SECRET_KEYS) p.push(`${at}.from.config.${k}: a credential does not belong in the repo; the job gives it in ${SECRET_KEYS[k]}`);
|
|
187
|
+
const conn = config.conn_str;
|
|
188
|
+
if (b.from.backend === "pg" && typeof conn === "string" && (/^postgres(ql)?:\/\/[^/@]*:[^/@]*@/i.test(conn) || /(^|\s)password\s*=/.test(conn))) p.push(`${at}.from.config.conn_str holds a password, which does not belong in the repo; the job gives it in PGPASSWORD`);
|
|
189
|
+
}
|
|
190
|
+
const needs = FROM_NEEDS[b.from.backend];
|
|
191
|
+
if (needs && needs.some((k) => typeof config[k] !== "string")) p.push(`${at}.from.config must name the ${needs.join(", ").replace(/, ([^,]*)$/, " and $1")} of the state`);
|
|
192
|
+
}
|
|
193
|
+
return p;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/** An HCL value from the JSON of a backend's configuration. */
|
|
197
|
+
function hcl(v: unknown): string {
|
|
198
|
+
if (v && typeof v === "object" && !Array.isArray(v)) return `{ ${Object.entries(v).filter(([, x]) => x !== null && x !== undefined).map(([k, x]) => `${k} = ${hcl(x)}`).join(", ")} }`;
|
|
199
|
+
if (Array.isArray(v)) return `[${v.map(hcl).join(", ")}]`;
|
|
200
|
+
return JSON.stringify(v);
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/** A terraform block whose backend is `type` with `config`, for an override file. */
|
|
204
|
+
export function backendBlock(type: string, config: Record<string, unknown>): string {
|
|
205
|
+
const body = Object.entries(config).filter(([, v]) => v !== null && v !== undefined).map(([k, v]) => ` ${k} = ${hcl(v)}\n`).join("");
|
|
206
|
+
return `terraform {\n backend ${JSON.stringify(type)} {\n${body} }\n}\n`;
|
|
207
|
+
}
|
|
208
|
+
|
|
98
209
|
/**
|
|
99
210
|
* The states after the moves. `before` holds each affected root's state, or
|
|
100
211
|
* null when it has none. A changed state keeps its lineage and gets the next
|
|
@@ -172,14 +283,20 @@ export interface TerraformMigrationInput {
|
|
|
172
283
|
moves?: Move[];
|
|
173
284
|
/** States put back to a version a migration recorded. */
|
|
174
285
|
restores?: Restore[];
|
|
175
|
-
/**
|
|
286
|
+
/** States moved to the backend each root's code names now (#4806). */
|
|
287
|
+
backends?: BackendMove[];
|
|
288
|
+
/** Every root the moves, restores or backend moves name. */
|
|
176
289
|
roots: MigrationRootRun[];
|
|
290
|
+
/** What a `file` source's relative path is read from. Default the working directory. */
|
|
291
|
+
repo?: string;
|
|
177
292
|
/** The work dir: new states, plans and data dirs. The caller removes it. */
|
|
178
293
|
work: string;
|
|
179
294
|
exec?: BinaryExec;
|
|
180
295
|
fetch?: StoreFetch;
|
|
181
296
|
/** What the migration's lock says. */
|
|
182
297
|
lock?: LockOperation;
|
|
298
|
+
/** What the lock of a workspace a backend move reads from says. Default the migration's operation, reading. */
|
|
299
|
+
sourceLock?: LockOperation;
|
|
183
300
|
/** Who the lock names. Default `chant@<host>`. */
|
|
184
301
|
who?: string;
|
|
185
302
|
/** The proof plan's override file. Default {@link DEFAULT_OVERRIDE_FILE}. */
|
|
@@ -193,15 +310,22 @@ export interface TerraformMigrationPlan {
|
|
|
193
310
|
engine: StateMigrationEngine;
|
|
194
311
|
}
|
|
195
312
|
|
|
313
|
+
/** Where a backend move's state is now: a backend read through the root with an override file, or a source read without the binary. */
|
|
314
|
+
type Source = { object: MigratedState; block: string; data: string } | { external: ExternalSource };
|
|
315
|
+
|
|
196
316
|
interface Planned {
|
|
197
317
|
run: MigrationRootRun & { env: NodeJS.ProcessEnv };
|
|
198
|
-
object:
|
|
318
|
+
object: MigratedState;
|
|
199
319
|
file: string;
|
|
200
320
|
gained: number;
|
|
321
|
+
source?: Source;
|
|
322
|
+
/** The state the root's backend holds is the empty one it wrote at init: the push is forced over it. */
|
|
323
|
+
replacesEmpty?: boolean;
|
|
201
324
|
}
|
|
202
325
|
|
|
203
326
|
/** A root's version id now, when its store keeps versions. `pin` lets an azurerm blob with snapshots only take one to name it. */
|
|
204
|
-
async function versionNow(o:
|
|
327
|
+
async function versionNow(o: MigratedState, fetchFn: StoreFetch | undefined, pin: boolean): Promise<string | undefined> {
|
|
328
|
+
if (isPulledState(o)) return undefined;
|
|
205
329
|
const src = stateVersionSource(o, fetchFn, { pin });
|
|
206
330
|
if ("refused" in src) return undefined;
|
|
207
331
|
return (await src.current()).versionId;
|
|
@@ -217,15 +341,20 @@ export async function planTerraformMigration(input: TerraformMigrationInput): Pr
|
|
|
217
341
|
const log = input.log ?? (() => {});
|
|
218
342
|
const lockOp = input.lock ?? DEFAULT_MIGRATE_LOCK;
|
|
219
343
|
const override = input.overrideFile ?? DEFAULT_OVERRIDE_FILE;
|
|
220
|
-
|
|
344
|
+
const kinds = [input.moves?.length && "moves", input.restores?.length && "restores", input.backends?.length && "backends"].filter(Boolean);
|
|
345
|
+
if (kinds.length !== 1) throw new MigrationError(`migration ${input.name}: give it moves, restores or backends, one of the three`);
|
|
346
|
+
if (input.backends) {
|
|
347
|
+
const problems = backendMoveProblems(input.backends);
|
|
348
|
+
if (problems.length > 0) throw new MigrationError(`migration ${input.name}:\n ${problems.join("\n ")}`);
|
|
349
|
+
}
|
|
221
350
|
mkdirSync(input.work, { recursive: true });
|
|
222
|
-
const named = [...new Set(input.moves ? input.moves.flatMap((m) => [m.from, m.to]) : input.restores
|
|
351
|
+
const named = [...new Set(input.moves ? input.moves.flatMap((m) => [m.from, m.to]) : input.restores ? input.restores.map((r) => r.root) : input.backends!.map((b) => b.root))].sort();
|
|
223
352
|
const byName = new Map(input.roots.map((r) => [r.root, r]));
|
|
224
353
|
const missing = named.filter((n) => !byName.has(n));
|
|
225
354
|
if (missing.length > 0) throw new MigrationError(`migration ${input.name}: ${missing.join(", ")} ${missing.length === 1 ? "is" : "are"} not among its roots`);
|
|
226
355
|
|
|
227
356
|
const runs = new Map<string, Planned["run"]>();
|
|
228
|
-
const objects = new Map<string,
|
|
357
|
+
const objects = new Map<string, MigratedState>();
|
|
229
358
|
const versions = new Map<string, string | undefined>();
|
|
230
359
|
const before = new Map<string, StateFile | null>();
|
|
231
360
|
for (const [i, name] of named.entries()) {
|
|
@@ -235,8 +364,10 @@ export async function planTerraformMigration(input: TerraformMigrationInput): Pr
|
|
|
235
364
|
env = { ...env, TF_DATA_DIR: join(input.work, `data-${i}`) };
|
|
236
365
|
await initRoot({ ...r, env, exec, reconfigure: true });
|
|
237
366
|
}
|
|
238
|
-
const
|
|
239
|
-
|
|
367
|
+
const found = stateObject(r.dir, env, undefined, r.binary);
|
|
368
|
+
const recorded = initialisedBackend(r.dir, env);
|
|
369
|
+
const o: MigratedState | undefined = !isRefused(found) ? found : found.refused === "backend" && recorded ? pulledState(recorded, env) : undefined;
|
|
370
|
+
if (!o) throw new MigrationError(`migration ${input.name} cannot run: ${name}: ${(found as { reason: string }).reason}`);
|
|
240
371
|
runs.set(name, { ...r, env });
|
|
241
372
|
objects.set(name, o);
|
|
242
373
|
versions.set(name, await versionNow(o, input.fetch, false));
|
|
@@ -244,12 +375,29 @@ export async function planTerraformMigration(input: TerraformMigrationInput): Pr
|
|
|
244
375
|
}
|
|
245
376
|
|
|
246
377
|
const after = new Map<string, StateFile>();
|
|
378
|
+
const sources = new Map<string, { source: Source; at: MigrationSource }>();
|
|
379
|
+
const replaces = new Set<string>();
|
|
247
380
|
if (input.moves) {
|
|
248
381
|
for (const [root, s] of moveResources(input.name, input.moves, before)) after.set(root, s);
|
|
382
|
+
} else if (input.backends) {
|
|
383
|
+
for (const [i, b] of input.backends.entries()) {
|
|
384
|
+
const run = runs.get(b.root)!;
|
|
385
|
+
const target = objects.get(b.root)!;
|
|
386
|
+
const source = sourceOf(input, b, run, target, i);
|
|
387
|
+
// A backend that writes an empty state of its own lineage at init (kubernetes) has it replaced; the digest binds that it was empty.
|
|
388
|
+
const held = before.get(b.root);
|
|
389
|
+
if (held && isPulledState(target) && isEmptyState(held)) replaces.add(b.root);
|
|
390
|
+
else if (held) throw new MigrationError(`migration ${input.name}: the backend ${b.root}'s code names, ${migratedLocation(target) ?? target.backend}, already holds a state, so a move would overwrite it`);
|
|
391
|
+
const read = await readSource(source, run, exec, input.fetch, override);
|
|
392
|
+
const where = "external" in source ? source.external.location : (migratedLocation(source.object) ?? b.from.backend);
|
|
393
|
+
if (!read.state) throw new MigrationError(`migration ${input.name}: ${where} holds no state of ${b.root} to move`);
|
|
394
|
+
sources.set(b.root, { source, at: { ...(where ? { location: where } : {}), ...(read.version ? { version_id: read.version } : {}), digest: stateDigest(read.state) } });
|
|
395
|
+
after.set(b.root, read.state);
|
|
396
|
+
}
|
|
249
397
|
} else {
|
|
250
398
|
for (const r of input.restores!) {
|
|
251
399
|
const o = objects.get(r.root)!;
|
|
252
|
-
const location =
|
|
400
|
+
const location = migratedLocation(o);
|
|
253
401
|
if (location !== r.location) throw new MigrationError(`migration ${input.name}: ${r.root}'s state is at ${location}, and the revert restores ${r.location}`);
|
|
254
402
|
const v = versions.get(r.root);
|
|
255
403
|
if ((r.from_version_id ?? undefined) !== v) throw new MigrationError(`migration ${input.name}: ${r.root}'s state is at version ${v ?? "none"}, and the migration it reverts left ${r.from_version_id ?? "none"}; it moved since, so putting the older version back would undo that change too`);
|
|
@@ -260,6 +408,7 @@ export async function planTerraformMigration(input: TerraformMigrationInput): Pr
|
|
|
260
408
|
after.set(r.root, emptyState(lineage, serial, now?.terraform_version));
|
|
261
409
|
continue;
|
|
262
410
|
}
|
|
411
|
+
if (isPulledState(o)) throw new MigrationError(`migration ${input.name}: ${r.root}'s ${o.backend} backend keeps no versions to put back`);
|
|
263
412
|
const src = stateVersionSource(o, input.fetch);
|
|
264
413
|
if ("refused" in src) throw new MigrationError(`migration ${input.name}: ${r.root}: ${src.refused}`);
|
|
265
414
|
const text = await src.readVersion(r.version_id);
|
|
@@ -277,17 +426,71 @@ export async function planTerraformMigration(input: TerraformMigrationInput): Pr
|
|
|
277
426
|
const run = runs.get(name)!;
|
|
278
427
|
const file = join(input.work, `${i}.tfstate`);
|
|
279
428
|
writeFileSync(file, `${JSON.stringify(state, null, 2)}\n`);
|
|
280
|
-
|
|
429
|
+
const src = sources.get(name);
|
|
430
|
+
planned.set(name, { run, object: objects.get(name)!, file, gained: state.resources.length - (before.get(name)?.resources.length ?? 0), ...(src ? { source: src.source } : {}), ...(replaces.has(name) ? { replacesEmpty: true } : {}) });
|
|
281
431
|
const proof = await proofPlan(run, exec, file, join(input.work, `proof-${i}`), override);
|
|
282
432
|
log(`${name}: ${proof.changes.length === 0 ? "no changes against its new state" : `${proof.changes.length} change${proof.changes.length === 1 ? "" : "s"} against its new state: ${proof.changes.join(", ")}`}`);
|
|
283
433
|
const was = before.get(name) ?? null;
|
|
284
434
|
const v = versions.get(name);
|
|
285
|
-
const location =
|
|
286
|
-
roots.push({ root: name, ...(location ? { location } : {}), before: { ...(v ? { version_id: v } : {}), digest: was ? stateDigest(was) : null }, after: { digest: stateDigest(state) }, proof });
|
|
435
|
+
const location = migratedLocation(objects.get(name)!);
|
|
436
|
+
roots.push({ root: name, ...(location ? { location } : {}), before: { ...(v ? { version_id: v } : {}), digest: was ? stateDigest(was) : null }, after: { digest: stateDigest(state) }, proof, ...(src ? { source: src.at } : {}) });
|
|
287
437
|
}
|
|
288
438
|
return { roots, engine: migrationEngine(planned, input, exec, lockOp) };
|
|
289
439
|
}
|
|
290
440
|
|
|
441
|
+
/**
|
|
442
|
+
* Where a backend move reads its state from: an external source (a TFE-API
|
|
443
|
+
* workspace, a state file), or the backend `from` names, read through the
|
|
444
|
+
* root. Throws MigrationError for a backend chant does not read, an s3
|
|
445
|
+
* backend that takes no lock file, and a source the root's code names
|
|
446
|
+
* already.
|
|
447
|
+
*/
|
|
448
|
+
function sourceOf(input: TerraformMigrationInput, b: BackendMove, run: Planned["run"], target: MigratedState, i: number): Source {
|
|
449
|
+
const external = externalSource(input.repo ?? process.cwd(), b.from.backend, b.from.config, run.env, input.fetch);
|
|
450
|
+
if (external) return { external };
|
|
451
|
+
const from = { type: b.from.backend, config: b.from.config };
|
|
452
|
+
const pulled = pulledState(from, run.env);
|
|
453
|
+
let object: MigratedState;
|
|
454
|
+
if (pulled) object = pulled;
|
|
455
|
+
else {
|
|
456
|
+
const o = readableStateObject(run.dir, run.env, from);
|
|
457
|
+
if (isRefused(o)) throw new MigrationError(`migration ${input.name} cannot run: the backend ${b.root} moves from: ${o.reason}`);
|
|
458
|
+
if (o.backend === "s3" && !o.lockfile) throw new MigrationError(`migration ${input.name} cannot run: the backend ${b.root} moves from: its s3 backend takes no lock file (use_lockfile = true), so a migration could not hold its lock`);
|
|
459
|
+
object = o;
|
|
460
|
+
}
|
|
461
|
+
const at = migratedLocation(object);
|
|
462
|
+
if (at !== undefined && at === migratedLocation(target)) throw new MigrationError(`migration ${input.name}: ${b.root}'s code names the backend it moves from, ${at}; change the backend block in the same change`);
|
|
463
|
+
return { object, block: backendBlock(b.from.backend, b.from.config), data: join(input.work, `source-${i}`) };
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
/** A source's state and version id now: an external source read directly, a backend through the root with an override file naming it, in its own data dir. */
|
|
467
|
+
async function readSource(s: Source, run: Planned["run"], exec: BinaryExec, fetchFn: StoreFetch | undefined, override: string): Promise<{ state: StateFile | null; version?: string }> {
|
|
468
|
+
if ("external" in s) {
|
|
469
|
+
const got = await s.external.read();
|
|
470
|
+
if (!got) return { state: null };
|
|
471
|
+
let state: StateFile;
|
|
472
|
+
try {
|
|
473
|
+
state = JSON.parse(got.text) as StateFile;
|
|
474
|
+
} catch {
|
|
475
|
+
throw new MigrationError(`${s.external.location} holds something that is not a state`);
|
|
476
|
+
}
|
|
477
|
+
if (!Array.isArray(state.resources)) state.resources = [];
|
|
478
|
+
return { state, ...(got.version ? { version: got.version } : {}) };
|
|
479
|
+
}
|
|
480
|
+
const at = join(run.dir, override);
|
|
481
|
+
const env = { ...run.env, TF_DATA_DIR: s.data };
|
|
482
|
+
writeFileSync(at, s.block);
|
|
483
|
+
let state: StateFile | null;
|
|
484
|
+
try {
|
|
485
|
+
await initRoot({ ...run, env, exec, reconfigure: true });
|
|
486
|
+
state = await pullState(exec, run.binary, run.root, run.dir, env);
|
|
487
|
+
} finally {
|
|
488
|
+
rmSync(at, { force: true });
|
|
489
|
+
}
|
|
490
|
+
const version = await versionNow(s.object, fetchFn, false);
|
|
491
|
+
return { state, ...(version ? { version } : {}) };
|
|
492
|
+
}
|
|
493
|
+
|
|
291
494
|
/**
|
|
292
495
|
* Plan a root against a state file: an override file switches its backend to
|
|
293
496
|
* that local file, in a data dir of its own, so the root's `.terraform` and
|
|
@@ -307,18 +510,56 @@ async function proofPlan(run: Planned["run"], exec: BinaryExec, file: string, da
|
|
|
307
510
|
}
|
|
308
511
|
}
|
|
309
512
|
|
|
513
|
+
const stored = (o: MigratedState): o is Exclude<LockableState, { backend: "local" }> & StateObject => !isPulledState(o) && isStored(o);
|
|
514
|
+
|
|
310
515
|
function migrationEngine(planned: Map<string, Planned>, input: TerraformMigrationInput, exec: BinaryExec, lockOp: LockOperation): StateMigrationEngine {
|
|
311
516
|
const log = input.log ?? (() => {});
|
|
517
|
+
const override = input.overrideFile ?? DEFAULT_OVERRIDE_FILE;
|
|
312
518
|
const stores = new Map<string, StateStore>();
|
|
313
|
-
for (const [root, p] of planned) if (
|
|
519
|
+
for (const [root, p] of planned) if (stored(p.object)) stores.set(root, stateStore(p.object, p.run.dir, input.fetch));
|
|
520
|
+
// A backend move holds the lock of the state it reads too, after the states it writes, and a source workspace's lock before any.
|
|
521
|
+
const sourceStores: { root: string; store: StateStore }[] = [];
|
|
522
|
+
const sourceLocks: SourceLock[] = [];
|
|
523
|
+
for (const [root, p] of planned) {
|
|
524
|
+
if (!p.source) continue;
|
|
525
|
+
if ("external" in p.source) {
|
|
526
|
+
const ext = p.source.external;
|
|
527
|
+
if (ext.lock) sourceLocks.push({ root, location: ext.location, lock: (reason) => ext.lock!(reason) });
|
|
528
|
+
} else if (stored(p.source.object)) sourceStores.push({ root, store: stateStore(p.source.object, p.run.dir, input.fetch) });
|
|
529
|
+
}
|
|
314
530
|
const pinned = new Map<string, string>();
|
|
315
531
|
return {
|
|
316
532
|
async lock() {
|
|
317
|
-
|
|
533
|
+
let got: Awaited<ReturnType<typeof takeLocksWithSources>>;
|
|
534
|
+
try {
|
|
535
|
+
got = await takeLocksWithSources({
|
|
536
|
+
sources: sourceLocks,
|
|
537
|
+
stores: [...[...stores].map(([root, store]) => ({ root, store })), ...sourceStores],
|
|
538
|
+
now: new Date().toISOString(),
|
|
539
|
+
op: lockOp,
|
|
540
|
+
sourceOp: input.sourceLock ?? { operation: lockOp.operation, info: "a state migration is reading this state" },
|
|
541
|
+
who: input.who ?? `chant@${hostname()}`,
|
|
542
|
+
});
|
|
543
|
+
} catch (e) {
|
|
544
|
+
if (e instanceof SourceLockedError) throw new MigrationError(`${e.root}: ${e.reason}`);
|
|
545
|
+
if (e instanceof StateLockedError) {
|
|
546
|
+
const h = e.holder;
|
|
547
|
+
const by = h ? [h.who && `by ${h.who}`, h.operation, h.info, h.created && `since ${h.created}`].filter(Boolean).join(", ") : "";
|
|
548
|
+
throw new MigrationError(`${e.root}: its state is locked (${e.lockLocation}${by ? `: ${by}` : ""}), so nothing was written`);
|
|
549
|
+
}
|
|
550
|
+
throw e;
|
|
551
|
+
}
|
|
318
552
|
return async () => {
|
|
319
553
|
for (const l of got.locks.reverse()) await l.release().catch((e: unknown) => log(`${l.root}: the lock could not be released: ${(e as Error).message}`));
|
|
554
|
+
for (const l of got.sources.reverse()) await l.release().catch((e: unknown) => log(`${l.root}: the lock of ${l.location} could not be released: ${(e as Error).message}`));
|
|
320
555
|
};
|
|
321
556
|
},
|
|
557
|
+
async source(root): Promise<MigrationStateAt> {
|
|
558
|
+
const p = planned.get(root)!;
|
|
559
|
+
if (!p.source) return { digest: null };
|
|
560
|
+
const read = await readSource(p.source, p.run, exec, input.fetch, override);
|
|
561
|
+
return { ...(read.version ? { version_id: read.version } : {}), digest: read.state ? stateDigest(read.state) : null };
|
|
562
|
+
},
|
|
322
563
|
async current(root): Promise<MigrationStateAt> {
|
|
323
564
|
const p = planned.get(root)!;
|
|
324
565
|
const v = await versionNow(p.object, input.fetch, false);
|
|
@@ -337,8 +578,9 @@ function migrationEngine(planned: Map<string, Planned>, input: TerraformMigratio
|
|
|
337
578
|
for (const [root, p] of order) {
|
|
338
579
|
const store = stores.get(root);
|
|
339
580
|
// A state whose lock keeps the binary out (azurerm's lease) is written under it; the binary writes the others with the lock held for it.
|
|
581
|
+
// Over the empty state a backend wrote at init, of another lineage, the push is forced; under the locks its digest was found still empty.
|
|
340
582
|
if (store?.write) await store.write(readFileSync(p.file, "utf-8"));
|
|
341
|
-
else await pushStateFile({ ...p.run, exec, file: p.file, ...(store ? { lock: false } : {}) });
|
|
583
|
+
else await pushStateFile({ ...p.run, exec, file: p.file, ...(store ? { lock: false } : p.replacesEmpty ? { force: true } : {}) });
|
|
342
584
|
log(`${root}: wrote its new state`);
|
|
343
585
|
}
|
|
344
586
|
},
|
package/src/state-ops.ts
CHANGED
|
@@ -46,9 +46,9 @@ import { exportStateVersion, StateExportError, type StateExportInput, type State
|
|
|
46
46
|
import { parseLockInfo, releaseStateLock, StateUnlockError, type LockInfo, type StateLockSource, type StateUnlockInput, type StateUnlockOutcome } from "@intentius/chant/state-unlock";
|
|
47
47
|
import { plannerForBinary, terraformChangeSetPart } from "./change-set";
|
|
48
48
|
import { DEFAULT_LOCK_TIMEOUT, lockTimeoutArgs } from "./lock-timeout";
|
|
49
|
-
import { initialisedBackend, isRefused, stateObject, type StateObject } from "./state/backend";
|
|
49
|
+
import { initialisedBackend, isGitLabState, isRefused, stateObject, type StateObject } from "./state/backend";
|
|
50
50
|
import { lastLines, runBinary, type BinaryExec } from "./state/exec";
|
|
51
|
-
import {
|
|
51
|
+
import { gitlabStore, shownUrl, type GitLabFetch } from "./state/gitlab";
|
|
52
52
|
import type { StoreFetch } from "./state/object-store";
|
|
53
53
|
import { stateStore } from "./state/store";
|
|
54
54
|
import { gitlabVersionSource, stateVersionSource } from "./state/versions";
|
|
@@ -243,22 +243,20 @@ async function withBackend<T>(r: RootBackendRun, act: (object: RootStateAt, env:
|
|
|
243
243
|
}
|
|
244
244
|
}
|
|
245
245
|
|
|
246
|
-
/** Where a root's state is: a state object,
|
|
247
|
-
type RootStateAt = StateObject
|
|
246
|
+
/** Where a root's state is: a state object, a GitLab-managed state among them (./state/gitlab.ts, #4793). */
|
|
247
|
+
type RootStateAt = StateObject;
|
|
248
248
|
|
|
249
|
-
const isGitLab =
|
|
249
|
+
const isGitLab = isGitLabState;
|
|
250
250
|
|
|
251
251
|
/**
|
|
252
252
|
* The root's state. A read needs no lock file, so an s3 backend that locks
|
|
253
253
|
* with DynamoDB is read as one that takes the lock file: the refusal is
|
|
254
|
-
* about holding its lock, which a read never does.
|
|
255
|
-
* address is GitLab's state API is that state.
|
|
254
|
+
* about holding its lock, which a read never does.
|
|
256
255
|
*/
|
|
257
256
|
function objectOf(dir: string, env: NodeJS.ProcessEnv, binary: string, readOnly: boolean): RootStateAt {
|
|
258
257
|
const o = stateObject(dir, env, undefined, binary);
|
|
259
258
|
if (!isRefused(o)) return o;
|
|
260
259
|
const recorded = initialisedBackend(dir, env);
|
|
261
|
-
if (o.backend === "http" && recorded?.type === "http") return gitlabState(recorded.config, env) ?? o;
|
|
262
260
|
if (!readOnly || o.refused !== "no-lockfile") return o;
|
|
263
261
|
return stateObject(dir, env, recorded ? { ...recorded, config: { ...recorded.config, use_lockfile: true } } : undefined, binary, { backendConfig: { use_lockfile: "true" } });
|
|
264
262
|
}
|