@caelo-cms/provisioning 0.11.3 → 0.11.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-db.js +46 -37
- package/dist/audit-job-setup.js +2 -2
- package/dist/audit-job-setup.js.map +1 -1
- package/dist/capture-job-setup.d.ts +46 -0
- package/dist/capture-job-setup.d.ts.map +1 -0
- package/dist/capture-job-setup.js +288 -0
- package/dist/capture-job-setup.js.map +1 -0
- package/dist/gcp-names.d.ts +17 -1
- package/dist/gcp-names.d.ts.map +1 -1
- package/dist/gcp-names.js +24 -0
- package/dist/gcp-names.js.map +1 -1
- package/dist/infra-contract.d.ts +1 -1
- package/dist/infra-contract.d.ts.map +1 -1
- package/dist/infra-contract.js +11 -1
- package/dist/infra-contract.js.map +1 -1
- package/dist/release-contract.d.ts.map +1 -1
- package/dist/release-contract.js +2 -0
- package/dist/release-contract.js.map +1 -1
- package/dist/stack-contract.d.ts +9 -0
- package/dist/stack-contract.d.ts.map +1 -1
- package/dist/stack-contract.js +16 -1
- package/dist/stack-contract.js.map +1 -1
- package/dist/upgrade/admin-image-jobs.d.ts +47 -0
- package/dist/upgrade/admin-image-jobs.d.ts.map +1 -0
- package/dist/upgrade/{audit-job-roll.js → admin-image-jobs.js} +19 -8
- package/dist/upgrade/admin-image-jobs.js.map +1 -0
- package/dist/upgrade/app-roll.d.ts.map +1 -1
- package/dist/upgrade/app-roll.js +31 -28
- package/dist/upgrade/app-roll.js.map +1 -1
- package/dist/upgrade/converge.d.ts.map +1 -1
- package/dist/upgrade/converge.js +31 -1
- package/dist/upgrade/converge.js.map +1 -1
- package/dist/upgrader-iam-preflight.d.ts.map +1 -1
- package/dist/upgrader-iam-preflight.js +8 -2
- package/dist/upgrader-iam-preflight.js.map +1 -1
- package/dist/upgrader-setup.d.ts.map +1 -1
- package/dist/upgrader-setup.js +9 -3
- package/dist/upgrader-setup.js.map +1 -1
- package/dist/wizards/gcp-cost.d.ts.map +1 -1
- package/dist/wizards/gcp-cost.js +5 -0
- package/dist/wizards/gcp-cost.js.map +1 -1
- package/dist/wizards/gcp.d.ts.map +1 -1
- package/dist/wizards/gcp.js +30 -0
- package/dist/wizards/gcp.js.map +1 -1
- package/package.json +2 -2
- package/src/audit-job-setup.ts +2 -2
- package/src/capture-job-setup.test.ts +213 -0
- package/src/capture-job-setup.ts +322 -0
- package/src/gcp-names.ts +29 -1
- package/src/infra-contract.test.ts +4 -3
- package/src/infra-contract.ts +11 -1
- package/src/release-contract.ts +2 -0
- package/src/stack-contract.test.ts +6 -0
- package/src/stack-contract.ts +17 -0
- package/src/stack-converge.test.ts +1 -1
- package/src/upgrade/{audit-job-roll.ts → admin-image-jobs.ts} +46 -16
- package/src/upgrade/app-roll.ts +34 -31
- package/src/upgrade/converge.ts +40 -0
- package/src/upgrade/upgrade.test.ts +40 -2
- package/src/upgrader-iam-preflight.test.ts +21 -0
- package/src/upgrader-iam-preflight.ts +8 -1
- package/src/upgrader-job.test.ts +1 -0
- package/src/upgrader-setup.test.ts +1 -0
- package/src/upgrader-setup.ts +9 -2
- package/src/wizards/gcp-cost.ts +6 -0
- package/src/wizards/gcp.ts +36 -0
- package/stacks/aws/index.ts +9 -0
- package/stacks/azure/index.ts +9 -0
- package/stacks/gcp/index.ts +13 -0
- package/stacks/gcp-firebase/index.ts +13 -0
- package/dist/upgrade/audit-job-roll.d.ts +0 -38
- package/dist/upgrade/audit-job-roll.d.ts.map +0 -1
- package/dist/upgrade/audit-job-roll.js.map +0 -1
|
@@ -0,0 +1,322 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MPL-2.0
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* #1217 — set up (and converge) the capture job on a Google Cloud install
|
|
5
|
+
* (`gcp`, `gcp-firebase`): the short-lived Cloud Run job that runs every
|
|
6
|
+
* server-side browser render (screenshot_page's server path, the external
|
|
7
|
+
* page tools, the site-import crawl), one execution per render. Run by the
|
|
8
|
+
* wizard after `pulumi up` and by `cms-provision upgrade`'s converge, both as
|
|
9
|
+
* the operator's own gcloud identity. App-roll moves the job's image with the
|
|
10
|
+
* admin's (upgrade/admin-image-jobs.ts), because the job runs the admin image
|
|
11
|
+
* (which bundles Chromium, #428).
|
|
12
|
+
*
|
|
13
|
+
* Why a separate job at all (CLAUDE.md §11.B Tier 5, second exception after
|
|
14
|
+
* the audit job): a headless Chromium costs 300–700 MB per render, and inside
|
|
15
|
+
* the 2 GiB admin it was the most plausible cause of the OOM kill of
|
|
16
|
+
* 2026-10-28 (#1209). In a job it costs the admin nothing.
|
|
17
|
+
*
|
|
18
|
+
* Why the CLI owns it and not the stacks: the same as audit-job-setup.ts —
|
|
19
|
+
* `upgrade` never runs Pulumi.
|
|
20
|
+
*
|
|
21
|
+
* Trust boundary. The job renders untrusted pages (third-party sites, module
|
|
22
|
+
* scripts in a preview), so its identity, `caelo-<env>-capture`, holds:
|
|
23
|
+
*
|
|
24
|
+
* - `roles/storage.objectCreator` on the capture bucket, conditioned to the
|
|
25
|
+
* `out/` prefix — create results only: it cannot read, overwrite or list
|
|
26
|
+
* anything, nor write where the admin puts inputs;
|
|
27
|
+
* - `roles/logging.logWriter`.
|
|
28
|
+
*
|
|
29
|
+
* Nothing else: no read right anywhere (its input and the preview's media
|
|
30
|
+
* arrive as V4 signed URLs the admin creates per capture, valid 5 minutes,
|
|
31
|
+
* deleted with the capture), no Secret Manager, no Cloud SQL, no VPC egress,
|
|
32
|
+
* and no IAP access to the admin — a browser exploit in the job reaches
|
|
33
|
+
* nothing but the internet it already renders.
|
|
34
|
+
*
|
|
35
|
+
* The admin's run SA may run the job WITH overrides (it passes the order),
|
|
36
|
+
* read its executions, use the capture bucket (`roles/storage.objectUser`:
|
|
37
|
+
* write inputs, read results, delete both) and sign URLs as itself
|
|
38
|
+
* (`roles/iam.serviceAccountTokenCreator` on its own account only: Cloud Run
|
|
39
|
+
* holds no key, so V4 signing goes through IAM signBlob; the admin can mint
|
|
40
|
+
* its own tokens through the metadata server already, so this widens
|
|
41
|
+
* nothing). {@link CAPTURE_FORBIDDEN_ROLES} names the job's line; the tests
|
|
42
|
+
* assert no grant crosses it.
|
|
43
|
+
*/
|
|
44
|
+
|
|
45
|
+
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
|
46
|
+
import { tmpdir } from "node:os";
|
|
47
|
+
import { join } from "node:path";
|
|
48
|
+
import { gcloud as defaultGcloud, type GcloudResult } from "./gcloud.js";
|
|
49
|
+
import { type GcloudRunner, realSleep, runWithRetry, type Sleep } from "./gcloud-retry.js";
|
|
50
|
+
import {
|
|
51
|
+
captureJobName,
|
|
52
|
+
captureServiceAccountEmail,
|
|
53
|
+
captureServiceAccountId,
|
|
54
|
+
gcpBucketName,
|
|
55
|
+
runServiceAccountEmail,
|
|
56
|
+
} from "./gcp-names.js";
|
|
57
|
+
import type { GcpProvider } from "./stack-contract.js";
|
|
58
|
+
|
|
59
|
+
/** Where the admin image ships the task's entry point. */
|
|
60
|
+
export const CAPTURE_TASK_SCRIPT = "/app/packages/admin-core/src/capture/capture-task.ts";
|
|
61
|
+
|
|
62
|
+
/** Per execution: container start, browser start and one render. */
|
|
63
|
+
const TASK_TIMEOUT = "3m";
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Roles the capture job's identity must never hold anywhere. Prefix matches
|
|
67
|
+
* (`roles/secretmanager.` = every Secret Manager role). Unlike the audit job
|
|
68
|
+
* it reads nothing, so every storage read role is on the list too.
|
|
69
|
+
*/
|
|
70
|
+
export const CAPTURE_FORBIDDEN_ROLES = [
|
|
71
|
+
"roles/owner",
|
|
72
|
+
"roles/editor",
|
|
73
|
+
"roles/viewer",
|
|
74
|
+
"roles/run.",
|
|
75
|
+
"roles/iam.",
|
|
76
|
+
"roles/iap.",
|
|
77
|
+
"roles/secretmanager.",
|
|
78
|
+
"roles/cloudsql.",
|
|
79
|
+
"roles/compute.",
|
|
80
|
+
"roles/resourcemanager.",
|
|
81
|
+
"roles/storage.admin",
|
|
82
|
+
"roles/storage.objectAdmin",
|
|
83
|
+
"roles/storage.objectUser",
|
|
84
|
+
"roles/storage.objectViewer",
|
|
85
|
+
"roles/storage.legacy",
|
|
86
|
+
] as const;
|
|
87
|
+
|
|
88
|
+
/** Inputs and results are deleted by the admin; the lifecycle catches leftovers. */
|
|
89
|
+
export const CAPTURE_BUCKET_LIFECYCLE = {
|
|
90
|
+
rule: [{ action: { type: "Delete" }, condition: { age: 1 } }],
|
|
91
|
+
};
|
|
92
|
+
|
|
93
|
+
/** The IAM condition that confines the job's create right to `out/`. */
|
|
94
|
+
export function captureResultsCondition(bucket: string): string {
|
|
95
|
+
return `expression=resource.name.startsWith("projects/_/buckets/${bucket}/objects/out/"),title=capture-results-only,description=The capture job creates its results under out/ and nothing else`;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
export interface CaptureJobTarget {
|
|
99
|
+
readonly provider: GcpProvider;
|
|
100
|
+
readonly projectId: string;
|
|
101
|
+
readonly region: string;
|
|
102
|
+
/** Pulumi stack name (`production`). */
|
|
103
|
+
readonly env: string;
|
|
104
|
+
/** The admin image the job runs, digest-pinned (`…/admin@sha256:…`). */
|
|
105
|
+
readonly adminImageRef: string;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
export type CaptureJobResult =
|
|
109
|
+
| { readonly ok: true; readonly done: string[] }
|
|
110
|
+
| { readonly ok: false; readonly done: string[]; readonly error: string };
|
|
111
|
+
|
|
112
|
+
/** Ensure the capture job and everything it needs. Never throws for a gcloud failure. */
|
|
113
|
+
export async function ensureCaptureJob(
|
|
114
|
+
target: CaptureJobTarget,
|
|
115
|
+
deps: { run?: GcloudRunner; sleep?: Sleep } = {},
|
|
116
|
+
): Promise<CaptureJobResult> {
|
|
117
|
+
const run = deps.run ?? defaultGcloud;
|
|
118
|
+
const sleep = deps.sleep ?? realSleep;
|
|
119
|
+
const retry = (args: string[]) => runWithRetry(run, sleep, args);
|
|
120
|
+
const { projectId, region, env, provider } = target;
|
|
121
|
+
const project = `--project=${projectId}`;
|
|
122
|
+
const sa = captureServiceAccountEmail(projectId, env);
|
|
123
|
+
const member = `serviceAccount:${sa}`;
|
|
124
|
+
const runSa = runServiceAccountEmail(projectId, env);
|
|
125
|
+
const runSaMember = `serviceAccount:${runSa}`;
|
|
126
|
+
const job = captureJobName(env);
|
|
127
|
+
const bucket = gcpBucketName(projectId, env, "capture");
|
|
128
|
+
const bucketUrl = `gs://${bucket}`;
|
|
129
|
+
const done: string[] = [];
|
|
130
|
+
const fail = (step: string, r: GcloudResult): CaptureJobResult => ({
|
|
131
|
+
ok: false,
|
|
132
|
+
done,
|
|
133
|
+
error: `${step}: ${r.stderr.trim()}`,
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
if (!/@sha256:[0-9a-f]{64}$/.test(target.adminImageRef)) {
|
|
137
|
+
return {
|
|
138
|
+
ok: false,
|
|
139
|
+
done,
|
|
140
|
+
error: `the capture job's image must be pinned by digest, got ${target.adminImageRef}`,
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
// 1. The job's own service account.
|
|
145
|
+
const describe = await run([
|
|
146
|
+
"iam",
|
|
147
|
+
"service-accounts",
|
|
148
|
+
"describe",
|
|
149
|
+
sa,
|
|
150
|
+
project,
|
|
151
|
+
"--format=value(email)",
|
|
152
|
+
]);
|
|
153
|
+
if (!describe.ok) {
|
|
154
|
+
const create = await run([
|
|
155
|
+
"iam",
|
|
156
|
+
"service-accounts",
|
|
157
|
+
"create",
|
|
158
|
+
captureServiceAccountId(env),
|
|
159
|
+
"--display-name=Caelo capture",
|
|
160
|
+
"--description=Runs server-side browser renders (screenshots); creates capture results, nothing else.",
|
|
161
|
+
project,
|
|
162
|
+
]);
|
|
163
|
+
if (!create.ok && !/already exists/i.test(create.stderr)) return fail(`create ${sa}`, create);
|
|
164
|
+
}
|
|
165
|
+
done.push(`service account ${sa}`);
|
|
166
|
+
|
|
167
|
+
// 2. The capture bucket: private, everything deleted after a day.
|
|
168
|
+
const owned = await run(["storage", "buckets", "list", project, "--format=value(name)"]);
|
|
169
|
+
if (!owned.ok) return fail(`list the buckets of project ${projectId}`, owned);
|
|
170
|
+
const names = owned.stdout.split("\n").map((l) =>
|
|
171
|
+
l
|
|
172
|
+
.trim()
|
|
173
|
+
.replace(/^gs:\/\//, "")
|
|
174
|
+
.replace(/\/$/, ""),
|
|
175
|
+
);
|
|
176
|
+
if (!names.includes(bucket)) {
|
|
177
|
+
const created = await run([
|
|
178
|
+
"storage",
|
|
179
|
+
"buckets",
|
|
180
|
+
"create",
|
|
181
|
+
bucketUrl,
|
|
182
|
+
project,
|
|
183
|
+
`--location=${region}`,
|
|
184
|
+
"--default-storage-class=STANDARD",
|
|
185
|
+
"--uniform-bucket-level-access",
|
|
186
|
+
"--public-access-prevention",
|
|
187
|
+
"--soft-delete-duration=0",
|
|
188
|
+
]);
|
|
189
|
+
if (!created.ok) {
|
|
190
|
+
return {
|
|
191
|
+
ok: false,
|
|
192
|
+
done,
|
|
193
|
+
error: /already exists|409/i.test(created.stderr)
|
|
194
|
+
? `${bucketUrl} exists but does not belong to project ${projectId}; delete or rename that bucket, this install needs the name`
|
|
195
|
+
: `create ${bucketUrl}: ${created.stderr.trim()}`,
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
// A private directory (mkdtemp, mode 700): no other local user can swap
|
|
200
|
+
// the file between write and read.
|
|
201
|
+
const dir = mkdtempSync(join(tmpdir(), "caelo-capture-"));
|
|
202
|
+
try {
|
|
203
|
+
const file = join(dir, "lifecycle.json");
|
|
204
|
+
writeFileSync(file, JSON.stringify(CAPTURE_BUCKET_LIFECYCLE), { mode: 0o600 });
|
|
205
|
+
const lifecycle = await run([
|
|
206
|
+
"storage",
|
|
207
|
+
"buckets",
|
|
208
|
+
"update",
|
|
209
|
+
bucketUrl,
|
|
210
|
+
project,
|
|
211
|
+
`--lifecycle-file=${file}`,
|
|
212
|
+
]);
|
|
213
|
+
if (!lifecycle.ok) return fail(`set the lifecycle of ${bucketUrl}`, lifecycle);
|
|
214
|
+
} finally {
|
|
215
|
+
rmSync(dir, { recursive: true, force: true });
|
|
216
|
+
}
|
|
217
|
+
done.push(`bucket ${bucketUrl} (objects deleted after 1 day)`);
|
|
218
|
+
|
|
219
|
+
// 3. Bucket grants: the job creates results under out/; the admin writes
|
|
220
|
+
// inputs, reads results and deletes both.
|
|
221
|
+
const bucketGrants: { member: string; role: string; condition?: string; why: string }[] = [
|
|
222
|
+
{
|
|
223
|
+
member,
|
|
224
|
+
role: "roles/storage.objectCreator",
|
|
225
|
+
condition: captureResultsCondition(bucket),
|
|
226
|
+
why: "the job writes its result under out/",
|
|
227
|
+
},
|
|
228
|
+
{
|
|
229
|
+
member: runSaMember,
|
|
230
|
+
role: "roles/storage.objectUser",
|
|
231
|
+
why: "the admin writes inputs, reads results and deletes both",
|
|
232
|
+
},
|
|
233
|
+
];
|
|
234
|
+
for (const g of bucketGrants) {
|
|
235
|
+
const r = await retry([
|
|
236
|
+
"storage",
|
|
237
|
+
"buckets",
|
|
238
|
+
"add-iam-policy-binding",
|
|
239
|
+
bucketUrl,
|
|
240
|
+
`--member=${g.member}`,
|
|
241
|
+
`--role=${g.role}`,
|
|
242
|
+
...(g.condition ? [`--condition=${g.condition}`] : []),
|
|
243
|
+
project,
|
|
244
|
+
"--quiet",
|
|
245
|
+
"--format=none",
|
|
246
|
+
]);
|
|
247
|
+
if (!r.ok) return fail(`grant ${g.role} on ${bucketUrl} (${g.why})`, r);
|
|
248
|
+
done.push(
|
|
249
|
+
`${g.member === member ? "capture job" : "admin"}: ${g.role} on ${bucketUrl}${g.condition ? " (out/ only)" : ""}`,
|
|
250
|
+
);
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
// 4. Logs.
|
|
254
|
+
const logs = await retry([
|
|
255
|
+
"projects",
|
|
256
|
+
"add-iam-policy-binding",
|
|
257
|
+
projectId,
|
|
258
|
+
`--member=${member}`,
|
|
259
|
+
"--role=roles/logging.logWriter",
|
|
260
|
+
"--condition=None",
|
|
261
|
+
"--quiet",
|
|
262
|
+
"--format=none",
|
|
263
|
+
]);
|
|
264
|
+
if (!logs.ok) return fail("grant the capture job roles/logging.logWriter", logs);
|
|
265
|
+
done.push("capture job: roles/logging.logWriter on the project");
|
|
266
|
+
|
|
267
|
+
// 5. The admin signs the job's URLs as itself (IAM signBlob).
|
|
268
|
+
const signer = await retry([
|
|
269
|
+
"iam",
|
|
270
|
+
"service-accounts",
|
|
271
|
+
"add-iam-policy-binding",
|
|
272
|
+
runSa,
|
|
273
|
+
`--member=${runSaMember}`,
|
|
274
|
+
"--role=roles/iam.serviceAccountTokenCreator",
|
|
275
|
+
project,
|
|
276
|
+
"--quiet",
|
|
277
|
+
"--format=none",
|
|
278
|
+
]);
|
|
279
|
+
if (!signer.ok) return fail("let the admin sign URLs as itself", signer);
|
|
280
|
+
done.push(`admin: roles/iam.serviceAccountTokenCreator on ${runSa} (itself, signed URLs)`);
|
|
281
|
+
|
|
282
|
+
// 6. The job: the admin image, its own SA, no secrets, no VPC.
|
|
283
|
+
const deploy = await run([
|
|
284
|
+
"run",
|
|
285
|
+
"jobs",
|
|
286
|
+
"deploy",
|
|
287
|
+
job,
|
|
288
|
+
`--image=${target.adminImageRef}`,
|
|
289
|
+
`--region=${region}`,
|
|
290
|
+
project,
|
|
291
|
+
`--service-account=${sa}`,
|
|
292
|
+
"--command=bun",
|
|
293
|
+
`--args=--bun,${CAPTURE_TASK_SCRIPT}`,
|
|
294
|
+
`--set-env-vars=CAELO_PROVIDER=${provider},CAELO_CAPTURE_BUCKET=${bucket}`,
|
|
295
|
+
"--max-retries=0",
|
|
296
|
+
`--task-timeout=${TASK_TIMEOUT}`,
|
|
297
|
+
"--cpu=1",
|
|
298
|
+
"--memory=2Gi",
|
|
299
|
+
"--quiet",
|
|
300
|
+
]);
|
|
301
|
+
if (!deploy.ok) return fail(`deploy job ${job}`, deploy);
|
|
302
|
+
done.push(`job ${job} (${target.adminImageRef.split("@")[1]?.slice(0, 19)}...)`);
|
|
303
|
+
|
|
304
|
+
// 7. The admin may run it with its order and read its executions.
|
|
305
|
+
for (const role of ["roles/run.jobsExecutorWithOverrides", "roles/run.viewer"]) {
|
|
306
|
+
const r = await retry([
|
|
307
|
+
"run",
|
|
308
|
+
"jobs",
|
|
309
|
+
"add-iam-policy-binding",
|
|
310
|
+
job,
|
|
311
|
+
`--region=${region}`,
|
|
312
|
+
project,
|
|
313
|
+
`--member=${runSaMember}`,
|
|
314
|
+
`--role=${role}`,
|
|
315
|
+
"--quiet",
|
|
316
|
+
"--format=none",
|
|
317
|
+
]);
|
|
318
|
+
if (!r.ok) return fail(`grant the admin ${role} on ${job}`, r);
|
|
319
|
+
done.push(`admin: ${role} on ${job}`);
|
|
320
|
+
}
|
|
321
|
+
return { ok: true, done };
|
|
322
|
+
}
|
package/src/gcp-names.ts
CHANGED
|
@@ -69,7 +69,7 @@ export function staticPublisherServiceAccountId(env: string): string {
|
|
|
69
69
|
export function gcpBucketName(
|
|
70
70
|
projectId: string,
|
|
71
71
|
env: string,
|
|
72
|
-
bucket: "media" | "static" | "staging" | "audit-results",
|
|
72
|
+
bucket: "media" | "static" | "staging" | "audit-results" | "capture",
|
|
73
73
|
): string {
|
|
74
74
|
return `${projectId}-${gcpNamePrefix(env)}-${bucket}`;
|
|
75
75
|
}
|
|
@@ -162,6 +162,34 @@ export function auditServiceAccountEmail(projectId: string, env: string): string
|
|
|
162
162
|
return `${auditServiceAccountId(env)}@${projectId}.iam.gserviceaccount.com`;
|
|
163
163
|
}
|
|
164
164
|
|
|
165
|
+
/**
|
|
166
|
+
* #1217 — Cloud Run job that runs every server-side browser render
|
|
167
|
+
* (screenshots, rendered HTML) of the install, one execution per render, so
|
|
168
|
+
* no Chromium runs in the admin (capture-job-setup.ts).
|
|
169
|
+
*/
|
|
170
|
+
export function captureJobName(env: string): string {
|
|
171
|
+
return `${gcpNamePrefix(env)}-capture`;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/** Full resource name of the capture job, as the admin's env carries it. */
|
|
175
|
+
export function captureJobResource(projectId: string, region: string, env: string): string {
|
|
176
|
+
return `projects/${projectId}/locations/${region}/jobs/${captureJobName(env)}`;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* Account id of the capture job's own service account: creates capture
|
|
181
|
+
* results and writes logs — nothing else; it reads only through the signed
|
|
182
|
+
* URLs the admin hands it (capture-job-setup.ts).
|
|
183
|
+
*/
|
|
184
|
+
export function captureServiceAccountId(env: string): string {
|
|
185
|
+
return `${gcpNamePrefix(env)}-capture`;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/** Email of the capture job's service account. */
|
|
189
|
+
export function captureServiceAccountEmail(projectId: string, env: string): string {
|
|
190
|
+
return `${captureServiceAccountId(env)}@${projectId}.iam.gserviceaccount.com`;
|
|
191
|
+
}
|
|
192
|
+
|
|
165
193
|
/**
|
|
166
194
|
* Account id of the upgrader job's own service account: may roll the two
|
|
167
195
|
* Cloud Run services, run the migration job and back up Cloud SQL, never
|
|
@@ -20,8 +20,8 @@ describe("infrastructure contract steps", () => {
|
|
|
20
20
|
);
|
|
21
21
|
});
|
|
22
22
|
|
|
23
|
-
it("is at
|
|
24
|
-
expect(INFRA_CONTRACT_VERSION).toBe(
|
|
23
|
+
it("is at 14: public_role revoke (3), URL-map redirects role (4), no gateway cms_admin login (5), ops-bucket list (6), gateway proxy hops (7), quality-audit job (8), converge job (9), remote MCP (10), Google sign-in (11), project-level sign-in client (12), edge security headers (13), capture job (14)", () => {
|
|
24
|
+
expect(INFRA_CONTRACT_VERSION).toBe(14);
|
|
25
25
|
expect(INFRA_CONTRACT_STEPS[3]).toContain("#633");
|
|
26
26
|
expect(INFRA_CONTRACT_STEPS[4]).toContain("#587");
|
|
27
27
|
expect(INFRA_CONTRACT_STEPS[5]).toContain("#635");
|
|
@@ -33,6 +33,7 @@ describe("infrastructure contract steps", () => {
|
|
|
33
33
|
expect(INFRA_CONTRACT_STEPS[11]).toContain("#914");
|
|
34
34
|
expect(INFRA_CONTRACT_STEPS[12]).toContain("project's IAP settings");
|
|
35
35
|
expect(INFRA_CONTRACT_STEPS[13]).toContain("#891");
|
|
36
|
+
expect(INFRA_CONTRACT_STEPS[14]).toContain("#1217");
|
|
36
37
|
});
|
|
37
38
|
|
|
38
39
|
it("lists the pending steps oldest first", () => {
|
|
@@ -43,6 +44,6 @@ describe("infrastructure contract steps", () => {
|
|
|
43
44
|
});
|
|
44
45
|
|
|
45
46
|
it("names a version a newer release added by number", () => {
|
|
46
|
-
expect(infraStepsText(
|
|
47
|
+
expect(infraStepsText(14, 15)).toBe("infrastructure step 15 (see that release's notes)");
|
|
47
48
|
});
|
|
48
49
|
});
|
package/src/infra-contract.ts
CHANGED
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
* whose app-roll references infrastructure an install does not have yet fails
|
|
23
23
|
* at roll time, after migrations ran.
|
|
24
24
|
*/
|
|
25
|
-
export const INFRA_CONTRACT_VERSION =
|
|
25
|
+
export const INFRA_CONTRACT_VERSION = 14;
|
|
26
26
|
|
|
27
27
|
// History (why each bump happened — an install below a number needs the
|
|
28
28
|
// converge that introduced it):
|
|
@@ -101,6 +101,15 @@ export const INFRA_CONTRACT_VERSION = 13;
|
|
|
101
101
|
// a Front Door rule set (static-security-headers.ts). Backend-bucket,
|
|
102
102
|
// object-metadata and CDN writes the in-cloud upgrader holds no rights
|
|
103
103
|
// for, so the converge applies them.
|
|
104
|
+
// 14 — #1217: the capture job (`caelo-<env>-capture`, capture-job-setup.ts)
|
|
105
|
+
// every server-side browser render runs in, its service account, the
|
|
106
|
+
// capture bucket and their grants (job: create under out/ only; admin:
|
|
107
|
+
// object user on the bucket, run-with-overrides on the job, and
|
|
108
|
+
// tokenCreator on its own account to sign the job's URLs), the
|
|
109
|
+
// upgrader's actAs on the job's account, plus CAELO_CAPTURE_JOB /
|
|
110
|
+
// CAELO_CAPTURE_BUCKET / CAELO_MEDIA_BUCKET on the admin. The new admin
|
|
111
|
+
// runs no browser itself on GCP and the app-roll moves the job's image,
|
|
112
|
+
// so it must exist before an in-cloud upgrader rolls onto it.
|
|
104
113
|
|
|
105
114
|
/**
|
|
106
115
|
* What each contract version adds, in operator terms — the upgrader's
|
|
@@ -121,6 +130,7 @@ export const INFRA_CONTRACT_STEPS: Readonly<Record<number, string>> = {
|
|
|
121
130
|
11: "Google sign-in: the renamed sign-in client secret, its IAM and mounts, the operator-access job's read access to it, and the admin's URL and organisation domain (#914)",
|
|
122
131
|
12: "Google sign-in at project level: the operator-access job's permission to set the sign-in client in the project's IAP settings (#914)",
|
|
123
132
|
13: "the public site's security headers at the edge: nosniff on every file, uploaded SVGs no longer run as documents on the site's origin, and a one-time CDN purge (#891)",
|
|
133
|
+
14: "the capture job that takes server-side screenshots outside the admin, its service account, bucket and grants (#1217)",
|
|
124
134
|
};
|
|
125
135
|
|
|
126
136
|
/**
|
package/src/release-contract.ts
CHANGED
|
@@ -53,6 +53,8 @@ export const INFRA_CONTRACT_PATHS: readonly string[] = [
|
|
|
53
53
|
// #875 — the quality-audit job, its service account, results bucket and
|
|
54
54
|
// grants.
|
|
55
55
|
"packages/provisioning/src/audit-job-setup.ts",
|
|
56
|
+
// #1217 — the capture job, its service account, bucket and grants.
|
|
57
|
+
"packages/provisioning/src/capture-job-setup.ts",
|
|
56
58
|
// #819 — the converge job, its database login and the admin's grant on it.
|
|
57
59
|
"packages/provisioning/src/converge-job-setup.ts",
|
|
58
60
|
// #891 — the edge security headers the stacks attach, and the converge's
|
|
@@ -71,6 +71,7 @@ describe("adminEnvContract", () => {
|
|
|
71
71
|
CAELO_GENERATOR_CLI: "/app/apps/static-generator/src/cli.ts",
|
|
72
72
|
MEDIA_ROOT_DIR: "/app/apps/admin/data/media",
|
|
73
73
|
CAELO_MCP_IAP_SERVICE_ACCOUNT: "caelo-mcp@acme.iam.gserviceaccount.com",
|
|
74
|
+
CAELO_MEDIA_BUCKET: "acme-caelo-production-media",
|
|
74
75
|
// #806 — where Settings → MCP → Remote writes the OAuth client.
|
|
75
76
|
CAELO_OPERATOR_OAUTH_CLIENT_STORE:
|
|
76
77
|
"projects/acme/secrets/caelo-production-operator-oauth-client",
|
|
@@ -80,6 +81,8 @@ describe("adminEnvContract", () => {
|
|
|
80
81
|
CAELO_CONVERGE_JOB: "projects/acme/locations/europe-west1/jobs/caelo-production-converge",
|
|
81
82
|
CAELO_AUDIT_JOB: "projects/acme/locations/europe-west1/jobs/caelo-production-audit",
|
|
82
83
|
CAELO_AUDIT_RESULTS_BUCKET: "acme-caelo-production-audit-results",
|
|
84
|
+
CAELO_CAPTURE_BUCKET: "acme-caelo-production-capture",
|
|
85
|
+
CAELO_CAPTURE_JOB: "projects/acme/locations/europe-west1/jobs/caelo-production-capture",
|
|
83
86
|
CAELO_STATIC_BUCKET: "acme-caelo-production-static",
|
|
84
87
|
CAELO_STAGING_BUCKET: "acme-caelo-production-staging",
|
|
85
88
|
...ADMIN_SECRETS,
|
|
@@ -107,6 +110,7 @@ describe("adminEnvContract", () => {
|
|
|
107
110
|
CAELO_GENERATOR_CLI: "/app/apps/static-generator/src/cli.ts",
|
|
108
111
|
MEDIA_ROOT_DIR: "/app/apps/admin/data/media",
|
|
109
112
|
CAELO_MCP_IAP_SERVICE_ACCOUNT: "caelo-mcp@acme.iam.gserviceaccount.com",
|
|
113
|
+
CAELO_MEDIA_BUCKET: "acme-caelo-production-media",
|
|
110
114
|
// #806 — where Settings → MCP → Remote writes the OAuth client.
|
|
111
115
|
CAELO_OPERATOR_OAUTH_CLIENT_STORE:
|
|
112
116
|
"projects/acme/secrets/caelo-production-operator-oauth-client",
|
|
@@ -116,6 +120,8 @@ describe("adminEnvContract", () => {
|
|
|
116
120
|
CAELO_CONVERGE_JOB: "projects/acme/locations/europe-west1/jobs/caelo-production-converge",
|
|
117
121
|
CAELO_AUDIT_JOB: "projects/acme/locations/europe-west1/jobs/caelo-production-audit",
|
|
118
122
|
CAELO_AUDIT_RESULTS_BUCKET: "acme-caelo-production-audit-results",
|
|
123
|
+
CAELO_CAPTURE_BUCKET: "acme-caelo-production-capture",
|
|
124
|
+
CAELO_CAPTURE_JOB: "projects/acme/locations/europe-west1/jobs/caelo-production-capture",
|
|
119
125
|
CAELO_FIREBASE_SITE: "caelo-production-site-abc123",
|
|
120
126
|
CAELO_GATEWAY_SERVICE: "caelo-production-gateway-1a2b3c4",
|
|
121
127
|
CAELO_GATEWAY_REGION: "europe-west1",
|
package/src/stack-contract.ts
CHANGED
|
@@ -38,6 +38,7 @@ import { z } from "zod";
|
|
|
38
38
|
import type { SupportedProvider } from "./adapter.js";
|
|
39
39
|
import {
|
|
40
40
|
auditJobResource,
|
|
41
|
+
captureJobResource,
|
|
41
42
|
convergeJobResource,
|
|
42
43
|
gatewayServiceAccountEmail,
|
|
43
44
|
gcpBucketName,
|
|
@@ -188,6 +189,16 @@ export const CONVERGE_JOB_ENV_VAR = "CAELO_CONVERGE_JOB";
|
|
|
188
189
|
export const AUDIT_JOB_ENV_VAR = "CAELO_AUDIT_JOB";
|
|
189
190
|
export const AUDIT_RESULTS_BUCKET_ENV_VAR = "CAELO_AUDIT_RESULTS_BUCKET";
|
|
190
191
|
|
|
192
|
+
/**
|
|
193
|
+
* #1217 — the admin env vars naming the capture job, the bucket it hands
|
|
194
|
+
* inputs and results through, and the media bucket whose objects the admin
|
|
195
|
+
* signs read URLs for (admin-core capture/capture-backend.ts). `upgrade`
|
|
196
|
+
* leaves the first two untouched when it could not set the job up.
|
|
197
|
+
*/
|
|
198
|
+
export const CAPTURE_JOB_ENV_VAR = "CAELO_CAPTURE_JOB";
|
|
199
|
+
export const CAPTURE_BUCKET_ENV_VAR = "CAELO_CAPTURE_BUCKET";
|
|
200
|
+
export const MEDIA_BUCKET_ENV_VAR = "CAELO_MEDIA_BUCKET";
|
|
201
|
+
|
|
191
202
|
/** Where the admin image ships the static-generator CLI. */
|
|
192
203
|
const GENERATOR_CLI = "/app/apps/static-generator/src/cli.ts";
|
|
193
204
|
|
|
@@ -916,6 +927,12 @@ export function adminEnvContract<V>(inputs: AdminEnvInputs<V>): CloudRunEnvVar<V
|
|
|
916
927
|
// where the tasks write their results (audit-job-setup.ts).
|
|
917
928
|
{ name: AUDIT_JOB_ENV_VAR, value: auditJobResource(projectId, inputs.region, env) },
|
|
918
929
|
{ name: AUDIT_RESULTS_BUCKET_ENV_VAR, value: gcpBucketName(projectId, env, "audit-results") },
|
|
930
|
+
// #1217 — the job every server-side browser render runs in, the bucket
|
|
931
|
+
// it exchanges inputs and results through, and the media bucket the
|
|
932
|
+
// admin signs a preview's media URLs for (capture-job-setup.ts).
|
|
933
|
+
{ name: CAPTURE_JOB_ENV_VAR, value: captureJobResource(projectId, inputs.region, env) },
|
|
934
|
+
{ name: CAPTURE_BUCKET_ENV_VAR, value: gcpBucketName(projectId, env, "capture") },
|
|
935
|
+
{ name: MEDIA_BUCKET_ENV_VAR, value: gcpBucketName(projectId, env, "media") },
|
|
919
936
|
// #914 — where Settings → Google sign-in writes the install's Google
|
|
920
937
|
// sign-in client (add-version only, see the run SA's secretVersionAdder
|
|
921
938
|
// invariant).
|
|
@@ -322,7 +322,7 @@ describe("planContractEnv", () => {
|
|
|
322
322
|
// #650 — the same roll converges the protected staging domain's host
|
|
323
323
|
// and gate key onto an install that predates it.
|
|
324
324
|
expect(plan.services.admin.flags).toEqual([
|
|
325
|
-
"--update-env-vars=CAELO_SITE_URL=https://acme.com,MEDIA_ROOT_DIR=/app/apps/admin/data/media,CAELO_MCP_IAP_SERVICE_ACCOUNT=caelo-mcp@acme.iam.gserviceaccount.com,CAELO_OPERATOR_ACCESS_JOB=projects/acme/locations/europe-west1/jobs/caelo-production-operator-access-sync,CAELO_UPGRADER_JOB=projects/acme/locations/europe-west1/jobs/caelo-production-upgrader,CAELO_CONVERGE_JOB=projects/acme/locations/europe-west1/jobs/caelo-production-converge,CAELO_AUDIT_JOB=projects/acme/locations/europe-west1/jobs/caelo-production-audit,CAELO_AUDIT_RESULTS_BUCKET=acme-caelo-production-audit-results,CAELO_OPERATOR_OAUTH_CLIENT_STORE=projects/acme/secrets/caelo-production-operator-oauth-client,CAELO_STAGING_HOST=staging.acme.com",
|
|
325
|
+
"--update-env-vars=CAELO_SITE_URL=https://acme.com,MEDIA_ROOT_DIR=/app/apps/admin/data/media,CAELO_MCP_IAP_SERVICE_ACCOUNT=caelo-mcp@acme.iam.gserviceaccount.com,CAELO_OPERATOR_ACCESS_JOB=projects/acme/locations/europe-west1/jobs/caelo-production-operator-access-sync,CAELO_UPGRADER_JOB=projects/acme/locations/europe-west1/jobs/caelo-production-upgrader,CAELO_CONVERGE_JOB=projects/acme/locations/europe-west1/jobs/caelo-production-converge,CAELO_AUDIT_JOB=projects/acme/locations/europe-west1/jobs/caelo-production-audit,CAELO_AUDIT_RESULTS_BUCKET=acme-caelo-production-audit-results,CAELO_CAPTURE_JOB=projects/acme/locations/europe-west1/jobs/caelo-production-capture,CAELO_CAPTURE_BUCKET=acme-caelo-production-capture,CAELO_MEDIA_BUCKET=acme-caelo-production-media,CAELO_OPERATOR_OAUTH_CLIENT_STORE=projects/acme/secrets/caelo-production-operator-oauth-client,CAELO_STAGING_HOST=staging.acme.com",
|
|
326
326
|
"--update-secrets=CAELO_STAGING_GATE_SECRET=caelo-production-staging-gate-secret:latest",
|
|
327
327
|
]);
|
|
328
328
|
expect(plan.services.gateway.flags).toEqual([
|
|
@@ -1,28 +1,57 @@
|
|
|
1
1
|
// SPDX-License-Identifier: MPL-2.0
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
|
-
*
|
|
5
|
-
* quality/audit-task.ts) and
|
|
6
|
-
*
|
|
7
|
-
*
|
|
4
|
+
* The Cloud Run jobs that run the admin image: the quality-audit job (#875,
|
|
5
|
+
* admin-core quality/audit-task.ts) and the capture job (#1217, admin-core
|
|
6
|
+
* capture/capture-task.ts). Each refuses an order from an admin on another
|
|
7
|
+
* version, so every app-roll moves them onto the admin's new image right
|
|
8
|
+
* after the services rolled, and a rollback moves them back.
|
|
8
9
|
*
|
|
9
|
-
* `run jobs update --image` changes nothing else of
|
|
10
|
+
* `run jobs update --image` changes nothing else of a job (identity, env,
|
|
10
11
|
* parallelism), so this stays inside the app-roll's rights: the upgrader's
|
|
11
|
-
*
|
|
12
|
-
* (upgrader-setup.ts).
|
|
12
|
+
* Cloud Run role plus actAs on each job's service account (upgrader-setup.ts).
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
15
|
import type { GcloudRunner } from "../gcloud-retry.js";
|
|
16
|
-
import { auditJobName } from "../gcp-names.js";
|
|
16
|
+
import { auditJobName, captureJobName } from "../gcp-names.js";
|
|
17
|
+
|
|
18
|
+
/** One job that follows the admin's image. */
|
|
19
|
+
export interface AdminImageJob {
|
|
20
|
+
/** The job's name (`caelo-<env>-audit`). */
|
|
21
|
+
readonly name: string;
|
|
22
|
+
/** How the upgrade names it (`quality-audit job`). */
|
|
23
|
+
readonly label: string;
|
|
24
|
+
/** What fails while it lags behind the admin. */
|
|
25
|
+
readonly lagging: string;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** The jobs that follow the admin's image, for one environment. */
|
|
29
|
+
export function adminImageJobs(env: string): readonly AdminImageJob[] {
|
|
30
|
+
return [
|
|
31
|
+
{ name: auditJobName(env), label: "quality-audit job", lagging: "Stage audits fail" },
|
|
32
|
+
{
|
|
33
|
+
name: captureJobName(env),
|
|
34
|
+
label: "capture job",
|
|
35
|
+
lagging: "server-side screenshots fail",
|
|
36
|
+
},
|
|
37
|
+
];
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
interface Install {
|
|
41
|
+
readonly projectId: string;
|
|
42
|
+
readonly region: string;
|
|
43
|
+
readonly env: string;
|
|
44
|
+
}
|
|
17
45
|
|
|
18
46
|
/**
|
|
19
|
-
* The job's image now; `image: null` only when the install has no
|
|
47
|
+
* The job's image now; `image: null` only when the install has no such job
|
|
20
48
|
* yet. Any other gcloud failure (permissions, API, network) is an error, not
|
|
21
49
|
* "no job", so the app-roll reports it instead of a misleading hint.
|
|
22
50
|
*/
|
|
23
|
-
export async function
|
|
51
|
+
export async function describeJobImage(
|
|
24
52
|
run: GcloudRunner,
|
|
25
|
-
install:
|
|
53
|
+
install: Install,
|
|
54
|
+
job: string,
|
|
26
55
|
): Promise<
|
|
27
56
|
| { readonly ok: true; readonly image: string | null }
|
|
28
57
|
| { readonly ok: false; readonly error: string }
|
|
@@ -31,7 +60,7 @@ export async function describeAuditJobImage(
|
|
|
31
60
|
"run",
|
|
32
61
|
"jobs",
|
|
33
62
|
"describe",
|
|
34
|
-
|
|
63
|
+
job,
|
|
35
64
|
`--region=${install.region}`,
|
|
36
65
|
`--project=${install.projectId}`,
|
|
37
66
|
"--format=value(spec.template.spec.template.spec.containers[0].image)",
|
|
@@ -50,17 +79,18 @@ export async function describeAuditJobImage(
|
|
|
50
79
|
return { ok: true, image };
|
|
51
80
|
}
|
|
52
81
|
|
|
53
|
-
/** Point
|
|
54
|
-
export async function
|
|
82
|
+
/** Point a job at `imageRef`. */
|
|
83
|
+
export async function setJobImage(
|
|
55
84
|
run: GcloudRunner,
|
|
56
|
-
install:
|
|
85
|
+
install: Install,
|
|
86
|
+
job: string,
|
|
57
87
|
imageRef: string,
|
|
58
88
|
): Promise<{ readonly ok: boolean; readonly error?: string }> {
|
|
59
89
|
const r = await run([
|
|
60
90
|
"run",
|
|
61
91
|
"jobs",
|
|
62
92
|
"update",
|
|
63
|
-
|
|
93
|
+
job,
|
|
64
94
|
`--image=${imageRef}`,
|
|
65
95
|
`--region=${install.region}`,
|
|
66
96
|
`--project=${install.projectId}`,
|