@caelo-cms/provisioning 0.10.53 → 0.10.57

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (244) hide show
  1. package/dist/agent-db.js +272 -128
  2. package/dist/audit-job-setup.d.ts +69 -0
  3. package/dist/audit-job-setup.d.ts.map +1 -0
  4. package/dist/audit-job-setup.js +317 -0
  5. package/dist/audit-job-setup.js.map +1 -0
  6. package/dist/caddy.d.ts +9 -0
  7. package/dist/caddy.d.ts.map +1 -1
  8. package/dist/caddy.js +38 -0
  9. package/dist/caddy.js.map +1 -1
  10. package/dist/cli-routing.js +1 -1
  11. package/dist/cli-routing.js.map +1 -1
  12. package/dist/cli.js +36 -9
  13. package/dist/cli.js.map +1 -1
  14. package/dist/compose.js +5 -0
  15. package/dist/compose.js.map +1 -1
  16. package/dist/converge-checkpoint.d.ts +66 -0
  17. package/dist/converge-checkpoint.d.ts.map +1 -0
  18. package/dist/converge-checkpoint.js +61 -0
  19. package/dist/converge-checkpoint.js.map +1 -0
  20. package/dist/converge-job-gcp.d.ts +46 -0
  21. package/dist/converge-job-gcp.d.ts.map +1 -0
  22. package/dist/converge-job-gcp.js +368 -0
  23. package/dist/converge-job-gcp.js.map +1 -0
  24. package/dist/converge-job-setup.d.ts +62 -0
  25. package/dist/converge-job-setup.d.ts.map +1 -0
  26. package/dist/converge-job-setup.js +228 -0
  27. package/dist/converge-job-setup.js.map +1 -0
  28. package/dist/converge-job.d.ts +141 -0
  29. package/dist/converge-job.d.ts.map +1 -0
  30. package/dist/converge-job.js +325 -0
  31. package/dist/converge-job.js.map +1 -0
  32. package/dist/gcp-names.d.ts +25 -1
  33. package/dist/gcp-names.d.ts.map +1 -1
  34. package/dist/gcp-names.js +36 -0
  35. package/dist/gcp-names.js.map +1 -1
  36. package/dist/index.d.ts +2 -0
  37. package/dist/index.d.ts.map +1 -1
  38. package/dist/index.js +2 -0
  39. package/dist/index.js.map +1 -1
  40. package/dist/infra-contract.d.ts +1 -1
  41. package/dist/infra-contract.d.ts.map +1 -1
  42. package/dist/infra-contract.js +28 -1
  43. package/dist/infra-contract.js.map +1 -1
  44. package/dist/install-job.d.ts +11 -3
  45. package/dist/install-job.d.ts.map +1 -1
  46. package/dist/install-job.js +22 -12
  47. package/dist/install-job.js.map +1 -1
  48. package/dist/lifecycle.d.ts +13 -1
  49. package/dist/lifecycle.d.ts.map +1 -1
  50. package/dist/lifecycle.js +115 -58
  51. package/dist/lifecycle.js.map +1 -1
  52. package/dist/operator-access.d.ts.map +1 -1
  53. package/dist/operator-access.js +13 -3
  54. package/dist/operator-access.js.map +1 -1
  55. package/dist/private-file.d.ts +26 -0
  56. package/dist/private-file.d.ts.map +1 -0
  57. package/dist/private-file.js +49 -0
  58. package/dist/private-file.js.map +1 -0
  59. package/dist/release-contract.d.ts +3 -2
  60. package/dist/release-contract.d.ts.map +1 -1
  61. package/dist/release-contract.js +8 -2
  62. package/dist/release-contract.js.map +1 -1
  63. package/dist/release-manifest.d.ts +2 -0
  64. package/dist/release-manifest.d.ts.map +1 -1
  65. package/dist/release-manifest.js.map +1 -1
  66. package/dist/remote-mcp-edge.d.ts +114 -0
  67. package/dist/remote-mcp-edge.d.ts.map +1 -0
  68. package/dist/remote-mcp-edge.js +116 -0
  69. package/dist/remote-mcp-edge.js.map +1 -0
  70. package/dist/remote-mcp.d.ts +117 -0
  71. package/dist/remote-mcp.d.ts.map +1 -0
  72. package/dist/remote-mcp.js +379 -0
  73. package/dist/remote-mcp.js.map +1 -0
  74. package/dist/runtime-secrets.d.ts +12 -0
  75. package/dist/runtime-secrets.d.ts.map +1 -1
  76. package/dist/runtime-secrets.js +32 -4
  77. package/dist/runtime-secrets.js.map +1 -1
  78. package/dist/seal-key-rotation.d.ts +38 -0
  79. package/dist/seal-key-rotation.d.ts.map +1 -0
  80. package/dist/seal-key-rotation.js +206 -0
  81. package/dist/seal-key-rotation.js.map +1 -0
  82. package/dist/stack-contract.d.ts +175 -3
  83. package/dist/stack-contract.d.ts.map +1 -1
  84. package/dist/stack-contract.js +183 -3
  85. package/dist/stack-contract.js.map +1 -1
  86. package/dist/stack-converge.d.ts +38 -1
  87. package/dist/stack-converge.d.ts.map +1 -1
  88. package/dist/stack-converge.js +74 -4
  89. package/dist/stack-converge.js.map +1 -1
  90. package/dist/state-lock-lease.d.ts +142 -0
  91. package/dist/state-lock-lease.d.ts.map +1 -0
  92. package/dist/state-lock-lease.js +347 -0
  93. package/dist/state-lock-lease.js.map +1 -0
  94. package/dist/state-store.d.ts +66 -9
  95. package/dist/state-store.d.ts.map +1 -1
  96. package/dist/state-store.js +249 -30
  97. package/dist/state-store.js.map +1 -1
  98. package/dist/tar-entries.d.ts.map +1 -1
  99. package/dist/tar-entries.js +84 -9
  100. package/dist/tar-entries.js.map +1 -1
  101. package/dist/upgrade/app-roll.d.ts +36 -7
  102. package/dist/upgrade/app-roll.d.ts.map +1 -1
  103. package/dist/upgrade/app-roll.js +88 -19
  104. package/dist/upgrade/app-roll.js.map +1 -1
  105. package/dist/upgrade/audit-job-roll.d.ts +38 -0
  106. package/dist/upgrade/audit-job-roll.d.ts.map +1 -0
  107. package/dist/upgrade/audit-job-roll.js +43 -0
  108. package/dist/upgrade/audit-job-roll.js.map +1 -0
  109. package/dist/upgrade/converge.d.ts.map +1 -1
  110. package/dist/upgrade/converge.js +141 -8
  111. package/dist/upgrade/converge.js.map +1 -1
  112. package/dist/upgrade/gcp-lookup.d.ts +27 -10
  113. package/dist/upgrade/gcp-lookup.d.ts.map +1 -1
  114. package/dist/upgrade/gcp-lookup.js +46 -7
  115. package/dist/upgrade/gcp-lookup.js.map +1 -1
  116. package/dist/upgrade/record-rolled.d.ts +40 -0
  117. package/dist/upgrade/record-rolled.d.ts.map +1 -0
  118. package/dist/upgrade/record-rolled.js +46 -0
  119. package/dist/upgrade/record-rolled.js.map +1 -0
  120. package/dist/upgrade/upgrade.d.ts +45 -1
  121. package/dist/upgrade/upgrade.d.ts.map +1 -1
  122. package/dist/upgrade/upgrade.js +60 -17
  123. package/dist/upgrade/upgrade.js.map +1 -1
  124. package/dist/upgrader-assess.d.ts +15 -1
  125. package/dist/upgrader-assess.d.ts.map +1 -1
  126. package/dist/upgrader-assess.js +36 -6
  127. package/dist/upgrader-assess.js.map +1 -1
  128. package/dist/upgrader-gcp.d.ts +15 -0
  129. package/dist/upgrader-gcp.d.ts.map +1 -1
  130. package/dist/upgrader-gcp.js +42 -7
  131. package/dist/upgrader-gcp.js.map +1 -1
  132. package/dist/upgrader-iam-preflight.d.ts +123 -0
  133. package/dist/upgrader-iam-preflight.d.ts.map +1 -0
  134. package/dist/upgrader-iam-preflight.js +414 -0
  135. package/dist/upgrader-iam-preflight.js.map +1 -0
  136. package/dist/upgrader-job.d.ts +33 -9
  137. package/dist/upgrader-job.d.ts.map +1 -1
  138. package/dist/upgrader-job.js +169 -27
  139. package/dist/upgrader-job.js.map +1 -1
  140. package/dist/upgrader-records.d.ts +58 -0
  141. package/dist/upgrader-records.d.ts.map +1 -0
  142. package/dist/upgrader-records.js +87 -0
  143. package/dist/upgrader-records.js.map +1 -0
  144. package/dist/upgrader-reporter.d.ts +3 -2
  145. package/dist/upgrader-reporter.d.ts.map +1 -1
  146. package/dist/upgrader-reporter.js +10 -2
  147. package/dist/upgrader-reporter.js.map +1 -1
  148. package/dist/upgrader-setup.d.ts +25 -3
  149. package/dist/upgrader-setup.d.ts.map +1 -1
  150. package/dist/upgrader-setup.js +132 -14
  151. package/dist/upgrader-setup.js.map +1 -1
  152. package/dist/wizards/gcp-cost.d.ts.map +1 -1
  153. package/dist/wizards/gcp-cost.js +14 -6
  154. package/dist/wizards/gcp-cost.js.map +1 -1
  155. package/dist/wizards/gcp-pulumi.d.ts +6 -0
  156. package/dist/wizards/gcp-pulumi.d.ts.map +1 -1
  157. package/dist/wizards/gcp-pulumi.js +7 -0
  158. package/dist/wizards/gcp-pulumi.js.map +1 -1
  159. package/dist/wizards/gcp.d.ts.map +1 -1
  160. package/dist/wizards/gcp.js +130 -4
  161. package/dist/wizards/gcp.js.map +1 -1
  162. package/package.json +4 -4
  163. package/src/agent/run.test.ts +15 -1
  164. package/src/audit-job-setup.test.ts +234 -0
  165. package/src/audit-job-setup.ts +360 -0
  166. package/src/caddy-redirects.docker.test.ts +19 -6
  167. package/src/caddy.test.ts +30 -0
  168. package/src/caddy.ts +43 -0
  169. package/src/cli-init-validation.test.ts +44 -0
  170. package/src/cli-routing.ts +1 -1
  171. package/src/cli.ts +40 -8
  172. package/src/compose.ts +5 -0
  173. package/src/converge-checkpoint.ts +75 -0
  174. package/src/converge-entrypoint.test.ts +213 -0
  175. package/src/converge-job-gcp.test.ts +222 -0
  176. package/src/converge-job-gcp.ts +443 -0
  177. package/src/converge-job-setup.test.ts +152 -0
  178. package/src/converge-job-setup.ts +268 -0
  179. package/src/converge-job.test.ts +486 -0
  180. package/src/converge-job.ts +542 -0
  181. package/src/gateway-env-isolation.test.ts +8 -1
  182. package/src/gcp-names.ts +43 -1
  183. package/src/gcp-url-map.test.ts +9 -1
  184. package/src/index.ts +8 -0
  185. package/src/infra-contract.test.ts +7 -4
  186. package/src/infra-contract.ts +28 -1
  187. package/src/install-job.test.ts +3 -1
  188. package/src/install-job.ts +31 -14
  189. package/src/lifecycle.ts +137 -71
  190. package/src/operator-access.test.ts +14 -0
  191. package/src/operator-access.ts +13 -3
  192. package/src/private-file.test.ts +80 -0
  193. package/src/private-file.ts +50 -0
  194. package/src/release-contract.ts +8 -2
  195. package/src/release-manifest.test.ts +42 -0
  196. package/src/release-manifest.ts +2 -0
  197. package/src/remote-mcp-edge.ts +137 -0
  198. package/src/remote-mcp.test.ts +517 -0
  199. package/src/remote-mcp.ts +460 -0
  200. package/src/runtime-secrets.test.ts +31 -6
  201. package/src/runtime-secrets.ts +41 -3
  202. package/src/seal-key-rotation.test.ts +131 -0
  203. package/src/seal-key-rotation.ts +258 -0
  204. package/src/stack-contract-parity.test.ts +20 -3
  205. package/src/stack-contract.test.ts +34 -2
  206. package/src/stack-contract.ts +305 -3
  207. package/src/stack-converge-secrets.test.ts +3 -0
  208. package/src/stack-converge.test.ts +21 -5
  209. package/src/stack-converge.ts +101 -7
  210. package/src/state-attach.test.ts +7 -1
  211. package/src/state-lock-lease.test.ts +302 -0
  212. package/src/state-lock-lease.ts +423 -0
  213. package/src/state-lock.test.ts +568 -0
  214. package/src/state-store.ts +347 -27
  215. package/src/tar-entries.test.ts +149 -7
  216. package/src/tar-entries.ts +90 -9
  217. package/src/upgrade/app-roll.ts +135 -21
  218. package/src/upgrade/audit-job-roll.ts +68 -0
  219. package/src/upgrade/converge.ts +170 -7
  220. package/src/upgrade/gcp-lookup.test.ts +80 -0
  221. package/src/upgrade/gcp-lookup.ts +65 -8
  222. package/src/upgrade/record-rolled.test.ts +105 -0
  223. package/src/upgrade/record-rolled.ts +91 -0
  224. package/src/upgrade/upgrade.test.ts +279 -5
  225. package/src/upgrade/upgrade.ts +120 -24
  226. package/src/upgrader-assess.test.ts +93 -3
  227. package/src/upgrader-assess.ts +58 -9
  228. package/src/upgrader-gcp.ts +47 -8
  229. package/src/upgrader-iam-preflight.test.ts +393 -0
  230. package/src/upgrader-iam-preflight.ts +510 -0
  231. package/src/upgrader-job.test.ts +587 -15
  232. package/src/upgrader-job.ts +237 -53
  233. package/src/upgrader-records.ts +151 -0
  234. package/src/upgrader-reporter.test.ts +6 -0
  235. package/src/upgrader-reporter.ts +7 -2
  236. package/src/upgrader-setup.test.ts +4 -3
  237. package/src/upgrader-setup.ts +137 -13
  238. package/src/wizards/gcp-cost.ts +15 -6
  239. package/src/wizards/gcp-pulumi.ts +7 -0
  240. package/src/wizards/gcp.ts +162 -4
  241. package/stacks/aws/index.ts +18 -0
  242. package/stacks/azure/index.ts +18 -0
  243. package/stacks/gcp/index.ts +59 -7
  244. package/stacks/gcp-firebase/index.ts +50 -6
@@ -17,12 +17,23 @@
17
17
  * The job's own service account, `caelo-<env>-upgrader`, may do exactly
18
18
  * what an app-roll needs (upgrade/app-roll.ts):
19
19
  *
20
- * - `roles/run.developer` on the project: deploy admin/gateway revisions,
21
- * move traffic, recreate the migration job, update its own job image.
22
- * It includes no `*.setIamPolicy`.
20
+ * - the custom role `caeloUpgraderRun` on the project
21
+ * ({@link UPGRADER_RUN_ROLE}): deploy admin/gateway revisions, move
22
+ * traffic, recreate and run the migration job, update its own job image.
23
+ * NOT `roles/run.developer`: that role holds `run.jobs.runWithOverrides`
24
+ * on every job of the project, including the converge job and the
25
+ * install job, which run as `caelo-provisioner` — the upgrader could run
26
+ * them with its own args (destroy, owner-reset) or env (CLOUDSDK_*,
27
+ * PULUMI_*) and so act as the provisioner (#883 review). The custom role
28
+ * has no `runWithOverrides` and no `*.setIamPolicy`. Updating another
29
+ * job (the converge job's image, say) additionally needs `actAs` on its
30
+ * service account, which the upgrader holds only on the admin, gateway
31
+ * and its own. A grant of `roles/run.developer` left by an earlier
32
+ * setup is removed.
23
33
  * - `roles/iam.serviceAccountUser` on exactly the admin's and the
24
- * gateway's runtime service accounts (a revision runs as them) and on
25
- * itself (its own job) — not on the project.
34
+ * gateway's runtime service accounts (a revision runs as them), on
35
+ * itself (its own job) and on the quality-audit job's account (the
36
+ * app-roll moves that job's image, #875) — not on the project.
26
37
  * - the custom role `caeloUpgraderSqlBackup`: create/read Cloud SQL
27
38
  * backups and read the instance — no restore, no instance changes.
28
39
  * - `roles/cloudsql.instanceUser` + a Cloud SQL IAM database user that
@@ -45,12 +56,26 @@
45
56
  * is a converge, which only the operator runs ("Run in Cloud Shell").
46
57
  * {@link UPGRADER_FORBIDDEN_ROLES} names that line; the tests assert no
47
58
  * grant crosses it.
59
+ *
60
+ * Direct VPC egress (#818): the migration job and both services run with
61
+ * `--network/--subnet`. Google documents the compute permissions that needs
62
+ * (`compute.subnetworks.use`, `compute.addresses.createInternal`, …) on the
63
+ * Cloud Run service agent, which `roles/run.serviceAgent` already gives it
64
+ * (cloud.google.com/run/docs/configuring/vpc-direct-vpc), not on the
65
+ * principal that deploys. So the upgrader gets no compute role; the
66
+ * real-GCP upgrader smoke test (.github/workflows/upgrader-smoke.yml) shows
67
+ * it once a stable release was installed on the smoke install through
68
+ * Settings → Updates (the next tag cannot ship before). Every permission
69
+ * the job needs is checked by the job itself before it changes anything
70
+ * (upgrader-iam-preflight.ts);
71
+ * a grant added or removed here must keep that list in step.
48
72
  */
49
73
 
50
74
  import { ensureCloudSqlIamUser } from "./cloudsql-iam-user.js";
51
75
  import { gcloud as defaultGcloud, type GcloudResult } from "./gcloud.js";
52
76
  import { type GcloudRunner, realSleep, runWithRetry, type Sleep } from "./gcloud-retry.js";
53
77
  import {
78
+ auditServiceAccountEmail,
54
79
  gatewayServiceAccountEmail,
55
80
  mcpIapServiceAccountEmail,
56
81
  runServiceAccountEmail,
@@ -89,22 +114,72 @@ export const UPGRADER_OPS_LIST_ROLE: CustomRoleSpec = {
89
114
  permissions: ["storage.objects.list"],
90
115
  };
91
116
 
92
- /** Project-level roles of the upgrader's service account. */
117
+ /**
118
+ * What an app-roll does on Cloud Run (upgrade/app-roll.ts, migration-runner.ts,
119
+ * the self-update, the #815 lock lease) — `roles/run.developer` minus
120
+ * everything else, above all `run.jobs.runWithOverrides` (see the file
121
+ * header). Keep in step with upgrader-iam-preflight.ts.
122
+ */
123
+ export const UPGRADER_RUN_ROLE: CustomRoleSpec = {
124
+ roleId: "caeloUpgraderRun",
125
+ title: "Caelo upgrader: roll Cloud Run",
126
+ description:
127
+ "Lets the Caelo upgrader job deploy admin/gateway revisions, run the migration job and update its own image. Cannot start any job with changed arguments or env, cannot change IAM.",
128
+ permissions: [
129
+ "run.configurations.get",
130
+ "run.configurations.list",
131
+ "run.executions.get",
132
+ "run.executions.list",
133
+ "run.jobs.create",
134
+ "run.jobs.delete",
135
+ "run.jobs.get",
136
+ "run.jobs.list",
137
+ "run.jobs.run",
138
+ "run.jobs.update",
139
+ "run.operations.get",
140
+ "run.revisions.get",
141
+ "run.revisions.list",
142
+ "run.routes.get",
143
+ "run.routes.list",
144
+ "run.services.get",
145
+ "run.services.list",
146
+ "run.services.update",
147
+ "run.tasks.get",
148
+ "run.tasks.list",
149
+ ],
150
+ };
151
+
152
+ /** Project-level predefined roles of the upgrader's service account. */
93
153
  export const UPGRADER_PROJECT_ROLES = [
94
- "roles/run.developer",
95
154
  "roles/cloudsql.instanceUser",
96
155
  "roles/logging.logWriter",
97
156
  ] as const;
98
157
 
158
+ /**
159
+ * Predefined roles that hold `run.jobs.runWithOverrides`. Neither the
160
+ * upgrader nor the admin may hold any of them anywhere: with it, either
161
+ * could run the converge or install job — which run as caelo-provisioner —
162
+ * with its own arguments or environment (#883 review).
163
+ */
164
+ export const RUN_WITH_OVERRIDES_ROLES = [
165
+ "roles/owner",
166
+ "roles/editor",
167
+ "roles/run.admin",
168
+ "roles/run.developer",
169
+ "roles/run.invoker",
170
+ "roles/run.jobsExecutorWithOverrides",
171
+ ] as const;
172
+
173
+ /** A project-level grant an earlier setup gave the upgrader; removed on every setup. */
174
+ const LEGACY_UPGRADER_PROJECT_ROLES = ["roles/run.developer"] as const;
175
+
99
176
  /**
100
177
  * Roles the upgrader must never hold anywhere — they would let it change
101
178
  * IAM, manage service accounts or secrets, or touch compute/projects.
102
179
  * Prefix matches (`roles/compute.` = every compute role).
103
180
  */
104
181
  export const UPGRADER_FORBIDDEN_ROLES = [
105
- "roles/owner",
106
- "roles/editor",
107
- "roles/run.admin",
182
+ ...RUN_WITH_OVERRIDES_ROLES,
108
183
  "roles/iam.serviceAccountAdmin",
109
184
  "roles/iam.serviceAccountKeyAdmin",
110
185
  "roles/iam.securityAdmin",
@@ -122,8 +197,14 @@ export const UPGRADER_FORBIDDEN_ROLES = [
122
197
  /** Database role migration 0267 creates for the job. */
123
198
  export const UPGRADER_DB_ROLE = "platform_updater";
124
199
 
125
- /** The job's resources: enough for a backup + roll; the job ends within it. */
126
- const JOB_TASK_TIMEOUT = "60m";
200
+ /**
201
+ * The job's task timeout in minutes: enough for a backup + roll; the job
202
+ * ends within it. It is also the lease on the state lock the job takes
203
+ * (#815, state-lock-lease.ts): after it, Cloud Run has killed the run.
204
+ */
205
+ export const UPGRADER_TASK_TIMEOUT_MINUTES = 60;
206
+
207
+ const JOB_TASK_TIMEOUT = `${UPGRADER_TASK_TIMEOUT_MINUTES}m`;
127
208
 
128
209
  export interface UpgraderSetupTarget {
129
210
  readonly projectId: string;
@@ -195,7 +276,7 @@ export async function ensureUpgraderJob(
195
276
  }
196
277
 
197
278
  // 1. The backup and ops-bucket-list roles.
198
- for (const role of [UPGRADER_SQL_BACKUP_ROLE, UPGRADER_OPS_LIST_ROLE]) {
279
+ for (const role of [UPGRADER_RUN_ROLE, UPGRADER_SQL_BACKUP_ROLE, UPGRADER_OPS_LIST_ROLE]) {
199
280
  const roleError = await ensureCustomRole(run, projectId, role);
200
281
  if (roleError) return { ok: false, done, error: roleError };
201
282
  done.push(`custom role ${role.roleId}`);
@@ -227,6 +308,7 @@ export async function ensureUpgraderJob(
227
308
  // 3. Project-level roles.
228
309
  for (const role of [
229
310
  ...UPGRADER_PROJECT_ROLES,
311
+ customRoleName(projectId, UPGRADER_RUN_ROLE.roleId),
230
312
  customRoleName(projectId, UPGRADER_SQL_BACKUP_ROLE.roleId),
231
313
  ]) {
232
314
  const r = await retry([
@@ -243,6 +325,34 @@ export async function ensureUpgraderJob(
243
325
  done.push(`upgrader: ${role} on the project`);
244
326
  }
245
327
 
328
+ // 3b. The project-wide roles/run.developer an earlier setup granted holds
329
+ // run.jobs.runWithOverrides on the provisioner's jobs: take it away.
330
+ // Only after the custom role above is bound, so a running upgrade
331
+ // never loses its Cloud Run rights in between.
332
+ const policy = await run(["projects", "get-iam-policy", projectId, "--format=json"]);
333
+ if (!policy.ok) return fail("read the project's IAM policy", policy);
334
+ for (const role of LEGACY_UPGRADER_PROJECT_ROLES) {
335
+ let held: boolean;
336
+ try {
337
+ held = projectPolicyHas(policy.stdout, role, member);
338
+ } catch (e) {
339
+ return { ok: false, done, error: `read the project's IAM policy: ${String(e)}` };
340
+ }
341
+ if (!held) continue;
342
+ const r = await retry([
343
+ "projects",
344
+ "remove-iam-policy-binding",
345
+ projectId,
346
+ `--member=${member}`,
347
+ `--role=${role}`,
348
+ "--all",
349
+ "--quiet",
350
+ "--format=none",
351
+ ]);
352
+ if (!r.ok) return fail(`remove the upgrader's ${role}`, r);
353
+ done.push(`upgrader: removed ${role} on the project`);
354
+ }
355
+
246
356
  // 4. Resource-level grants on single service accounts.
247
357
  const onAccounts: { account: string; role: string; why: string }[] = [
248
358
  {
@@ -256,6 +366,11 @@ export async function ensureUpgraderJob(
256
366
  why: "gateway revisions run as it",
257
367
  },
258
368
  { account: sa, role: "roles/iam.serviceAccountUser", why: "the job updates its own image" },
369
+ {
370
+ account: auditServiceAccountEmail(projectId, env),
371
+ role: "roles/iam.serviceAccountUser",
372
+ why: "the app-roll moves the quality-audit job, which runs as it, onto the new image",
373
+ },
259
374
  {
260
375
  account: mcpIapServiceAccountEmail(projectId),
261
376
  role: "roles/iam.serviceAccountTokenCreator",
@@ -366,6 +481,15 @@ export async function ensureUpgraderJob(
366
481
  return { ok: true, done };
367
482
  }
368
483
 
484
+ /** Whether a gcloud IAM policy (JSON) grants `role` to `member`. */
485
+ export function projectPolicyHas(policyJson: string, role: string, member: string): boolean {
486
+ // Throws on unreadable JSON: the caller reports it, never guesses "absent".
487
+ const policy = JSON.parse(policyJson.trim() || "{}") as {
488
+ bindings?: { role: string; members?: string[] }[];
489
+ };
490
+ return (policy.bindings ?? []).some((b) => b.role === role && (b.members ?? []).includes(member));
491
+ }
492
+
369
493
  /**
370
494
  * {@link ensureUpgraderJob}'s target, with the VPC read from the deployed
371
495
  * admin service (the job reaches Cloud SQL's private IP the same way).
@@ -101,10 +101,10 @@ export function estimateGcpCost(inputs: CostEstimateInputs): {
101
101
  : `${inputs.adminMinInstances} min-instance${inputs.adminMinInstances > 1 ? "s" : ""} (no cold start)`,
102
102
  },
103
103
  {
104
- name: "Quality checks (Lighthouse on the admin)",
105
- monthlyUsd: 1,
104
+ name: "Quality checks (Cloud Run audit job)",
105
+ monthlyUsd: 2,
106
106
  notes:
107
- "~2-4 min of admin CPU per audited Stage (up to 5 pages x 5 runs); ~$1/mo at a few Stages a day",
107
+ "one container per Lighthouse run, all in parallel (up to 5 pages x 3 runs, ~90 s at 2 vCPU each): ~$0.05 per audited Stage, partly in the free tier; results bucket ~$0",
108
108
  },
109
109
  {
110
110
  name: "Cloud Run gateway",
@@ -138,6 +138,14 @@ export function estimateGcpCost(inputs: CostEstimateInputs): {
138
138
  ? "Adaptive protection (ML-based bot mitigation)"
139
139
  : "Free tier — rate limit + OWASP basic rules",
140
140
  },
141
+ {
142
+ // #806 — the remote-MCP throttle (src/remote-mcp-edge.ts) is one
143
+ // more rule on the gateway's policy; Cloud Armor Standard bills
144
+ // each rule at about $1/mo.
145
+ name: "Cloud Armor rule: remote MCP throttle",
146
+ monthlyUsd: 1,
147
+ notes: "Per-IP throttle for /api/mcp/* (remote MCP clients)",
148
+ },
141
149
  ] satisfies CostLine[])
142
150
  : ([
143
151
  {
@@ -155,9 +163,10 @@ export function estimateGcpCost(inputs: CostEstimateInputs): {
155
163
  notes: "~5 GB storage + low egress; growth roughly $0.02/GB",
156
164
  },
157
165
  {
158
- name: "Secret Manager (5 secrets)",
159
- monthlyUsd: 0,
160
- notes: "Free tier",
166
+ name: "Secret Manager (runtime secrets)",
167
+ monthlyUsd: 0.2,
168
+ notes:
169
+ "6 active versions free, then ~$0.06 each; remote MCP (#806) adds the OAuth client and its seal key",
161
170
  },
162
171
  {
163
172
  name: "BigQuery edge log sink",
@@ -13,6 +13,7 @@
13
13
 
14
14
  import { resolve as resolvePath } from "node:path";
15
15
  import * as pulumi from "@pulumi/pulumi/automation";
16
+ import { AUDIT_PARALLELISM_DEFAULT } from "../audit-job-setup.js";
16
17
  import { checkpointableOutputs } from "../checkpoint-secrets.js";
17
18
  import { GCP_STACK_ENV } from "../gcp-names.js";
18
19
  import type { ImageDigests } from "../install-state.js";
@@ -36,6 +37,12 @@ export const GCP_STACK_KNOB_DEFAULTS = {
36
37
  adminMemory: ADMIN_MEMORY_DEFAULT,
37
38
  gatewayMinInstances: 0,
38
39
  wafAdaptiveProtection: false,
40
+ /**
41
+ * #875 — audit tasks at once. Not stack config: the audit job is
42
+ * CLI-owned (audit-job-setup.ts), so the wizard passes it to the job and
43
+ * `upgrade` keeps whatever the job has.
44
+ */
45
+ auditParallelism: AUDIT_PARALLELISM_DEFAULT,
39
46
  } as const;
40
47
 
41
48
  export interface PulumiUpInputs {
@@ -33,8 +33,10 @@ import { join } from "node:path";
33
33
  import { cancel, confirm, isCancel, log, note, select, spinner } from "@clack/prompts";
34
34
  import { bold, cyan, dim, green, red, yellow } from "kleur/colors";
35
35
  import { checkAdcImpersonation } from "../adc-impersonation.js";
36
+ import { ensureAuditJob, resolveAdminImage } from "../audit-job-setup.js";
36
37
  import { CERT_WAIT_TIMEOUT_MS, lbCertProbe, waitForCert } from "../cert-wait.js";
37
38
  import { redactCredentials } from "../checkpoint-secrets.js";
39
+ import { ensureConvergeJob } from "../converge-job-setup.js";
38
40
  import { removeDefaultFirewallRules } from "../default-firewall.js";
39
41
  import {
40
42
  checkDeployedRuntimeEnv,
@@ -100,6 +102,7 @@ import {
100
102
  } from "../provisioner-identity.js";
101
103
  import { revokePublicRoleMemberships } from "../public-role-isolation.js";
102
104
  import { decideRegion, detectCliRegion, installableRegions, suggestRegion } from "../regions.js";
105
+ import { ensureRemoteMcpEdge, ensureRemoteMcpIdentityEnv } from "../remote-mcp.js";
103
106
  import type { RunMode } from "../run-mode.js";
104
107
  import { selfCommand } from "../self-command.js";
105
108
  import { ABSENT_DEFAULT_FIREWALL_RULES } from "../stack-contract.js";
@@ -114,6 +117,7 @@ import {
114
117
  resolveBackend,
115
118
  } from "../state-backend.js";
116
119
  import { acquireStateLock, GcsStateStore } from "../state-store.js";
120
+ import { resolveGcpResourceName } from "../upgrade/gcp-lookup.js";
117
121
  import {
118
122
  releaseVersionForDigest,
119
123
  resolveOpsImageRef,
@@ -454,11 +458,17 @@ export async function runGcpConverge(opts: GcpConvergeOpts): Promise<void> {
454
458
 
455
459
  phase("Finishing: operator access, welcome page");
456
460
 
461
+ // === 11.3. The quality-audit job: parallel Lighthouse runs (#875) ===
462
+ // Before the upgrader, which gets actAs on the job's service account.
463
+ await stepAuditJob(projectId, region, provider);
464
+
457
465
  // === 11.4. The upgrader job: Settings → Updates → Install update (#696) ===
458
466
  // Before operator access, which keeps the upgrader's caelo-mcp grant.
459
467
  const releaseVersion = await releaseVersionForDigest(gcloud, "admin", imageDigests.admin);
460
468
  const upgraderReady = await stepUpgraderJob(projectId, region, releaseVersion);
461
469
  const keepSigner = upgraderReady || (await upgraderExists(projectId));
470
+ // === 11.45. The converge job: releases that change the cloud setup (#819) ===
471
+ const convergeReady = await stepConvergeJob(projectId, region, releaseVersion);
462
472
 
463
473
  // === 11.5. Operator access: IAP follows the Caelo user list ===
464
474
  await stepOperatorAccessSync(
@@ -469,11 +479,18 @@ export async function runGcpConverge(opts: GcpConvergeOpts): Promise<void> {
469
479
  keepSigner ? [`serviceAccount:${upgraderServiceAccountEmail(projectId, GCP_STACK_ENV)}`] : [],
470
480
  );
471
481
 
482
+ // === 11.6. Remote MCP (#806): LB rules (gcp) + the identity env ===
483
+ // After `pulumi up` on every pass: the identity env names services Pulumi
484
+ // generated, so the stack cannot declare it, and an `up` that changed a
485
+ // service's template dropped it (remote-mcp.ts).
486
+ const remoteMcpReady = await stepRemoteMcp(projectId, region, provider, domain);
487
+
472
488
  // A fresh install ran every converge step above, so it is at this CLI's
473
- // infrastructure contract — recorded only when the upgrader is in place
474
- // (the last step a contract bump added). With the release version, the
475
- // admin can install the next update itself.
476
- if (upgraderReady) {
489
+ // infrastructure contract — recorded only when the upgrader and the
490
+ // converge job are in place (the steps contract bumps 2 and 9 added) and
491
+ // remote MCP's steps (#806, bump 10) succeeded. With the release version,
492
+ // the admin can install the next update itself.
493
+ if (upgraderReady && convergeReady && remoteMcpReady) {
477
494
  const version = releaseVersion;
478
495
  recordImageDigests(installId, imageDigests, version);
479
496
  recordInfraContractVersion(installId, INFRA_CONTRACT_VERSION);
@@ -976,6 +993,95 @@ async function stepUpgraderJob(
976
993
  return true;
977
994
  }
978
995
 
996
+ /**
997
+ * Set up the converge job (converge-job-setup.ts, #819): what the admin's
998
+ * "Install update" starts for a release that changes the cloud setup. Same
999
+ * shape as {@link stepUpgraderJob}: after migrations (its database login
1000
+ * gets migration 0267's role), only a verified ops image, and a failure
1001
+ * only warns — such releases then install from a terminal until a later
1002
+ * `upgrade` sets the job up. Idempotent.
1003
+ *
1004
+ * @returns whether the job is ready
1005
+ */
1006
+ async function stepConvergeJob(
1007
+ projectId: string,
1008
+ region: string,
1009
+ releaseVersion: string | null,
1010
+ ): Promise<boolean> {
1011
+ const opsImageRef = releaseVersion ? await resolveOpsImageRef(releaseVersion) : null;
1012
+ const digest = opsImageRef ? /@(sha256:[0-9a-f]{64})$/.exec(opsImageRef)?.[1] : undefined;
1013
+ // The job runs as the provisioner: only a verified image, ever.
1014
+ if (digest && !(await verifyCosignAll([{ slug: "ops", digest }]))) {
1015
+ log.warn(
1016
+ `The converge job is not set up: its image's signature did not verify. The install works; ${bold(selfCommand("upgrade"))} sets it up later.`,
1017
+ );
1018
+ return false;
1019
+ }
1020
+ const s = spinner();
1021
+ s.start("Setting up the converge job (cloud-setup updates from the admin)...");
1022
+ const resolved = opsImageRef
1023
+ ? await resolveUpgraderTarget({ projectId, region, env: GCP_STACK_ENV, opsImageRef })
1024
+ : {
1025
+ ok: false as const,
1026
+ error: releaseVersion
1027
+ ? `release ${releaseVersion} has no caelo-ops image in the registry`
1028
+ : "the release this install runs could not be told from its images",
1029
+ };
1030
+ const r = resolved.ok
1031
+ ? await ensureConvergeJob(resolved.target)
1032
+ : { ok: false as const, done: [], error: resolved.error };
1033
+ if (!r.ok && r.reason === "no-provisioner") {
1034
+ // #883 review — not needed for the contract: such an install installs
1035
+ // cloud-setup releases from a terminal, and its admin says so.
1036
+ s.stop(yellow("No converge job: this install has no caelo-provisioner to run it as"));
1037
+ log.info(
1038
+ "Releases that change the cloud setup install from a terminal; Settings → Updates says so when one comes.",
1039
+ );
1040
+ return true;
1041
+ }
1042
+ if (!r.ok) {
1043
+ s.stop(yellow(`Converge job not set up: ${r.error}`));
1044
+ log.warn(
1045
+ `The install works; releases that change the cloud setup install from a terminal until ${bold(selfCommand("upgrade"))} sets the job up.`,
1046
+ );
1047
+ return false;
1048
+ }
1049
+ s.stop(green(`Converge job ready (${r.done.length} steps)`));
1050
+ return true;
1051
+ }
1052
+
1053
+ /**
1054
+ * Set up the quality-audit job (audit-job-setup.ts) on the deployed admin's
1055
+ * image. A failure stops the wizard: without the job every Stage audit
1056
+ * fails. Idempotent, so it runs on every wizard pass.
1057
+ */
1058
+ async function stepAuditJob(
1059
+ projectId: string,
1060
+ region: string,
1061
+ provider: "gcp" | "gcp-firebase",
1062
+ ): Promise<void> {
1063
+ const s = spinner();
1064
+ s.start("Setting up the quality-audit job (parallel Lighthouse runs)...");
1065
+ const image = await resolveAdminImage({ projectId, region, env: GCP_STACK_ENV });
1066
+ const r = image.ok
1067
+ ? await ensureAuditJob({
1068
+ provider,
1069
+ projectId,
1070
+ region,
1071
+ env: GCP_STACK_ENV,
1072
+ adminImageRef: image.imageRef,
1073
+ parallelism: GCP_STACK_KNOB_DEFAULTS.auditParallelism,
1074
+ })
1075
+ : { ok: false as const, done: [], error: image.error };
1076
+ if (!r.ok) {
1077
+ s.stop(red(`Failed: ${r.error}`));
1078
+ log.error("Re-run the wizard once the cause is fixed (this step is idempotent).");
1079
+ cancel("Aborted.");
1080
+ process.exit(1);
1081
+ }
1082
+ s.stop(green(`Quality-audit job ready (${r.done.length} steps)`));
1083
+ }
1084
+
979
1085
  /**
980
1086
  * Whether an upgrader set up by an earlier run exists, so a failed setup
981
1087
  * this time still keeps its caelo-mcp signing grant (the operator-access
@@ -1027,6 +1133,58 @@ async function stepOperatorAccessSync(
1027
1133
  s.stop(green(`Operator-access sync job ready (${r.done.length} steps)`));
1028
1134
  }
1029
1135
 
1136
+ /**
1137
+ * #806 — the CLI's part of remote MCP (remote-mcp.ts): on `gcp` the load
1138
+ * balancer's discovery routes and Cloud Armor rule, on both providers the
1139
+ * env vars naming the admin URL, the broker's public URL and the IAP
1140
+ * audience. Remote MCP stays off until the Owner sets it up under Settings →
1141
+ * MCP, so a failure does not stop the install: it warns, and the next pass
1142
+ * or `upgrade` retries. Idempotent.
1143
+ *
1144
+ * @returns whether every part is in place
1145
+ */
1146
+ async function stepRemoteMcp(
1147
+ projectId: string,
1148
+ region: string,
1149
+ provider: "gcp" | "gcp-firebase",
1150
+ domain: string,
1151
+ ): Promise<boolean> {
1152
+ const s = spinner();
1153
+ s.start(
1154
+ "Preparing remote MCP (ChatGPT / claude.ai connectors; off until the Owner enables it)...",
1155
+ );
1156
+ const name = (slug: "admin" | "gateway") =>
1157
+ resolveGcpResourceName(
1158
+ gcloud,
1159
+ "run-service",
1160
+ `caelo-${GCP_STACK_ENV}-${slug}`,
1161
+ projectId,
1162
+ region,
1163
+ );
1164
+ const [admin, gateway] = [await name("admin"), await name("gateway")];
1165
+ const failed = (error: string) => {
1166
+ s.stop(yellow(`Remote MCP not prepared: ${error}`));
1167
+ log.warn(
1168
+ `The install works; Settings → MCP reports remote MCP as unavailable until ${bold(selfCommand("upgrade"))} prepares it.`,
1169
+ );
1170
+ return false;
1171
+ };
1172
+ if (!admin || !gateway) return failed("the admin or gateway Cloud Run service was not found");
1173
+ const edge = await ensureRemoteMcpEdge({ provider, projectId, env: GCP_STACK_ENV, domain });
1174
+ if (!edge.ok) return failed(edge.error);
1175
+ const env = await ensureRemoteMcpIdentityEnv({
1176
+ provider,
1177
+ projectId,
1178
+ region,
1179
+ env: GCP_STACK_ENV,
1180
+ domain,
1181
+ services: { admin, gateway },
1182
+ });
1183
+ if (!env.ok) return failed(env.error);
1184
+ s.stop(green(`Remote MCP prepared (${env.identity.mcpPublicUrl})`));
1185
+ return true;
1186
+ }
1187
+
1030
1188
  /**
1031
1189
  * Delete the firewall rules GCP pre-populates on the `default` network
1032
1190
  * (SSH, RDP, ICMP, internal; default-firewall.ts). A customised rule of the
@@ -349,10 +349,28 @@ new aws.cloudwatch.LogDelivery(
349
349
  //
350
350
  // Each Caelo service runs as its own Fargate service; sharing a cluster
351
351
  // keeps the per-month Fargate-cluster fee at zero (clusters are free).
352
+ //
353
+ // FIXME(issue #749): no quality-audit task definition yet. On GCP every
354
+ // Lighthouse run of a Stage audit runs in its own container (#875,
355
+ // src/audit-job-setup.ts); the AWS equivalent is a Fargate task per run
356
+ // (admin-core quality/parallel-runner.ts `AuditTaskLauncher`). Until it
357
+ // exists the admin records every Lighthouse audit as failed with that gap
358
+ // (quality/lighthouse-runner-select.ts) instead of measuring on its own
359
+ // shared CPU. Nothing is staged on AWS before #749 either.
360
+ //
361
+ // FIXME(issue #696): no in-cloud upgrader or converge job (#819) yet. On
362
+ // GCP Settings → Updates hands app-roll releases to the upgrader job and
363
+ // cloud-setup releases to the converge job; on AWS the admin refuses a
364
+ // cloud-setup release with the CLI command that installs it (admin-core
365
+ // platform-updates/update-route.ts) until this adapter gains both jobs.
352
366
  const cluster = new aws.ecs.Cluster(`${namePrefix}-ecs`, {
353
367
  settings: [{ name: "containerInsights", value: "enabled" }],
354
368
  });
355
369
 
370
+ // #806 — no remote MCP on AWS: it rides Google IAP (gcp stacks only). The
371
+ // gateway task definition, once it exists, sets none of its env, so /api/mcp/*
372
+ // answers 503 remote_mcp_not_configured and the admin reports it unavailable.
373
+ //
356
374
  // One task role per service (#635): the gateway's may read only the
357
375
  // public_role password, never a secret that unlocks a cms_admin login
358
376
  // (src/aws-task-roles.ts AWS_SERVICE_SECRETS, asserted by
@@ -330,6 +330,21 @@ const logWorkspace = new azure.operationalinsights.Workspace(`${namePrefix}-logs
330
330
  });
331
331
 
332
332
  // === 6. Container Apps environment + apps ===
333
+ //
334
+ // FIXME(issue #750): no quality-audit Container Apps job yet. On GCP every
335
+ // Lighthouse run of a Stage audit runs in its own container (#875,
336
+ // src/audit-job-setup.ts); the Azure equivalent is a Container Apps job
337
+ // with one replica per run (admin-core quality/parallel-runner.ts
338
+ // `AuditTaskLauncher`). Until it exists the admin records every Lighthouse
339
+ // audit as failed with that gap (quality/lighthouse-runner-select.ts)
340
+ // instead of measuring on its own shared CPU. Nothing is staged on Azure
341
+ // before #750 either.
342
+ //
343
+ // FIXME(issue #696): no in-cloud upgrader or converge job (#819) yet. On
344
+ // GCP Settings → Updates hands app-roll releases to the upgrader job and
345
+ // cloud-setup releases to the converge job; on Azure the admin refuses a
346
+ // cloud-setup release with the CLI command that installs it (admin-core
347
+ // platform-updates/update-route.ts) until this adapter gains both jobs.
333
348
  const cappEnv = new azure.app.ManagedEnvironment(`${namePrefix}-capp-env`, {
334
349
  resourceGroupName: rg.name,
335
350
  environmentName: `${namePrefix}-capp-env`,
@@ -421,6 +436,9 @@ const _gatewayApp = containerApp({
421
436
  serviceName: "gateway",
422
437
  databaseEnv: [{ name: "PUBLIC_DATABASE_URL", value: publicDatabaseUrl }],
423
438
  // Which X-Forwarded-For entry is the visitor (api-gateway client-ip.ts).
439
+ // #806 — no remote-MCP env: remote MCP rides Google IAP (gcp stacks
440
+ // only); unset, the gateway answers /api/mcp/* with 503
441
+ // remote_mcp_not_configured and the admin reports it unavailable.
424
442
  extraEnv: [{ name: TRUSTED_PROXY_HOPS_ENV, value: String(GATEWAY_TRUSTED_PROXY_HOPS.azure) }],
425
443
  });
426
444
  const _orchestratorApp = containerApp({