@caelo-cms/provisioning 0.10.55 → 0.10.58

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (221) hide show
  1. package/dist/agent-db.js +97 -24
  2. package/dist/cli-routing.js +1 -1
  3. package/dist/cli-routing.js.map +1 -1
  4. package/dist/cli.js +4 -1
  5. package/dist/cli.js.map +1 -1
  6. package/dist/compose.js +5 -0
  7. package/dist/compose.js.map +1 -1
  8. package/dist/converge-checkpoint.d.ts +66 -0
  9. package/dist/converge-checkpoint.d.ts.map +1 -0
  10. package/dist/converge-checkpoint.js +61 -0
  11. package/dist/converge-checkpoint.js.map +1 -0
  12. package/dist/converge-job-gcp.d.ts +46 -0
  13. package/dist/converge-job-gcp.d.ts.map +1 -0
  14. package/dist/converge-job-gcp.js +368 -0
  15. package/dist/converge-job-gcp.js.map +1 -0
  16. package/dist/converge-job-setup.d.ts +62 -0
  17. package/dist/converge-job-setup.d.ts.map +1 -0
  18. package/dist/converge-job-setup.js +228 -0
  19. package/dist/converge-job-setup.js.map +1 -0
  20. package/dist/converge-job.d.ts +141 -0
  21. package/dist/converge-job.d.ts.map +1 -0
  22. package/dist/converge-job.js +325 -0
  23. package/dist/converge-job.js.map +1 -0
  24. package/dist/gcp-names.d.ts +9 -0
  25. package/dist/gcp-names.d.ts.map +1 -1
  26. package/dist/gcp-names.js +13 -0
  27. package/dist/gcp-names.js.map +1 -1
  28. package/dist/index.d.ts +2 -0
  29. package/dist/index.d.ts.map +1 -1
  30. package/dist/index.js +2 -0
  31. package/dist/index.js.map +1 -1
  32. package/dist/infra-contract.d.ts +1 -1
  33. package/dist/infra-contract.d.ts.map +1 -1
  34. package/dist/infra-contract.js +33 -1
  35. package/dist/infra-contract.js.map +1 -1
  36. package/dist/install-job.d.ts +11 -3
  37. package/dist/install-job.d.ts.map +1 -1
  38. package/dist/install-job.js +22 -12
  39. package/dist/install-job.js.map +1 -1
  40. package/dist/lifecycle.d.ts +13 -1
  41. package/dist/lifecycle.d.ts.map +1 -1
  42. package/dist/lifecycle.js +115 -58
  43. package/dist/lifecycle.js.map +1 -1
  44. package/dist/operator-access.d.ts +4 -0
  45. package/dist/operator-access.d.ts.map +1 -1
  46. package/dist/operator-access.js +43 -7
  47. package/dist/operator-access.js.map +1 -1
  48. package/dist/release-contract.d.ts +3 -2
  49. package/dist/release-contract.d.ts.map +1 -1
  50. package/dist/release-contract.js +5 -2
  51. package/dist/release-contract.js.map +1 -1
  52. package/dist/release-manifest.d.ts +2 -0
  53. package/dist/release-manifest.d.ts.map +1 -1
  54. package/dist/release-manifest.js.map +1 -1
  55. package/dist/remote-mcp-edge.d.ts +114 -0
  56. package/dist/remote-mcp-edge.d.ts.map +1 -0
  57. package/dist/remote-mcp-edge.js +116 -0
  58. package/dist/remote-mcp-edge.js.map +1 -0
  59. package/dist/remote-mcp.d.ts +139 -0
  60. package/dist/remote-mcp.d.ts.map +1 -0
  61. package/dist/remote-mcp.js +434 -0
  62. package/dist/remote-mcp.js.map +1 -0
  63. package/dist/runtime-secrets.d.ts +12 -0
  64. package/dist/runtime-secrets.d.ts.map +1 -1
  65. package/dist/runtime-secrets.js +32 -4
  66. package/dist/runtime-secrets.js.map +1 -1
  67. package/dist/seal-key-rotation.d.ts +38 -0
  68. package/dist/seal-key-rotation.d.ts.map +1 -0
  69. package/dist/seal-key-rotation.js +206 -0
  70. package/dist/seal-key-rotation.js.map +1 -0
  71. package/dist/stack-contract.d.ts +206 -3
  72. package/dist/stack-contract.d.ts.map +1 -1
  73. package/dist/stack-contract.js +219 -4
  74. package/dist/stack-contract.js.map +1 -1
  75. package/dist/stack-converge.d.ts +53 -1
  76. package/dist/stack-converge.d.ts.map +1 -1
  77. package/dist/stack-converge.js +124 -4
  78. package/dist/stack-converge.js.map +1 -1
  79. package/dist/state-lock-lease.d.ts +142 -0
  80. package/dist/state-lock-lease.d.ts.map +1 -0
  81. package/dist/state-lock-lease.js +347 -0
  82. package/dist/state-lock-lease.js.map +1 -0
  83. package/dist/state-store.d.ts +66 -9
  84. package/dist/state-store.d.ts.map +1 -1
  85. package/dist/state-store.js +249 -30
  86. package/dist/state-store.js.map +1 -1
  87. package/dist/upgrade/app-roll.d.ts +36 -7
  88. package/dist/upgrade/app-roll.d.ts.map +1 -1
  89. package/dist/upgrade/app-roll.js +53 -19
  90. package/dist/upgrade/app-roll.js.map +1 -1
  91. package/dist/upgrade/audit-job-roll.d.ts.map +1 -1
  92. package/dist/upgrade/audit-job-roll.js +3 -1
  93. package/dist/upgrade/audit-job-roll.js.map +1 -1
  94. package/dist/upgrade/converge.d.ts.map +1 -1
  95. package/dist/upgrade/converge.js +125 -12
  96. package/dist/upgrade/converge.js.map +1 -1
  97. package/dist/upgrade/gcp-lookup.d.ts +27 -10
  98. package/dist/upgrade/gcp-lookup.d.ts.map +1 -1
  99. package/dist/upgrade/gcp-lookup.js +46 -7
  100. package/dist/upgrade/gcp-lookup.js.map +1 -1
  101. package/dist/upgrade/record-rolled.d.ts +40 -0
  102. package/dist/upgrade/record-rolled.d.ts.map +1 -0
  103. package/dist/upgrade/record-rolled.js +46 -0
  104. package/dist/upgrade/record-rolled.js.map +1 -0
  105. package/dist/upgrade/upgrade.d.ts +45 -1
  106. package/dist/upgrade/upgrade.d.ts.map +1 -1
  107. package/dist/upgrade/upgrade.js +60 -17
  108. package/dist/upgrade/upgrade.js.map +1 -1
  109. package/dist/upgrader-assess.d.ts +15 -1
  110. package/dist/upgrader-assess.d.ts.map +1 -1
  111. package/dist/upgrader-assess.js +36 -6
  112. package/dist/upgrader-assess.js.map +1 -1
  113. package/dist/upgrader-gcp.d.ts +15 -0
  114. package/dist/upgrader-gcp.d.ts.map +1 -1
  115. package/dist/upgrader-gcp.js +42 -7
  116. package/dist/upgrader-gcp.js.map +1 -1
  117. package/dist/upgrader-iam-preflight.d.ts +123 -0
  118. package/dist/upgrader-iam-preflight.d.ts.map +1 -0
  119. package/dist/upgrader-iam-preflight.js +414 -0
  120. package/dist/upgrader-iam-preflight.js.map +1 -0
  121. package/dist/upgrader-job.d.ts +33 -9
  122. package/dist/upgrader-job.d.ts.map +1 -1
  123. package/dist/upgrader-job.js +169 -27
  124. package/dist/upgrader-job.js.map +1 -1
  125. package/dist/upgrader-records.d.ts +58 -0
  126. package/dist/upgrader-records.d.ts.map +1 -0
  127. package/dist/upgrader-records.js +87 -0
  128. package/dist/upgrader-records.js.map +1 -0
  129. package/dist/upgrader-reporter.d.ts +3 -2
  130. package/dist/upgrader-reporter.d.ts.map +1 -1
  131. package/dist/upgrader-reporter.js +10 -2
  132. package/dist/upgrader-reporter.js.map +1 -1
  133. package/dist/upgrader-setup.d.ts +25 -3
  134. package/dist/upgrader-setup.d.ts.map +1 -1
  135. package/dist/upgrader-setup.js +123 -11
  136. package/dist/upgrader-setup.js.map +1 -1
  137. package/dist/wizards/gcp-cost.d.ts.map +1 -1
  138. package/dist/wizards/gcp-cost.js +11 -3
  139. package/dist/wizards/gcp-cost.js.map +1 -1
  140. package/dist/wizards/gcp-pulumi.d.ts +6 -0
  141. package/dist/wizards/gcp-pulumi.d.ts.map +1 -1
  142. package/dist/wizards/gcp-pulumi.js +14 -2
  143. package/dist/wizards/gcp-pulumi.js.map +1 -1
  144. package/dist/wizards/gcp.d.ts.map +1 -1
  145. package/dist/wizards/gcp.js +106 -4
  146. package/dist/wizards/gcp.js.map +1 -1
  147. package/package.json +2 -2
  148. package/src/agent/run.test.ts +14 -1
  149. package/src/audit-job-setup.test.ts +7 -1
  150. package/src/cli-routing.ts +1 -1
  151. package/src/cli.ts +4 -1
  152. package/src/compose.ts +5 -0
  153. package/src/converge-checkpoint.ts +75 -0
  154. package/src/converge-entrypoint.test.ts +213 -0
  155. package/src/converge-job-gcp.test.ts +222 -0
  156. package/src/converge-job-gcp.ts +443 -0
  157. package/src/converge-job-setup.test.ts +152 -0
  158. package/src/converge-job-setup.ts +268 -0
  159. package/src/converge-job.test.ts +486 -0
  160. package/src/converge-job.ts +542 -0
  161. package/src/gateway-env-isolation.test.ts +8 -1
  162. package/src/gcp-names.ts +15 -0
  163. package/src/gcp-url-map.test.ts +9 -1
  164. package/src/index.ts +8 -0
  165. package/src/infra-contract.test.ts +8 -4
  166. package/src/infra-contract.ts +33 -1
  167. package/src/install-job.test.ts +3 -1
  168. package/src/install-job.ts +31 -14
  169. package/src/lifecycle.ts +137 -71
  170. package/src/operator-access.test.ts +24 -1
  171. package/src/operator-access.ts +44 -6
  172. package/src/release-contract.ts +5 -2
  173. package/src/release-manifest.test.ts +42 -0
  174. package/src/release-manifest.ts +2 -0
  175. package/src/remote-mcp-edge.ts +137 -0
  176. package/src/remote-mcp.test.ts +690 -0
  177. package/src/remote-mcp.ts +524 -0
  178. package/src/runtime-secrets.test.ts +31 -6
  179. package/src/runtime-secrets.ts +41 -3
  180. package/src/seal-key-rotation.test.ts +131 -0
  181. package/src/seal-key-rotation.ts +258 -0
  182. package/src/stack-contract-parity.test.ts +20 -3
  183. package/src/stack-contract.test.ts +32 -2
  184. package/src/stack-contract.ts +363 -4
  185. package/src/stack-converge-secrets.test.ts +4 -0
  186. package/src/stack-converge.test.ts +21 -5
  187. package/src/stack-converge.ts +162 -7
  188. package/src/state-attach.test.ts +7 -1
  189. package/src/state-lock-lease.test.ts +302 -0
  190. package/src/state-lock-lease.ts +423 -0
  191. package/src/state-lock.test.ts +568 -0
  192. package/src/state-store.ts +347 -27
  193. package/src/upgrade/app-roll.ts +102 -21
  194. package/src/upgrade/audit-job-roll.ts +3 -1
  195. package/src/upgrade/converge.ts +148 -10
  196. package/src/upgrade/gcp-lookup.test.ts +80 -0
  197. package/src/upgrade/gcp-lookup.ts +65 -8
  198. package/src/upgrade/record-rolled.test.ts +105 -0
  199. package/src/upgrade/record-rolled.ts +91 -0
  200. package/src/upgrade/upgrade.test.ts +203 -3
  201. package/src/upgrade/upgrade.ts +120 -24
  202. package/src/upgrader-assess.test.ts +92 -3
  203. package/src/upgrader-assess.ts +58 -9
  204. package/src/upgrader-gcp.ts +47 -8
  205. package/src/upgrader-iam-preflight.test.ts +393 -0
  206. package/src/upgrader-iam-preflight.ts +510 -0
  207. package/src/upgrader-job.test.ts +586 -15
  208. package/src/upgrader-job.ts +237 -53
  209. package/src/upgrader-records.ts +151 -0
  210. package/src/upgrader-reporter.test.ts +6 -0
  211. package/src/upgrader-reporter.ts +7 -2
  212. package/src/upgrader-setup.test.ts +3 -3
  213. package/src/upgrader-setup.ts +128 -11
  214. package/src/wizards/gcp-cost.ts +12 -3
  215. package/src/wizards/gcp-pulumi.test.ts +15 -0
  216. package/src/wizards/gcp-pulumi.ts +18 -2
  217. package/src/wizards/gcp.ts +134 -4
  218. package/stacks/aws/index.ts +10 -0
  219. package/stacks/azure/index.ts +9 -0
  220. package/stacks/gcp/index.ts +47 -7
  221. package/stacks/gcp-firebase/index.ts +38 -6
@@ -20,12 +20,13 @@
20
20
  * the bucket store shells out to `gcloud storage` synchronously.
21
21
  */
22
22
  import { spawnSync } from "node:child_process";
23
+ import { randomUUID } from "node:crypto";
23
24
  import { existsSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
24
- import { hostname } from "node:os";
25
25
  import { join } from "node:path";
26
26
  import { assertNoCheckpointSecrets, assertNoSecrets } from "./checkpoint-secrets.js";
27
27
  import { gcloudEnv } from "./gcloud.js";
28
28
  import { OPS_INSTALL_PREFIX } from "./state-backend.js";
29
+ import { describeExecutionArgs, judgeHeldLock, lockBody, PROCESS_LEASE_MS, parseExecutionState, parseLockBody, renewedLockBody, } from "./state-lock-lease.js";
29
30
  /** The real {@link SyncGcloud}: `gcloud` with prompts disabled. */
30
31
  export const realSyncGcloud = (args, input) => {
31
32
  const r = spawnSync("gcloud", args, {
@@ -150,51 +151,239 @@ export class StateLockHeldError extends Error {
150
151
  export function bucketStateLockUrl(bucket) {
151
152
  return `gs://${bucket}/${OPS_INSTALL_PREFIX}/state.lock`;
152
153
  }
153
- /** Who holds a lock, for the refusal message. */
154
- function lockBody(purpose) {
155
- return JSON.stringify({
156
- purpose,
157
- host: hostname(),
158
- pid: process.pid,
159
- startedAt: new Date().toISOString(),
160
- });
161
- }
154
+ /** gcloud's wording for a failed `--if-generation-match` precondition. */
155
+ const PRECONDITION_FAILED = /PreconditionFailed|412|At least one of the pre-conditions/i;
156
+ const defaultSchedule = (tick, everyMs) => {
157
+ const timer = setInterval(tick, everyMs);
158
+ // The heartbeat must never keep a finished CLI alive.
159
+ timer.unref?.();
160
+ return () => clearInterval(timer);
161
+ };
162
162
  /**
163
163
  * The install's state lock in its ops bucket, created only if absent
164
164
  * (`--if-generation-match=0`). Held by every writer of a bucket-backed
165
- * install's state — an installer run's `pulumi up`, `state migrate` — so
166
- * none writes a backend another is switching away from. A lock left by a
167
- * crashed run is reported with the command to remove it, never broken
168
- * automatically.
165
+ * install's state — an installer run's `pulumi up`, `state migrate`, the
166
+ * CLI `upgrade`, the upgrader job — so none writes a backend another is
167
+ * switching away from.
169
168
  *
170
- * @returns the release function
169
+ * A lock that is held is judged (state-lock-lease.ts): one whose Cloud Run
170
+ * execution has finished, whose execution is gone after its lease, whose
171
+ * CLI process is gone on this host, or whose CLI lease ran out unrenewed,
172
+ * is broken with a generation-matched delete and taken once more. The
173
+ * generation is read BEFORE the body, so the delete can only remove the
174
+ * lock that was judged: if another run replaced it in between, the
175
+ * delete's precondition fails and nothing is removed. Two runs breaking the
176
+ * same dead lock cannot both win either — the slower one's create fails
177
+ * its own precondition and reports the lock as held.
178
+ *
179
+ * A CLI holder renews its lease every third of {@link PROCESS_LEASE_MS}
180
+ * with a generation-matched rewrite, so a run that is alive keeps its lock
181
+ * however long it takes, and a killed one loses it within one lease.
182
+ *
183
+ * @returns the release function; it stops the heartbeat and deletes only
184
+ * this lock's generation
185
+ * @throws StateLockHeldError when another run holds the lock
186
+ * @throws Error when the lock or its holder cannot be read (no access)
171
187
  */
172
- export function acquireBucketStateLock(bucket, purpose, run = realSyncGcloud) {
188
+ export function acquireBucketStateLock(bucket, purpose, run = realSyncGcloud, opts = {}) {
173
189
  const url = bucketStateLockUrl(bucket);
174
- const r = run(["storage", "cp", "-", url, "--if-generation-match=0"], lockBody(purpose));
175
- if (!r.ok) {
176
- if (/PreconditionFailed|412|At least one of the pre-conditions/i.test(r.stderr)) {
177
- const held = run(["storage", "cat", url]);
178
- throw new StateLockHeldError(`Another run is changing this install's state (${held.ok ? held.stdout.trim() : url}). Re-run once it has finished; if none is running, remove the stale lock: gcloud storage rm ${url}`);
190
+ const now = opts.now ?? (() => new Date());
191
+ const holder = opts.holder ?? { leaseMs: PROCESS_LEASE_MS };
192
+ const id = randomUUID();
193
+ const create = () => run(["storage", "cp", "-", url, "--if-generation-match=0"], lockBody(purpose, now(), holder, id));
194
+ let r = create();
195
+ if (!r.ok && PRECONDITION_FAILED.test(r.stderr)) {
196
+ breakDeadLock(url, run, now(), opts.onBreak, opts.here);
197
+ r = create();
198
+ if (!r.ok && PRECONDITION_FAILED.test(r.stderr)) {
199
+ throw new StateLockHeldError(`Another run took this install's state lock at the same moment (${readLockText(url, run)}). Re-run once it has finished.`);
179
200
  }
201
+ }
202
+ if (!r.ok)
180
203
  throw new Error(`Could not take the state lock ${url}: ${r.stderr.trim()}`);
204
+ const owned = ownedGeneration(url, run, id);
205
+ if (owned.kind !== "owned") {
206
+ // Ours, just created — but without its generation a delete could hit a
207
+ // successor's lock. Give it back only once a second look proves it is
208
+ // still ours at a known generation; otherwise its lease ends it.
209
+ const retry = ownedGeneration(url, run, id);
210
+ if (retry.kind === "owned") {
211
+ run(["storage", "rm", url, `--if-generation-match=${retry.generation}`]);
212
+ }
213
+ throw new Error(`Could not read the generation of the state lock ${url} this run just took (${owned.why}). ${retry.kind === "owned"
214
+ ? "It was given back."
215
+ : "It was left in place and expires with its lease; re-run after that."}`);
181
216
  }
217
+ let generation = owned.generation;
218
+ let lost = null;
219
+ const stopHeartbeat = "execution" in holder
220
+ ? () => { }
221
+ : (opts.schedule ?? defaultSchedule)(() => {
222
+ if (lost !== null)
223
+ return;
224
+ const renewed = renewLock(url, run, id, generation, now(), holder.leaseMs);
225
+ if (renewed.kind === "owned") {
226
+ generation = renewed.generation;
227
+ return;
228
+ }
229
+ if (renewed.kind === "lost") {
230
+ lost = renewed.why;
231
+ opts.onLost?.(`This run no longer holds the install's state lock (${renewed.why}). Stop it and re-run once the other run has finished.`);
232
+ }
233
+ // "error": a transient failure; the next beat tries again within the lease.
234
+ }, Math.floor(holder.leaseMs / 3));
182
235
  return () => {
183
- const rm = run(["storage", "rm", url]);
184
- if (!rm.ok)
185
- throw new Error(`Could not release ${url}; remove it: gcloud storage rm ${url}`);
236
+ stopHeartbeat();
237
+ const rm = run(["storage", "rm", url, `--if-generation-match=${generation}`]);
238
+ if (rm.ok)
239
+ return;
240
+ if (PRECONDITION_FAILED.test(rm.stderr) || NOT_FOUND.test(rm.stderr)) {
241
+ throw new Error(`The state lock ${url} was no longer this run's when it finished (another run broke it); it was left alone.`);
242
+ }
243
+ throw new Error(`Could not release ${url}; remove it: gcloud storage rm ${url}`);
186
244
  };
187
245
  }
246
+ /**
247
+ * The lock's generation, provided the lock is still the one with `id`. The
248
+ * generation is read before the body: if another run replaced the lock in
249
+ * between, the body names its id and this returns "lost" — never a
250
+ * generation that is not ours.
251
+ */
252
+ function ownedGeneration(url, run, id) {
253
+ const g = lockGeneration(url, run);
254
+ if (g.kind === "absent")
255
+ return { kind: "lost", why: "the lock is gone" };
256
+ if (g.kind === "error")
257
+ return { kind: "error", why: g.error };
258
+ const body = run(["storage", "cat", url]);
259
+ if (!body.ok) {
260
+ return NOT_FOUND.test(body.stderr)
261
+ ? { kind: "lost", why: "the lock is gone" }
262
+ : { kind: "error", why: body.stderr.trim() || `gcloud exited ${body.exitCode}` };
263
+ }
264
+ const parsed = parseLockBody(body.stdout);
265
+ if (parsed?.id !== id) {
266
+ return { kind: "lost", why: `another run holds it now (${body.stdout.trim().slice(0, 200)})` };
267
+ }
268
+ return { kind: "owned", generation: g.generation };
269
+ }
270
+ /**
271
+ * One heartbeat: rewrite the lock with a later lease, only if it is still
272
+ * at `generation` (ours), and return its new generation.
273
+ */
274
+ function renewLock(url, run, id, generation, now, leaseMs) {
275
+ const current = run(["storage", "cat", url]);
276
+ if (!current.ok) {
277
+ return NOT_FOUND.test(current.stderr)
278
+ ? { kind: "lost", why: "the lock is gone" }
279
+ : { kind: "error", why: current.stderr.trim() };
280
+ }
281
+ const body = parseLockBody(current.stdout);
282
+ if (body?.id !== id) {
283
+ return {
284
+ kind: "lost",
285
+ why: `another run holds it now (${current.stdout.trim().slice(0, 200)})`,
286
+ };
287
+ }
288
+ const w = run(["storage", "cp", "-", url, `--if-generation-match=${generation}`], renewedLockBody(body, now, leaseMs));
289
+ if (!w.ok) {
290
+ return PRECONDITION_FAILED.test(w.stderr)
291
+ ? { kind: "lost", why: "another run replaced it" }
292
+ : { kind: "error", why: w.stderr.trim() };
293
+ }
294
+ return ownedGeneration(url, run, id);
295
+ }
296
+ /** The lock object's current generation. */
297
+ function lockGeneration(url, run) {
298
+ const d = run(["storage", "objects", "describe", url, "--format=json"]);
299
+ if (!d.ok) {
300
+ return NOT_FOUND.test(d.stderr)
301
+ ? { kind: "absent" }
302
+ : { kind: "error", error: d.stderr.trim() || `gcloud exited ${d.exitCode}` };
303
+ }
304
+ try {
305
+ // `gcloud storage objects describe` prints the generation as a string
306
+ // (`generation: '1706304784726341'`, cloud.google.com/storage/docs/
307
+ // viewing-editing-metadata); a JSON number would lose digits past 2^53,
308
+ // so only a string of digits is a generation the delete may match.
309
+ const g = JSON.parse(d.stdout).generation;
310
+ if (typeof g === "string" && /^[1-9][0-9]*$/.test(g)) {
311
+ return { kind: "found", generation: g };
312
+ }
313
+ }
314
+ catch {
315
+ // Reported below with the same message as a missing field.
316
+ }
317
+ return { kind: "error", error: "gcloud described the lock without a generation" };
318
+ }
319
+ function readLockText(url, run) {
320
+ const held = run(["storage", "cat", url]);
321
+ return held.ok ? held.stdout.trim() : url;
322
+ }
323
+ /**
324
+ * Judge the held lock at `url` and break it when its holder is dead.
325
+ * Returns normally when the lock was broken or vanished meanwhile, so the
326
+ * caller creates it once more; throws when it is live or cannot be judged.
327
+ */
328
+ function breakDeadLock(url, run, now, onBreak, here) {
329
+ const found = lockGeneration(url, run);
330
+ if (found.kind === "absent")
331
+ return;
332
+ if (found.kind === "error") {
333
+ throw new Error(`Could not read the state lock ${url} another run holds: ${found.error}`);
334
+ }
335
+ const held = run(["storage", "cat", url]);
336
+ if (!held.ok) {
337
+ if (NOT_FOUND.test(held.stderr))
338
+ return;
339
+ throw new Error(`Could not read the state lock ${url}: ${held.stderr.trim()}`);
340
+ }
341
+ const verdict = judgeHeldLock(held.stdout, (ref) => parseExecutionState(run(describeExecutionArgs(ref))), now, here);
342
+ if (verdict.kind === "unknown") {
343
+ // Not contention: retrying will not help, and guessing "dead" could let
344
+ // two runs write the same state.
345
+ throw new Error(`The state lock ${url} is held and ${verdict.why}`);
346
+ }
347
+ if (verdict.kind === "held") {
348
+ throw new StateLockHeldError(verdict.manualRemoval
349
+ ? `Another run is changing this install's state (${verdict.why}). Re-run once it has finished; if none is running, remove the stale lock: gcloud storage rm ${url}`
350
+ : `Another run is changing this install's state: ${verdict.why}. Re-run once it has finished.`);
351
+ }
352
+ const rm = run(["storage", "rm", url, `--if-generation-match=${found.generation}`]);
353
+ if (!rm.ok && !NOT_FOUND.test(rm.stderr)) {
354
+ if (PRECONDITION_FAILED.test(rm.stderr)) {
355
+ throw new StateLockHeldError(`Another run took this install's state lock while this one removed a stale one (${readLockText(url, run)}). Re-run once it has finished.`);
356
+ }
357
+ throw new Error(`Could not remove the stale state lock ${url}: ${rm.stderr.trim()}`);
358
+ }
359
+ onBreak?.(`Removed a stale state lock: ${verdict.why}.`);
360
+ }
188
361
  /**
189
362
  * The same lock for an install whose state is on this machine:
190
- * `~/.caelo-<id>/state.lock`, created exclusively.
363
+ * `~/.caelo-<id>/state.lock`, created exclusively. A lock left by a
364
+ * process that is gone on this machine is taken over (#815).
191
365
  *
366
+ * @param here this host and its pid check; tests pass a fake
192
367
  * @returns the release function
193
368
  */
194
- export function acquireLocalStateLock(root, purpose) {
369
+ export function acquireLocalStateLock(root, purpose, here) {
195
370
  const path = join(root, "state.lock");
371
+ const take = () => writeFileSync(path, lockBody(purpose, new Date()), { flag: "wx", mode: 0o600 });
196
372
  try {
197
- writeFileSync(path, lockBody(purpose), { flag: "wx", mode: 0o600 });
373
+ try {
374
+ take();
375
+ }
376
+ catch (e) {
377
+ if (e.code !== "EEXIST")
378
+ throw e;
379
+ // #815 — the file is on this machine, so a holder whose process has
380
+ // ended is known dead: take its lock over instead of asking for `rm`.
381
+ const verdict = judgeHeldLock(readFileSync(path, "utf8"), () => ({ kind: "unknown", error: "a machine-local lock names no Cloud Run execution" }), new Date(), here);
382
+ if (verdict.kind !== "break")
383
+ throw e;
384
+ rmSync(path, { force: true });
385
+ take();
386
+ }
198
387
  }
199
388
  catch (e) {
200
389
  if (e.code === "EEXIST") {
@@ -205,9 +394,39 @@ export function acquireLocalStateLock(root, purpose) {
205
394
  return () => rmSync(path, { force: true });
206
395
  }
207
396
  /** The state lock for whichever backend `bucket` (gcs) or `root` (local) names. */
208
- export function acquireStateLock(backend, root, purpose, run = realSyncGcloud) {
397
+ export function acquireStateLock(backend, root, purpose, run = realSyncGcloud, opts = {}) {
209
398
  return backend.kind === "gcs"
210
- ? acquireBucketStateLock(backend.bucket, purpose, run)
211
- : acquireLocalStateLock(root, purpose);
399
+ ? acquireBucketStateLock(backend.bucket, purpose, run, opts)
400
+ : acquireLocalStateLock(root, purpose, opts.here);
401
+ }
402
+ /**
403
+ * Give the state lock back when the CLI is stopped (Ctrl-C, a closed Cloud
404
+ * Shell tab's SIGTERM), then exit with the signal's conventional code
405
+ * (#815). Without this a stopped run leaves its lock until its lease ends.
406
+ *
407
+ * @param report told what happened to the lock
408
+ * @returns removes the handlers; call it once the run released the lock itself
409
+ */
410
+ export function releaseOnSignal(release, report, proc = process) {
411
+ const dispose = () => {
412
+ proc.removeListener("SIGINT", onInt);
413
+ proc.removeListener("SIGTERM", onTerm);
414
+ };
415
+ const stop = (signal, code) => {
416
+ dispose();
417
+ try {
418
+ release();
419
+ report(`Stopped (${signal}); the install's state lock was released.`);
420
+ }
421
+ catch (e) {
422
+ report(`Stopped (${signal}); ${e instanceof Error ? e.message : String(e)}`);
423
+ }
424
+ return proc.exit(code);
425
+ };
426
+ const onInt = () => stop("SIGINT", 130);
427
+ const onTerm = () => stop("SIGTERM", 143);
428
+ proc.once("SIGINT", onInt);
429
+ proc.once("SIGTERM", onTerm);
430
+ return dispose;
212
431
  }
213
432
  //# sourceMappingURL=state-store.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"state-store.js","sourceRoot":"","sources":["../src/state-store.ts"],"names":[],"mappings":"AAAA,mCAAmC;AAEnC;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACtF,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,yBAAyB,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AACrF,OAAO,EAAqB,SAAS,EAAE,MAAM,aAAa,CAAC;AAC3D,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAiBxD,mEAAmE;AACnE,MAAM,CAAC,MAAM,cAAc,GAAe,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;IACxD,MAAM,CAAC,GAAG,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE;QAClC,GAAG,EAAE,SAAS,EAAE;QAChB,QAAQ,EAAE,MAAM;QAChB,SAAS,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;QAC3B,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;KAC7E,CAAC,CAAC;IACH,IAAI,CAAC,CAAC,KAAK;QAAE,MAAM,CAAC,CAAC,KAAK,CAAC;IAC3B,OAAO;QACL,EAAE,EAAE,CAAC,CAAC,MAAM,KAAK,CAAC;QAClB,MAAM,EAAE,CAAC,CAAC,MAAM,IAAI,EAAE;QACtB,MAAM,EAAE,CAAC,CAAC,MAAM,IAAI,EAAE;QACtB,QAAQ,EAAE,CAAC,CAAC,MAAM,IAAI,CAAC;KACxB,CAAC;AACJ,CAAC,CAAC;AAEF;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAe,EAAE,KAAc;IAC9D,IAAI,IAAI,KAAK,cAAc;QAAE,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;;QACrD,yBAAyB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC5C,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC;AAC/C,CAAC;AAED,yEAAyE;AACzE,SAAS,eAAe,CAAC,IAAY,EAAE,IAAY;IACjD,MAAM,GAAG,GAAG,GAAG,IAAI,QAAQ,OAAO,CAAC,GAAG,EAAE,CAAC;IACzC,IAAI,CAAC;QACH,aAAa,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAC1C,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IACxB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7B,MAAM,CAAC,CAAC;IACV,CAAC;AACH,CAAC;AAED,wDAAwD;AACxD,MAAM,OAAO,eAAe;IAEG;IADpB,IAAI,GAAG,OAAgB,CAAC;IACjC,YAA6B,IAAY;QAAZ,SAAI,GAAJ,IAAI,CAAQ;IAAG,CAAC;IAE7C,IAAI,CAAC,IAAe;QAClB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACnC,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,IAAe,EAAE,KAAc;QACnC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,gBAAgB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;IACxE,CAAC;CACF;AAED,wDAAwD;AACxD,MAAM,SAAS,GAAG,iEAAiE,CAAC;AAEpF;;;;;;GAMG;AACH,MAAM,OAAO,aAAa;IAKL;IACA;IACA;IANV,IAAI,GAAG,KAAc,CAAC;IACd,KAAK,GAAG,IAAI,GAAG,EAA4B,CAAC;IAE7D,YACmB,MAAc,EACd,IAAY,EACZ,MAAkB,cAAc;QAFhC,WAAM,GAAN,MAAM,CAAQ;QACd,SAAI,GAAJ,IAAI,CAAQ;QACZ,QAAG,GAAH,GAAG,CAA6B;IAChD,CAAC;IAEJ,sCAAsC;IACtC,SAAS,CAAC,IAAe;QACvB,OAAO,QAAQ,IAAI,CAAC,MAAM,IAAI,kBAAkB,IAAI,IAAI,EAAE,CAAC;IAC7D,CAAC;IAED;;;;OAIG;IACH,IAAI,CAAC,IAAe;QAClB,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC7D,IAAI,CAAC,CAAC,EAAE;YAAE,OAAO,CAAC,CAAC,MAAM,CAAC;QAC1B,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;YAAE,OAAO,IAAI,CAAC;QAC1C,MAAM,IAAI,KAAK,CACb,kBAAkB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,iBAAiB,CAAC,CAAC,QAAQ,EAAE,2GAA2G,CACvM,CAAC;IACJ,CAAC;IAED,IAAI,CAAC,IAAe;QAClB,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC;QAC9D,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC3B,IAAI,IAAI,KAAK,IAAI;YAAE,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC;QAChE,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,IAAe,EAAE,KAAc;QACnC,MAAM,IAAI,GAAG,gBAAgB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC3C,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAChB,CAAC,SAAS,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,iCAAiC,CAAC,EAC/E,IAAI,CACL,CAAC;QACF,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CACb,mBAAmB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,iBAAiB,CAAC,CAAC,QAAQ,EAAE,yEAAyE,CACtK,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC3B,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC;IAC/C,CAAC;CACF;AAED;;;;GAIG;AACH,MAAM,OAAO,kBAAmB,SAAQ,KAAK;IACzB,IAAI,GAAG,oBAAoB,CAAC;CAC/C;AAED,+EAA+E;AAC/E,MAAM,UAAU,kBAAkB,CAAC,MAAc;IAC/C,OAAO,QAAQ,MAAM,IAAI,kBAAkB,aAAa,CAAC;AAC3D,CAAC;AAED,iDAAiD;AACjD,SAAS,QAAQ,CAAC,OAAe;IAC/B,OAAO,IAAI,CAAC,SAAS,CAAC;QACpB,OAAO;QACP,IAAI,EAAE,QAAQ,EAAE;QAChB,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACpC,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,sBAAsB,CACpC,MAAc,EACd,OAAe,EACf,MAAkB,cAAc;IAEhC,MAAM,GAAG,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC;IACvC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,SAAS,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,yBAAyB,CAAC,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;IACzF,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QACV,IAAI,4DAA4D,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;YAChF,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;YAC1C,MAAM,IAAI,kBAAkB,CAC1B,iDAAiD,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,gGAAgG,GAAG,EAAE,CACzL,CAAC;QACJ,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,iCAAiC,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IAC9E,CAAC;IACD,OAAO,GAAG,EAAE;QACV,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,SAAS,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;QACvC,IAAI,CAAC,EAAE,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,qBAAqB,GAAG,kCAAkC,GAAG,EAAE,CAAC,CAAC;IAC/F,CAAC,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,qBAAqB,CAAC,IAAY,EAAE,OAAe;IACjE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;IACtC,IAAI,CAAC;QACH,aAAa,CAAC,IAAI,EAAE,QAAQ,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACtE,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAK,CAA2B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACnD,MAAM,IAAI,kBAAkB,CAC1B,iDAAiD,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,iFAAiF,IAAI,EAAE,CAC1K,CAAC;QACJ,CAAC;QACD,MAAM,CAAC,CAAC;IACV,CAAC;IACD,OAAO,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AAC7C,CAAC;AAED,mFAAmF;AACnF,MAAM,UAAU,gBAAgB,CAC9B,OAAuF,EACvF,IAAY,EACZ,OAAe,EACf,MAAkB,cAAc;IAEhC,OAAO,OAAO,CAAC,IAAI,KAAK,KAAK;QAC3B,CAAC,CAAC,sBAAsB,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,GAAG,CAAC;QACtD,CAAC,CAAC,qBAAqB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AAC3C,CAAC"}
1
+ {"version":3,"file":"state-store.js","sourceRoot":"","sources":["../src/state-store.ts"],"names":[],"mappings":"AAAA,mCAAmC;AAEnC;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACtF,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,yBAAyB,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AACrF,OAAO,EAAqB,SAAS,EAAE,MAAM,aAAa,CAAC;AAC3D,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EACL,qBAAqB,EAErB,aAAa,EAGb,QAAQ,EACR,gBAAgB,EAChB,mBAAmB,EACnB,aAAa,EACb,eAAe,GAChB,MAAM,uBAAuB,CAAC;AAiB/B,mEAAmE;AACnE,MAAM,CAAC,MAAM,cAAc,GAAe,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;IACxD,MAAM,CAAC,GAAG,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE;QAClC,GAAG,EAAE,SAAS,EAAE;QAChB,QAAQ,EAAE,MAAM;QAChB,SAAS,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;QAC3B,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;KAC7E,CAAC,CAAC;IACH,IAAI,CAAC,CAAC,KAAK;QAAE,MAAM,CAAC,CAAC,KAAK,CAAC;IAC3B,OAAO;QACL,EAAE,EAAE,CAAC,CAAC,MAAM,KAAK,CAAC;QAClB,MAAM,EAAE,CAAC,CAAC,MAAM,IAAI,EAAE;QACtB,MAAM,EAAE,CAAC,CAAC,MAAM,IAAI,EAAE;QACtB,QAAQ,EAAE,CAAC,CAAC,MAAM,IAAI,CAAC;KACxB,CAAC;AACJ,CAAC,CAAC;AAEF;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAe,EAAE,KAAc;IAC9D,IAAI,IAAI,KAAK,cAAc;QAAE,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;;QACrD,yBAAyB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC5C,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC;AAC/C,CAAC;AAED,yEAAyE;AACzE,SAAS,eAAe,CAAC,IAAY,EAAE,IAAY;IACjD,MAAM,GAAG,GAAG,GAAG,IAAI,QAAQ,OAAO,CAAC,GAAG,EAAE,CAAC;IACzC,IAAI,CAAC;QACH,aAAa,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAC1C,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IACxB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7B,MAAM,CAAC,CAAC;IACV,CAAC;AACH,CAAC;AAED,wDAAwD;AACxD,MAAM,OAAO,eAAe;IAEG;IADpB,IAAI,GAAG,OAAgB,CAAC;IACjC,YAA6B,IAAY;QAAZ,SAAI,GAAJ,IAAI,CAAQ;IAAG,CAAC;IAE7C,IAAI,CAAC,IAAe;QAClB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACnC,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,IAAe,EAAE,KAAc;QACnC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,gBAAgB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;IACxE,CAAC;CACF;AAED,wDAAwD;AACxD,MAAM,SAAS,GAAG,iEAAiE,CAAC;AAEpF;;;;;;GAMG;AACH,MAAM,OAAO,aAAa;IAKL;IACA;IACA;IANV,IAAI,GAAG,KAAc,CAAC;IACd,KAAK,GAAG,IAAI,GAAG,EAA4B,CAAC;IAE7D,YACmB,MAAc,EACd,IAAY,EACZ,MAAkB,cAAc;QAFhC,WAAM,GAAN,MAAM,CAAQ;QACd,SAAI,GAAJ,IAAI,CAAQ;QACZ,QAAG,GAAH,GAAG,CAA6B;IAChD,CAAC;IAEJ,sCAAsC;IACtC,SAAS,CAAC,IAAe;QACvB,OAAO,QAAQ,IAAI,CAAC,MAAM,IAAI,kBAAkB,IAAI,IAAI,EAAE,CAAC;IAC7D,CAAC;IAED;;;;OAIG;IACH,IAAI,CAAC,IAAe;QAClB,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC7D,IAAI,CAAC,CAAC,EAAE;YAAE,OAAO,CAAC,CAAC,MAAM,CAAC;QAC1B,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;YAAE,OAAO,IAAI,CAAC;QAC1C,MAAM,IAAI,KAAK,CACb,kBAAkB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,iBAAiB,CAAC,CAAC,QAAQ,EAAE,2GAA2G,CACvM,CAAC;IACJ,CAAC;IAED,IAAI,CAAC,IAAe;QAClB,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC;QAC9D,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC3B,IAAI,IAAI,KAAK,IAAI;YAAE,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC;QAChE,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,IAAe,EAAE,KAAc;QACnC,MAAM,IAAI,GAAG,gBAAgB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC3C,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAChB,CAAC,SAAS,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,iCAAiC,CAAC,EAC/E,IAAI,CACL,CAAC;QACF,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CACb,mBAAmB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,iBAAiB,CAAC,CAAC,QAAQ,EAAE,yEAAyE,CACtK,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC3B,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC;IAC/C,CAAC;CACF;AAED;;;;GAIG;AACH,MAAM,OAAO,kBAAmB,SAAQ,KAAK;IACzB,IAAI,GAAG,oBAAoB,CAAC;CAC/C;AAED,+EAA+E;AAC/E,MAAM,UAAU,kBAAkB,CAAC,MAAc;IAC/C,OAAO,QAAQ,MAAM,IAAI,kBAAkB,aAAa,CAAC;AAC3D,CAAC;AAED,0EAA0E;AAC1E,MAAM,mBAAmB,GAAG,4DAA4D,CAAC;AA0BzF,MAAM,eAAe,GAAG,CAAC,IAAgB,EAAE,OAAe,EAAgB,EAAE;IAC1E,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACzC,sDAAsD;IACtD,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;IAChB,OAAO,GAAG,EAAE,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;AACpC,CAAC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,UAAU,sBAAsB,CACpC,MAAc,EACd,OAAe,EACf,MAAkB,cAAc,EAChC,OAA0B,EAAE;IAE5B,MAAM,GAAG,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC;IACvC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAe,IAAI,CAAC,MAAM,IAAI,EAAE,OAAO,EAAE,gBAAgB,EAAE,CAAC;IACxE,MAAM,EAAE,GAAG,UAAU,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,GAAG,EAAE,CAClB,GAAG,CACD,CAAC,SAAS,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,yBAAyB,CAAC,EACtD,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC,CACrC,CAAC;IACJ,IAAI,CAAC,GAAG,MAAM,EAAE,CAAC;IACjB,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;QAChD,aAAa,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QACxD,CAAC,GAAG,MAAM,EAAE,CAAC;QACb,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;YAChD,MAAM,IAAI,kBAAkB,CAC1B,kEAAkE,YAAY,CAAC,GAAG,EAAE,GAAG,CAAC,iCAAiC,CAC1H,CAAC;QACJ,CAAC;IACH,CAAC;IACD,IAAI,CAAC,CAAC,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IACvF,MAAM,KAAK,GAAG,eAAe,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;IAC5C,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC3B,uEAAuE;QACvE,sEAAsE;QACtE,iEAAiE;QACjE,MAAM,KAAK,GAAG,eAAe,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;QAC5C,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC3B,GAAG,CAAC,CAAC,SAAS,EAAE,IAAI,EAAE,GAAG,EAAE,yBAAyB,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QAC3E,CAAC;QACD,MAAM,IAAI,KAAK,CACb,mDAAmD,GAAG,wBAAwB,KAAK,CAAC,GAAG,MACrF,KAAK,CAAC,IAAI,KAAK,OAAO;YACpB,CAAC,CAAC,oBAAoB;YACtB,CAAC,CAAC,qEACN,EAAE,CACH,CAAC;IACJ,CAAC;IACD,IAAI,UAAU,GAAG,KAAK,CAAC,UAAU,CAAC;IAClC,IAAI,IAAI,GAAkB,IAAI,CAAC;IAC/B,MAAM,aAAa,GACjB,WAAW,IAAI,MAAM;QACnB,CAAC,CAAC,GAAG,EAAE,GAAE,CAAC;QACV,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,IAAI,eAAe,CAAC,CAChC,GAAG,EAAE;YACH,IAAI,IAAI,KAAK,IAAI;gBAAE,OAAO;YAC1B,MAAM,OAAO,GAAG,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;YAC3E,IAAI,OAAO,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;gBAC7B,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;gBAChC,OAAO;YACT,CAAC;YACD,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBAC5B,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC;gBACnB,IAAI,CAAC,MAAM,EAAE,CACX,sDAAsD,OAAO,CAAC,GAAG,wDAAwD,CAC1H,CAAC;YACJ,CAAC;YACD,4EAA4E;QAC9E,CAAC,EACD,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,GAAG,CAAC,CAAC,CAC/B,CAAC;IACR,OAAO,GAAG,EAAE;QACV,aAAa,EAAE,CAAC;QAChB,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,SAAS,EAAE,IAAI,EAAE,GAAG,EAAE,yBAAyB,UAAU,EAAE,CAAC,CAAC,CAAC;QAC9E,IAAI,EAAE,CAAC,EAAE;YAAE,OAAO;QAClB,IAAI,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC;YACrE,MAAM,IAAI,KAAK,CACb,kBAAkB,GAAG,uFAAuF,CAC7G,CAAC;QACJ,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,qBAAqB,GAAG,kCAAkC,GAAG,EAAE,CAAC,CAAC;IACnF,CAAC,CAAC;AACJ,CAAC;AAOD;;;;;GAKG;AACH,SAAS,eAAe,CAAC,GAAW,EAAE,GAAe,EAAE,EAAU;IAC/D,MAAM,CAAC,GAAG,cAAc,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IACnC,IAAI,CAAC,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,kBAAkB,EAAE,CAAC;IAC1E,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO;QAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;IAC/D,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;IAC1C,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;QACb,OAAO,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;YAChC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,kBAAkB,EAAE;YAC3C,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,iBAAiB,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC;IACrF,CAAC;IACD,MAAM,MAAM,GAAG,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC1C,IAAI,MAAM,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC;QACtB,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,6BAA6B,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC;IACjG,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC;AACrD,CAAC;AAED;;;GAGG;AACH,SAAS,SAAS,CAChB,GAAW,EACX,GAAe,EACf,EAAU,EACV,UAAkB,EAClB,GAAS,EACT,OAAe;IAEf,MAAM,OAAO,GAAG,GAAG,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;IAC7C,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC;QAChB,OAAO,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YACnC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,kBAAkB,EAAE;YAC3C,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;IACpD,CAAC;IACD,MAAM,IAAI,GAAG,aAAa,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC3C,IAAI,IAAI,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC;QACpB,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,GAAG,EAAE,6BAA6B,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG;SACzE,CAAC;IACJ,CAAC;IACD,MAAM,CAAC,GAAG,GAAG,CACX,CAAC,SAAS,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,yBAAyB,UAAU,EAAE,CAAC,EAClE,eAAe,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,CACpC,CAAC;IACF,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QACV,OAAO,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;YACvC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,yBAAyB,EAAE;YAClD,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;IAC9C,CAAC;IACD,OAAO,eAAe,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;AACvC,CAAC;AAOD,4CAA4C;AAC5C,SAAS,cAAc,CAAC,GAAW,EAAE,GAAe;IAClD,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,EAAE,eAAe,CAAC,CAAC,CAAC;IACxE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QACV,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;YAC7B,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE;YACpB,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,iBAAiB,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC;IACjF,CAAC;IACD,IAAI,CAAC;QACH,sEAAsE;QACtE,oEAAoE;QACpE,wEAAwE;QACxE,mEAAmE;QACnE,MAAM,CAAC,GAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAA8B,CAAC,UAAU,CAAC;QACxE,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACrD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC;QAC1C,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,2DAA2D;IAC7D,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,gDAAgD,EAAE,CAAC;AACpF,CAAC;AAED,SAAS,YAAY,CAAC,GAAW,EAAE,GAAe;IAChD,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;IAC1C,OAAO,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;AAC5C,CAAC;AAED;;;;GAIG;AACH,SAAS,aAAa,CACpB,GAAW,EACX,GAAe,EACf,GAAS,EACT,OAAgD,EAChD,IAA2B;IAE3B,MAAM,KAAK,GAAG,cAAc,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IACvC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO;IACpC,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CAAC,iCAAiC,GAAG,uBAAuB,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IAC5F,CAAC;IACD,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;IAC1C,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;QACb,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;YAAE,OAAO;QACxC,MAAM,IAAI,KAAK,CAAC,iCAAiC,GAAG,KAAK,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IACjF,CAAC;IACD,MAAM,OAAO,GAAG,aAAa,CAC3B,IAAI,CAAC,MAAM,EACX,CAAC,GAAG,EAAE,EAAE,CAAC,mBAAmB,CAAC,GAAG,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC,CAAC,EAC7D,GAAG,EACH,IAAI,CACL,CAAC;IACF,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC/B,wEAAwE;QACxE,iCAAiC;QACjC,MAAM,IAAI,KAAK,CAAC,kBAAkB,GAAG,gBAAgB,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IACtE,CAAC;IACD,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC5B,MAAM,IAAI,kBAAkB,CAC1B,OAAO,CAAC,aAAa;YACnB,CAAC,CAAC,iDAAiD,OAAO,CAAC,GAAG,gGAAgG,GAAG,EAAE;YACnK,CAAC,CAAC,iDAAiD,OAAO,CAAC,GAAG,gCAAgC,CACjG,CAAC;IACJ,CAAC;IACD,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,SAAS,EAAE,IAAI,EAAE,GAAG,EAAE,yBAAyB,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;IACpF,IAAI,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC;QACzC,IAAI,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC;YACxC,MAAM,IAAI,kBAAkB,CAC1B,kFAAkF,YAAY,CAAC,GAAG,EAAE,GAAG,CAAC,iCAAiC,CAC1I,CAAC;QACJ,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,yCAAyC,GAAG,KAAK,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IACvF,CAAC;IACD,OAAO,EAAE,CAAC,+BAA+B,OAAO,CAAC,GAAG,GAAG,CAAC,CAAC;AAC3D,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,qBAAqB,CAAC,IAAY,EAAE,OAAe,EAAE,IAAgB;IACnF,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;IACtC,MAAM,IAAI,GAAG,GAAG,EAAE,CAChB,aAAa,CAAC,IAAI,EAAE,QAAQ,CAAC,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAClF,IAAI,CAAC;QACH,IAAI,CAAC;YACH,IAAI,EAAE,CAAC;QACT,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAK,CAA2B,CAAC,IAAI,KAAK,QAAQ;gBAAE,MAAM,CAAC,CAAC;YAC5D,oEAAoE;YACpE,sEAAsE;YACtE,MAAM,OAAO,GAAG,aAAa,CAC3B,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,EAC1B,GAAG,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,mDAAmD,EAAE,CAAC,EACvF,IAAI,IAAI,EAAE,EACV,IAAI,CACL,CAAC;YACF,IAAI,OAAO,CAAC,IAAI,KAAK,OAAO;gBAAE,MAAM,CAAC,CAAC;YACtC,MAAM,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAC9B,IAAI,EAAE,CAAC;QACT,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAK,CAA2B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACnD,MAAM,IAAI,kBAAkB,CAC1B,iDAAiD,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,iFAAiF,IAAI,EAAE,CAC1K,CAAC;QACJ,CAAC;QACD,MAAM,CAAC,CAAC;IACV,CAAC;IACD,OAAO,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AAC7C,CAAC;AAED,mFAAmF;AACnF,MAAM,UAAU,gBAAgB,CAC9B,OAAuF,EACvF,IAAY,EACZ,OAAe,EACf,MAAkB,cAAc,EAChC,OAA0B,EAAE;IAE5B,OAAO,OAAO,CAAC,IAAI,KAAK,KAAK;QAC3B,CAAC,CAAC,sBAAsB,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC;QAC5D,CAAC,CAAC,qBAAqB,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;AACtD,CAAC;AASD;;;;;;;GAOG;AACH,MAAM,UAAU,eAAe,CAC7B,OAAmB,EACnB,MAAiC,EACjC,OAAqB,OAAO;IAE5B,MAAM,OAAO,GAAG,GAAG,EAAE;QACnB,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QACrC,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IACzC,CAAC,CAAC;IACF,MAAM,IAAI,GAAG,CAAC,MAA4B,EAAE,IAAY,EAAS,EAAE;QACjE,OAAO,EAAE,CAAC;QACV,IAAI,CAAC;YACH,OAAO,EAAE,CAAC;YACV,MAAM,CAAC,YAAY,MAAM,2CAA2C,CAAC,CAAC;QACxE,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,CAAC,YAAY,MAAM,MAAM,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC/E,CAAC;QACD,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACzB,CAAC,CAAC;IACF,MAAM,KAAK,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;IAC1C,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IAC3B,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAC7B,OAAO,OAAO,CAAC;AACjB,CAAC"}
@@ -23,6 +23,7 @@
23
23
  */
24
24
  import type { GcloudRunner, Sleep } from "../gcloud-retry.js";
25
25
  import type { ImageDigests } from "../install-state.js";
26
+ import { type CloudRunEnvVar } from "../stack-contract.js";
26
27
  import { type EnvChange, type LiveEnvValue, type LiveVolumes } from "../stack-converge.js";
27
28
  import { type SqlBackup } from "./backup.js";
28
29
  import type { HealthProbe } from "./health.js";
@@ -58,14 +59,24 @@ export interface ServicePlan {
58
59
  /** Container memory limit now (null = Cloud Run's default). */
59
60
  readonly liveMemory: string | null;
60
61
  readonly liveVolumes: LiveVolumes;
62
+ /** Request timeout now, in seconds (null = Cloud Run's default). */
63
+ readonly liveTimeout: number | null;
61
64
  }
62
65
  /** A service plan plus the changes its roll applies. */
63
66
  export interface RollPlan extends ServicePlan {
64
67
  readonly envFlags: readonly string[];
65
68
  readonly envChanges: readonly EnvChange[];
66
- /** #553 — e.g. `--memory=2Gi` when the admin runs below the stack default. */
69
+ /**
70
+ * #553 — e.g. `--memory=2Gi` when the admin runs below the stack default;
71
+ * #806 — `--timeout=3600` when the gateway runs below its contract timeout.
72
+ */
67
73
  readonly resourceFlags: readonly string[];
68
- /** The admin's media bucket volume when it is (partly) missing; empty otherwise. */
74
+ readonly resourceChanges: readonly string[];
75
+ /**
76
+ * The admin's media bucket volume when it is (partly) missing, the
77
+ * service's secret files (#806, the gateway's OAuth client) when not
78
+ * mounted; empty otherwise.
79
+ */
69
80
  readonly volumeFlags: readonly string[];
70
81
  readonly volumeChanges: readonly string[];
71
82
  }
@@ -90,11 +101,22 @@ export type PlanResult = {
90
101
  * no change; fails when anything a roll needs is missing.
91
102
  */
92
103
  export declare function planAppRoll(install: AppRollInstall, images: Readonly<Record<RollSlug, ReleaseImage>>, run: GcloudRunner): Promise<PlanResult>;
104
+ /** Env the converge half adds to, or keeps out of, the rolls ({@link AppRollHooks.afterMigrations}). */
105
+ export interface EnvAdjustments {
106
+ /** Contract vars to leave exactly as the services have them. */
107
+ readonly leaveUntouched?: readonly string[];
108
+ /**
109
+ * Vars outside the env contract the rolls set too — #806's remote-MCP
110
+ * identity env, which only the converge (operator identity) can resolve.
111
+ */
112
+ readonly extraEnv?: Readonly<Partial<Record<RollSlug, readonly CloudRunEnvVar[]>>>;
113
+ }
93
114
  /**
94
- * The plan with `names` left exactly as the services have them — for env
95
- * vars that point at something a converge step could not set up.
115
+ * The plan with the converge's adjustments: vars left exactly as the
116
+ * services have them (they point at something a converge step could not set
117
+ * up) and vars outside the contract the rolls set too.
96
118
  */
97
- export declare function leavingEnvUntouched(plan: AppRollPlan, names: readonly string[]): AppRollPlan;
119
+ export declare function adjustingEnv(plan: AppRollPlan, adjust: EnvAdjustments): AppRollPlan;
98
120
  /** Print what the plan will change, one line per change. */
99
121
  export declare function reportPlan(plan: AppRollPlan, reporter: UpgradeReporter): void;
100
122
  /** Converge steps that must sit between app-roll steps (none for the upgrader job). */
@@ -104,10 +126,10 @@ export interface AppRollHooks {
104
126
  * vars the rolls must leave untouched because what they point at could
105
127
  * not be set up; `ok: false` aborts the upgrade (nothing rolled yet).
106
128
  */
107
- afterMigrations?(): Promise<{
129
+ afterMigrations?(): Promise<({
108
130
  readonly ok: true;
109
131
  readonly leaveUntouched: readonly string[];
110
- } | {
132
+ } & Pick<EnvAdjustments, "extraEnv">) | {
111
133
  readonly ok: false;
112
134
  }>;
113
135
  /** Right before `slug` rolls; `ok: false` aborts and rolls back what rolled. */
@@ -158,6 +180,13 @@ export type AppRollResult = {
158
180
  readonly phase: AppRollPhase;
159
181
  /** Whether migrations ran before the failure (the backup matters then). */
160
182
  readonly migrated: boolean;
183
+ /**
184
+ * Whether services had rolled and EVERY one of them was pinned back to
185
+ * the revision that served before — i.e. the previous release serves
186
+ * again. False when nothing had rolled, or when a pin failed (then a
187
+ * person must pin it; the reporter named which).
188
+ */
189
+ readonly rolledBack: boolean;
161
190
  };
162
191
  /** Run the app-roll. See the file comment for the sequence. */
163
192
  export declare function executeAppRoll(input: AppRollInput, hooks: AppRollHooks, deps: AppRollDeps): Promise<AppRollResult>;
@@ -1 +1 @@
1
- {"version":3,"file":"app-roll.d.ts","sourceRoot":"","sources":["../../src/upgrade/app-roll.ts"],"names":[],"mappings":"AAEA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAE9D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAExD,OAAO,EAEL,KAAK,SAAS,EACd,KAAK,YAAY,EACjB,KAAK,WAAW,EASjB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAkB,KAAK,SAAS,EAAE,MAAM,aAAa,CAAC;AAQ7D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAErD,MAAM,MAAM,QAAQ,GAAG,OAAO,GAAG,SAAS,CAAC;AAE3C,gFAAgF;AAChF,eAAO,MAAM,UAAU,EAAE,SAAS,QAAQ,EAAyB,CAAC;AAEpE,uCAAuC;AACvC,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,QAAQ,EAAE,KAAK,GAAG,cAAc,CAAC;IAC1C,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,wCAAwC;IACxC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,iEAAiE;IACjE,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAED,8CAA8C;AAC9C,MAAM,WAAW,YAAY;IAC3B,qEAAqE;IACrE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,4DAA4D;AAC5D,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,OAAO,EAAE,WAAW,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IACpD,+DAA+D;IAC/D,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;CACnC;AAED,wDAAwD;AACxD,MAAM,WAAW,QAAS,SAAQ,WAAW;IAC3C,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,QAAQ,CAAC,UAAU,EAAE,SAAS,SAAS,EAAE,CAAC;IAC1C,8EAA8E;IAC9E,QAAQ,CAAC,aAAa,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1C,oFAAoF;IACpF,QAAQ,CAAC,WAAW,EAAE,SAAS,MAAM,EAAE,CAAC;IACxC,QAAQ,CAAC,aAAa,EAAE,SAAS,MAAM,EAAE,CAAC;CAC3C;AAED,kFAAkF;AAClF,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAC;IACjC,6BAA6B;IAC7B,QAAQ,CAAC,QAAQ,EAAE,SAAS,QAAQ,EAAE,CAAC;IACvC,+DAA+D;IAC/D,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;CACtC;AAED,MAAM,MAAM,UAAU,GAClB;IAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAA;CAAE,GACjD;IAAE,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAEnD;;;;GAIG;AACH,wBAAsB,WAAW,CAC/B,OAAO,EAAE,cAAc,EACvB,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC,EAChD,GAAG,EAAE,YAAY,GAChB,OAAO,CAAC,UAAU,CAAC,CAsErB;AAwBD;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,WAAW,CAY5F;AAED,4DAA4D;AAC5D,wBAAgB,UAAU,CAAC,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,eAAe,GAAG,IAAI,CAa7E;AAED,uFAAuF;AACvF,MAAM,WAAW,YAAY;IAC3B;;;;OAIG;IACH,eAAe,CAAC,IAAI,OAAO,CACzB;QAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;QAAC,QAAQ,CAAC,cAAc,EAAE,SAAS,MAAM,EAAE,CAAA;KAAE,GAAG;QAAE,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAA;KAAE,CAC3F,CAAC;IACF,gFAAgF;IAChF,UAAU,CAAC,CACT,IAAI,EAAE,QAAQ,GACb,OAAO,CAAC;QAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAA;KAAE,GAAG;QAAE,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACnF,yEAAyE;IACzE,aAAa,CAAC,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC/C;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,GAAG,EAAE,YAAY,CAAC;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC;IACvB,QAAQ,CAAC,QAAQ,EAAE,eAAe,CAAC;IACnC,yEAAyE;IACzE,QAAQ,CAAC,OAAO,EAAE,CAChB,QAAQ,EAAE,MAAM,KACb,OAAO,CAAC;QAAE,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAChE,8EAA8E;IAC9E,QAAQ,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI,CAAC;IACpC,uEAAuE;IACvE,QAAQ,CAAC,aAAa,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAClE;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;IAC3B;;;OAGG;IACH,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC;IAC3B,oFAAoF;IACpF,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC;;;OAGG;IACH,QAAQ,CAAC,kBAAkB,EAAE,OAAO,GAAG,IAAI,CAAC;CAC7C;AAED,MAAM,MAAM,YAAY,GAAG,iBAAiB,GAAG,YAAY,GAAG,UAAU,GAAG,MAAM,GAAG,QAAQ,CAAC;AAE7F,MAAM,MAAM,aAAa,GACrB;IAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAA;CAAE,GACrD;IACE,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC;IACnB,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IAC7B,2EAA2E;IAC3E,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;CAC5B,CAAC;AAEN,+DAA+D;AAC/D,wBAAsB,cAAc,CAClC,KAAK,EAAE,YAAY,EACnB,KAAK,EAAE,YAAY,EACnB,IAAI,EAAE,WAAW,GAChB,OAAO,CAAC,aAAa,CAAC,CAiMxB"}
1
+ {"version":3,"file":"app-roll.d.ts","sourceRoot":"","sources":["../../src/upgrade/app-roll.ts"],"names":[],"mappings":"AAEA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAE9D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAEL,KAAK,cAAc,EAIpB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAEL,KAAK,SAAS,EACd,KAAK,YAAY,EACjB,KAAK,WAAW,EAYjB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAkB,KAAK,SAAS,EAAE,MAAM,aAAa,CAAC;AAQ7D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAErD,MAAM,MAAM,QAAQ,GAAG,OAAO,GAAG,SAAS,CAAC;AAE3C,gFAAgF;AAChF,eAAO,MAAM,UAAU,EAAE,SAAS,QAAQ,EAAyB,CAAC;AAEpE,uCAAuC;AACvC,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,QAAQ,EAAE,KAAK,GAAG,cAAc,CAAC;IAC1C,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,wCAAwC;IACxC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,iEAAiE;IACjE,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAED,8CAA8C;AAC9C,MAAM,WAAW,YAAY;IAC3B,qEAAqE;IACrE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,4DAA4D;AAC5D,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,OAAO,EAAE,WAAW,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IACpD,+DAA+D;IAC/D,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;IAClC,oEAAoE;IACpE,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC;AAED,wDAAwD;AACxD,MAAM,WAAW,QAAS,SAAQ,WAAW;IAC3C,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,QAAQ,CAAC,UAAU,EAAE,SAAS,SAAS,EAAE,CAAC;IAC1C;;;OAGG;IACH,QAAQ,CAAC,aAAa,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1C,QAAQ,CAAC,eAAe,EAAE,SAAS,MAAM,EAAE,CAAC;IAC5C;;;;OAIG;IACH,QAAQ,CAAC,WAAW,EAAE,SAAS,MAAM,EAAE,CAAC;IACxC,QAAQ,CAAC,aAAa,EAAE,SAAS,MAAM,EAAE,CAAC;CAC3C;AAED,kFAAkF;AAClF,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAC;IACjC,6BAA6B;IAC7B,QAAQ,CAAC,QAAQ,EAAE,SAAS,QAAQ,EAAE,CAAC;IACvC,+DAA+D;IAC/D,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;CACtC;AAED,MAAM,MAAM,UAAU,GAClB;IAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAA;CAAE,GACjD;IAAE,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAEnD;;;;GAIG;AACH,wBAAsB,WAAW,CAC/B,OAAO,EAAE,cAAc,EACvB,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC,EAChD,GAAG,EAAE,YAAY,GAChB,OAAO,CAAC,UAAU,CAAC,CAiGrB;AAED,wGAAwG;AACxG,MAAM,WAAW,cAAc;IAC7B,gEAAgE;IAChE,QAAQ,CAAC,cAAc,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC5C;;;OAGG;IACH,QAAQ,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,SAAS,cAAc,EAAE,CAAC,CAAC,CAAC,CAAC;CACpF;AA2BD;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,cAAc,GAAG,WAAW,CAanF;AAED,4DAA4D;AAC5D,wBAAgB,UAAU,CAAC,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,eAAe,GAAG,IAAI,CAW7E;AAED,uFAAuF;AACvF,MAAM,WAAW,YAAY;IAC3B;;;;OAIG;IACH,eAAe,CAAC,IAAI,OAAO,CACvB,CAAC;QAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;QAAC,QAAQ,CAAC,cAAc,EAAE,SAAS,MAAM,EAAE,CAAA;KAAE,GAAG,IAAI,CACvE,cAAc,EACd,UAAU,CACX,CAAC,GACF;QAAE,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAA;KAAE,CACzB,CAAC;IACF,gFAAgF;IAChF,UAAU,CAAC,CACT,IAAI,EAAE,QAAQ,GACb,OAAO,CAAC;QAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAA;KAAE,GAAG;QAAE,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACnF,yEAAyE;IACzE,aAAa,CAAC,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC/C;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,GAAG,EAAE,YAAY,CAAC;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC;IACvB,QAAQ,CAAC,QAAQ,EAAE,eAAe,CAAC;IACnC,yEAAyE;IACzE,QAAQ,CAAC,OAAO,EAAE,CAChB,QAAQ,EAAE,MAAM,KACb,OAAO,CAAC;QAAE,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAChE,8EAA8E;IAC9E,QAAQ,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI,CAAC;IACpC,uEAAuE;IACvE,QAAQ,CAAC,aAAa,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAClE;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;IAC3B;;;OAGG;IACH,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC;IAC3B,oFAAoF;IACpF,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC;;;OAGG;IACH,QAAQ,CAAC,kBAAkB,EAAE,OAAO,GAAG,IAAI,CAAC;CAC7C;AAED,MAAM,MAAM,YAAY,GAAG,iBAAiB,GAAG,YAAY,GAAG,UAAU,GAAG,MAAM,GAAG,QAAQ,CAAC;AAE7F,MAAM,MAAM,aAAa,GACrB;IAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAA;CAAE,GACrD;IACE,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC;IACnB,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IAC7B,2EAA2E;IAC3E,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;IAC3B;;;;;OAKG;IACH,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC;CAC9B,CAAC;AAEN,+DAA+D;AAC/D,wBAAsB,cAAc,CAClC,KAAK,EAAE,YAAY,EACnB,KAAK,EAAE,YAAY,EACnB,IAAI,EAAE,WAAW,GAChB,OAAO,CAAC,aAAa,CAAC,CA2MxB"}
@@ -1,7 +1,7 @@
1
1
  // SPDX-License-Identifier: MPL-2.0
2
2
  import { gatewayServiceAccountEmail, runServiceAccountEmail } from "../gcp-names.js";
3
- import { adminMediaVolume } from "../stack-contract.js";
4
- import { liveContainerEnv, liveContainerMemory, liveVolumes, planAdminMemory, planContractEnv, planMediaVolume, rollService, serviceRollArgs, } from "../stack-converge.js";
3
+ import { adminMediaVolume, GATEWAY_TIMEOUT_SECONDS, RETIRED_SECRET_FILES, SERVICE_SECRET_FILES, } from "../stack-contract.js";
4
+ import { liveContainerEnv, liveContainerMemory, liveTimeoutSeconds, liveVolumes, planAdminMemory, planContractEnv, planMediaVolume, planRequestTimeout, planSecretFiles, rollService, serviceRollArgs, } from "../stack-converge.js";
5
5
  import { describeAuditJobImage, setAuditJobImage } from "./audit-job-roll.js";
6
6
  import { restoreCommand } from "./backup.js";
7
7
  import { describeServiceJson, findCurrentRevision, findServiceUrl, pinTraffic, resolveGcpResourceName, } from "./gcp-lookup.js";
@@ -41,9 +41,10 @@ export async function planAppRoll(install, images, run) {
41
41
  liveEnv: liveContainerEnv(json),
42
42
  liveMemory: liveContainerMemory(json),
43
43
  liveVolumes: liveVolumes(json),
44
+ liveTimeout: liveTimeoutSeconds(json),
44
45
  });
45
46
  }
46
- const env = planEnv(install, services, []);
47
+ const env = planEnv(install, services, {});
47
48
  if (!env.ok)
48
49
  return { ok: false, error: `the install's env contract can't be applied: ${env.error}` };
49
50
  // Without the media volume every upload is lost on the next revision or
@@ -52,6 +53,17 @@ export async function planAppRoll(install, images, run) {
52
53
  const media = planMediaVolume(admin.liveVolumes, adminMediaVolume(install.projectId, install.env));
53
54
  if (!media.ok)
54
55
  return { ok: false, error: `the admin's media volume can't be set up: ${media.error}` };
56
+ // #806/#914 — each service's secret files (the gateway's Google sign-in
57
+ // client). A file the roll cannot mount fails the plan: the new gateway
58
+ // reads its path from the env contract.
59
+ const secretFiles = {};
60
+ for (const s of services) {
61
+ const files = planSecretFiles(s.liveVolumes, SERVICE_SECRET_FILES[s.slug], install.env, RETIRED_SECRET_FILES[s.slug]);
62
+ if (!files.ok) {
63
+ return { ok: false, error: `the ${s.slug}'s secret files can't be mounted: ${files.error}` };
64
+ }
65
+ secretFiles[s.slug] = files;
66
+ }
55
67
  const warnings = [];
56
68
  const rolls = services.map((s) => {
57
69
  // #553 — the admin runs the Lighthouse quality audit; raise it to the
@@ -59,18 +71,27 @@ export async function planAppRoll(install, images, run) {
59
71
  const memory = s.slug === "admin" ? planAdminMemory(s.liveMemory) : null;
60
72
  if (memory && !memory.ok)
61
73
  warnings.push(`admin memory left as is: ${memory.error}`);
74
+ // #806 — the gateway streams remote-MCP chat turns (stack-contract.ts
75
+ // GATEWAY_TIMEOUT_SECONDS). The admin's timeout is the stacks' alone.
76
+ const timeout = s.slug === "gateway" ? planRequestTimeout(s.liveTimeout, GATEWAY_TIMEOUT_SECONDS) : null;
77
+ const memoryFlags = memory?.ok ? memory.flags : [];
78
+ const timeoutFlags = timeout?.ok ? timeout.flags : [];
62
79
  return {
63
80
  ...s,
64
81
  envFlags: env.services[s.slug].flags,
65
82
  envChanges: env.services[s.slug].changes,
66
- resourceFlags: memory?.ok ? memory.flags : [],
67
- volumeFlags: s.slug === "admin" ? media.flags : [],
68
- volumeChanges: s.slug === "admin" ? media.changes : [],
83
+ resourceFlags: [...memoryFlags, ...timeoutFlags],
84
+ resourceChanges: [
85
+ ...memoryFlags.map((f) => `${s.liveMemory ?? "(default)"} → ${f}`),
86
+ ...(timeout?.ok ? timeout.changes : []),
87
+ ],
88
+ volumeFlags: [...(s.slug === "admin" ? media.flags : []), ...secretFiles[s.slug].flags],
89
+ volumeChanges: [...(s.slug === "admin" ? media.changes : []), ...secretFiles[s.slug].changes],
69
90
  };
70
91
  });
71
92
  return { ok: true, plan: { install, services: rolls, warnings } };
72
93
  }
73
- function planEnv(install, services, leaveUntouched) {
94
+ function planEnv(install, services, adjust) {
74
95
  const deployed = Object.fromEntries(services.map((s) => [s.slug, { serviceName: s.serviceName, liveEnv: s.liveEnv }]));
75
96
  return planContractEnv({
76
97
  provider: install.provider,
@@ -79,16 +100,21 @@ function planEnv(install, services, leaveUntouched) {
79
100
  domain: install.domain,
80
101
  region: install.region,
81
102
  stagingHost: install.stagingHost,
82
- }, deployed, { leaveUntouched });
103
+ }, deployed, {
104
+ leaveUntouched: adjust.leaveUntouched ?? [],
105
+ ...(adjust.extraEnv ? { extraEnv: adjust.extraEnv } : {}),
106
+ });
83
107
  }
84
108
  /**
85
- * The plan with `names` left exactly as the services have them — for env
86
- * vars that point at something a converge step could not set up.
109
+ * The plan with the converge's adjustments: vars left exactly as the
110
+ * services have them (they point at something a converge step could not set
111
+ * up) and vars outside the contract the rolls set too.
87
112
  */
88
- export function leavingEnvUntouched(plan, names) {
89
- if (names.length === 0)
113
+ export function adjustingEnv(plan, adjust) {
114
+ const extra = Object.values(adjust.extraEnv ?? {}).some((v) => (v?.length ?? 0) > 0);
115
+ if ((adjust.leaveUntouched?.length ?? 0) === 0 && !extra)
90
116
  return plan;
91
- const env = planEnv(plan.install, plan.services, names);
117
+ const env = planEnv(plan.install, plan.services, adjust);
92
118
  if (!env.ok)
93
119
  return plan;
94
120
  return {
@@ -106,10 +132,9 @@ export function reportPlan(plan, reporter) {
106
132
  reporter.warn(w);
107
133
  for (const s of plan.services) {
108
134
  for (const c of s.volumeChanges)
109
- reporter.info(`${s.slug} media: ${c}`);
110
- for (const f of s.resourceFlags) {
111
- reporter.info(`${s.slug} resources: ${s.liveMemory ?? "(default)"} → ${f}`);
112
- }
135
+ reporter.info(`${s.slug} volumes: ${c}`);
136
+ for (const c of s.resourceChanges)
137
+ reporter.info(`${s.slug} resources: ${c}`);
113
138
  for (const c of s.envChanges) {
114
139
  reporter.info(`${s.slug} env ${c.name}: ${c.from === undefined ? "(unset)" : c.from} → ${c.to === undefined ? "(removed)" : c.to}`);
115
140
  }
@@ -120,10 +145,13 @@ export async function executeAppRoll(input, hooks, deps) {
120
145
  const { reporter: r, run } = deps;
121
146
  const { install } = input.plan;
122
147
  let plan = input.plan;
148
+ /** Set by {@link rollBack}: null until one ran with services to restore. */
149
+ let rollbackOk = null;
123
150
  const fail = (phase, migrated) => ({
124
151
  ok: false,
125
152
  phase,
126
153
  migrated,
154
+ rolledBack: rollbackOk === true,
127
155
  });
128
156
  // A probe that cannot reach the install now would only fail after the
129
157
  // migrations, and roll a healthy release back.
@@ -158,7 +186,10 @@ export async function executeAppRoll(input, hooks, deps) {
158
186
  const converged = await hooks.afterMigrations();
159
187
  if (!converged.ok)
160
188
  return fail("converge", true);
161
- plan = leavingEnvUntouched(plan, converged.leaveUntouched);
189
+ plan = adjustingEnv(plan, {
190
+ leaveUntouched: converged.leaveUntouched,
191
+ ...(converged.extraEnv ? { extraEnv: converged.extraEnv } : {}),
192
+ });
162
193
  }
163
194
  const rolled = [];
164
195
  /** The audit job's image before this roll moved it (#875); null = not moved. */
@@ -175,12 +206,15 @@ export async function executeAppRoll(input, hooks, deps) {
175
206
  if (services.length === 0)
176
207
  return;
177
208
  r.warn(`Rolling back ${services.length} service(s) to the revisions that served before...`);
209
+ rollbackOk = true;
178
210
  for (const p of [...services].reverse()) {
179
211
  const ok = await pinTraffic(run, install.projectId, install.region, p.serviceName, p.priorRevision);
180
212
  if (ok)
181
213
  r.info(` ${p.slug} traffic restored to ${p.priorRevision}`);
182
- else
214
+ else {
215
+ rollbackOk = false;
183
216
  r.error(` ${p.slug} rollback FAILED — pin ${p.priorRevision} by hand`);
217
+ }
184
218
  await hooks.afterRollback?.(p.slug);
185
219
  }
186
220
  };