@wtfalch/foundry 0.16.15 → 0.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/dist/apply.d.ts +22 -1
  2. package/dist/bootstrap/bootstrap.d.ts +6 -1
  3. package/dist/bootstrap/bootstrap.js +26 -3
  4. package/dist/bootstrap/declare.d.ts +11 -1
  5. package/dist/bootstrap/declare.js +12 -2
  6. package/dist/index.d.ts +9 -4
  7. package/dist/index.js +7 -0
  8. package/dist/migrations/0038_calls_version_pin.sql +8 -0
  9. package/dist/migrations/0039_destroy_record.sql +7 -0
  10. package/dist/migrations/0040_company_included_uuid.sql +72 -0
  11. package/dist/migrations/0041_company_included_uuid_keys.sql +132 -0
  12. package/dist/migrations/0042_mutation_leases.sql +12 -0
  13. package/dist/org.d.ts +82 -5
  14. package/dist/org.js +62 -1
  15. package/dist/plan/adopt.d.ts +76 -0
  16. package/dist/plan/adopt.js +184 -0
  17. package/dist/plan/agora.d.ts +21 -0
  18. package/dist/plan/agora.js +37 -0
  19. package/dist/plan/broker-entries.d.ts +9 -1
  20. package/dist/plan/broker-entries.js +13 -5
  21. package/dist/plan/chat-matrix.d.ts +1 -1
  22. package/dist/plan/chat-matrix.js +11 -3
  23. package/dist/plan/core-files.d.ts +25 -0
  24. package/dist/plan/core-files.js +25 -0
  25. package/dist/plan/deploy-health.d.ts +136 -0
  26. package/dist/plan/deploy-health.js +153 -0
  27. package/dist/plan/diff.d.ts +11 -3
  28. package/dist/plan/diff.js +77 -71
  29. package/dist/plan/entry-ai-key.d.ts +3 -3
  30. package/dist/plan/entry-files-key.d.ts +6 -4
  31. package/dist/plan/entry-files-key.js +7 -10
  32. package/dist/plan/executor.d.ts +109 -28
  33. package/dist/plan/executor.js +213 -56
  34. package/dist/plan/hostnames.d.ts +6 -0
  35. package/dist/plan/hostnames.js +13 -0
  36. package/dist/plan/index.d.ts +5 -1
  37. package/dist/plan/index.js +3 -1
  38. package/dist/plan/live.d.ts +5 -0
  39. package/dist/plan/login-host.d.ts +1 -1
  40. package/dist/plan/login-host.js +6 -5
  41. package/dist/plan/plan-apply.d.ts +153 -3
  42. package/dist/plan/plan-apply.js +584 -69
  43. package/dist/plan/platform-diff.d.ts +103 -9
  44. package/dist/plan/platform-diff.js +192 -14
  45. package/dist/plan/platform-org.d.ts +25 -0
  46. package/dist/plan/platform-org.js +17 -1
  47. package/dist/plan/platform-providers.d.ts +38 -5
  48. package/dist/plan/platform-providers.js +592 -109
  49. package/dist/plan/providers-readers.d.ts +2 -0
  50. package/dist/plan/providers-readers.js +4 -0
  51. package/dist/plan/rollout.d.ts +52 -0
  52. package/dist/plan/rollout.js +37 -0
  53. package/dist/plan/step-system.d.ts +22 -0
  54. package/dist/plan/step-system.js +37 -0
  55. package/dist/plan.d.ts +3 -0
  56. package/dist/state.d.ts +13 -0
  57. package/dist/store/audit.js +15 -1
  58. package/dist/store/postgres.d.ts +1 -1
  59. package/dist/store/postgres.js +127 -14
  60. package/dist/store/schema.d.ts +134 -5
  61. package/dist/store/schema.js +12 -1
  62. package/dist/store.d.ts +47 -4
  63. package/dist/store.js +9 -1
  64. package/package.json +2 -2
@@ -4,9 +4,10 @@ import { applyObjects, openInstance, platformMailSettingsObjects } from '@wtfalc
4
4
  import { VERSIONS, floatingTagBlocked } from '../bootstrap/versions.js';
5
5
  import { redactSecrets } from '../calls-sfu-deploy-support.js';
6
6
  import { ensureCloudflaredConnector } from './cloudflared-connector.js';
7
+ import { waitForDeploymentFinished, waitForHealthyVersion, waitForNewDeployment, } from './deploy-health.js';
7
8
  import { ACCESS_POLICY_NAME, getDatabaseStarted, isNotFoundError, isRunningOrStarting, } from './executor.js';
8
- import { coolifyDomains } from './hostnames.js';
9
- import { AUTH_ZITADEL_RUN_OPTIONS, JOBS_SCHEDULER_KEY, PLATFORM_AI_APPLIED_GITREF_SCOPE_KEY, PLATFORM_AI_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_AI_DEPLOY_PENDING_SCOPE_KEY, PLATFORM_AI_INTERNAL_URL_SCOPE_KEY, PLATFORM_AI_KEY, PLATFORM_AI_MANAGEMENT_KEY_CONFLICT_SCOPE_KEY, PLATFORM_AI_MANAGEMENT_KEY_SCOPE_KEY, PLATFORM_AI_MANAGEMENT_KEY_UNDELIVERED_SCOPE_KEY, PLATFORM_AI_MIGRATE_BLOCKED_SCOPE_KEY, PLATFORM_AI_ORGANISATION_SCOPE_KEY, PLATFORM_AI_RUNNER_DEPLOYMENT_SCOPE_KEY, PLATFORM_AI_SIGNING_KEY_SET_SCOPE_KEY, PLATFORM_AI_SIGNING_KEY_UNDELIVERED_SCOPE_KEY, PLATFORM_ARCHON_APPLIED_BROKER_SCOPE_KEY, PLATFORM_ARCHON_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_ARCHON_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_ARCHON_CONNECTOR_APP_SCOPE_KEY, PLATFORM_ARCHON_GATE_CLOSED_SCOPE_KEY, PLATFORM_ARCHON_KEY, PLATFORM_ARCHON_MASTER_CLOUDFLARE_TOKEN_REF, PLATFORM_ARCHON_MASTER_COOLIFY_TOKEN_REF, PLATFORM_ARCHON_MASTER_COOLIFY_URL_REF, PLATFORM_AUTH_ADMIN_UNDELIVERED_SCOPE_KEY, PLATFORM_AUTH_ADMIN_USER_SCOPE_KEY, PLATFORM_AUTH_BROKER_DEPLOYED_SCOPE_KEY, PLATFORM_AUTH_BROKER_SCOPE_KEY, PLATFORM_AUTH_CONNECTOR_APP_SCOPE_KEY, PLATFORM_AUTH_INSTANCE_SCOPE_KEY, PLATFORM_AUTH_INSTANCE_UNDELIVERED_SCOPE_KEY, PLATFORM_AUTH_KEY, PLATFORM_AUTH_LOGIN_CLIENT_UNDELIVERED_SCOPE_KEY, PLATFORM_AUTH_LOGIN_CLIENT_USER_SCOPE_KEY, PLATFORM_AUTH_OWNER_INVITE_SCOPE_KEY, PLATFORM_AUTH_SMTP_SCOPE_KEY, PLATFORM_AUTH_ZITADEL_APPLIED_IMAGE_TAG_SCOPE_KEY, PLATFORM_AUTH_ZITADEL_RUN_OPTIONS_SCOPE_KEY, PLATFORM_BACKUP_DATABASE_SCOPE_KEYS, PLATFORM_BACKUP_SERVICE_KEYS, PLATFORM_FILES_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_FILES_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_FILES_INTERNAL_URL_SCOPE_KEY, PLATFORM_FILES_INTROSPECTION_SCOPE_KEY, PLATFORM_FILES_INTROSPECTION_UNDELIVERED_SCOPE_KEY, PLATFORM_FILES_KEY, PLATFORM_FILES_MANAGEMENT_KEY_SCOPE_KEY, PLATFORM_FILES_MANAGEMENT_UNDELIVERED_SCOPE_KEY, PLATFORM_FILES_ZITADEL_PROJECT_SCOPE_KEY, PLATFORM_INTEGRATIONS_APPLIED_GITREF_SCOPE_KEY, PLATFORM_INTEGRATIONS_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_INTEGRATIONS_KEY, PLATFORM_INTEGRATIONS_KEY_SET_SCOPE_KEY, PLATFORM_INTEGRATIONS_KEY_UNDELIVERED_SCOPE_KEY, PLATFORM_MAIL_ADMIN_KEY_SCOPE_KEY, PLATFORM_MAIL_ADMIN_UNDELIVERED_SCOPE_KEY, PLATFORM_MAIL_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_MAIL_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_MAIL_CONNECTOR_APP_SCOPE_KEY, PLATFORM_MAIL_INTERNAL_URL_SCOPE_KEY, PLATFORM_MAIL_KEY, PLATFORM_MAIL_ZITADEL_PROJECT_SCOPE_KEY, parseBrokerDeployRecord, parseLoginClientRecord, platformMailMxHost, } from './platform-diff.js';
9
+ import { coolifyDomains, filesShareHostname } from './hostnames.js';
10
+ import { AUTH_ZITADEL_RUN_OPTIONS, JOBS_SCHEDULER_KEY, PLATFORM_AI_APPLIED_GITREF_SCOPE_KEY, PLATFORM_AI_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_AI_DEPLOY_PENDING_SCOPE_KEY, PLATFORM_AI_INTERNAL_URL_SCOPE_KEY, PLATFORM_AI_KEY, PLATFORM_AI_MANAGEMENT_KEY_CONFLICT_SCOPE_KEY, PLATFORM_AI_MANAGEMENT_KEY_SCOPE_KEY, PLATFORM_AI_MANAGEMENT_KEY_UNDELIVERED_SCOPE_KEY, PLATFORM_AI_MIGRATE_BLOCKED_SCOPE_KEY, PLATFORM_AI_ORGANISATION_SCOPE_KEY, PLATFORM_AI_SIGNING_KEY_SET_SCOPE_KEY, PLATFORM_AI_SIGNING_KEY_UNDELIVERED_SCOPE_KEY, PLATFORM_ARCHON_APPLIED_BROKER_SCOPE_KEY, PLATFORM_ARCHON_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_ARCHON_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_ARCHON_APPLIED_OPERATOR_SCOPE_KEY, PLATFORM_ARCHON_CONNECTOR_APP_SCOPE_KEY, PLATFORM_ARCHON_COOLIFY_READ_TOKEN_REF, PLATFORM_ARCHON_GATE_CLOSED_SCOPE_KEY, PLATFORM_ARCHON_INTEGRATIONS_KEY_SYNCED_SCOPE_KEY, PLATFORM_ARCHON_KEY, PLATFORM_ARCHON_MASTER_CLOUDFLARE_TOKEN_REF, PLATFORM_ARCHON_MASTER_COOLIFY_TOKEN_REF, PLATFORM_ARCHON_MASTER_COOLIFY_URL_REF, PLATFORM_AUTH_ADMIN_UNDELIVERED_SCOPE_KEY, PLATFORM_AUTH_ADMIN_USER_SCOPE_KEY, PLATFORM_AUTH_BROKER_DEPLOYED_SCOPE_KEY, PLATFORM_AUTH_BROKER_SCOPE_KEY, PLATFORM_AUTH_CONNECTOR_APP_SCOPE_KEY, PLATFORM_AUTH_INSTANCE_SCOPE_KEY, PLATFORM_AUTH_INSTANCE_UNDELIVERED_SCOPE_KEY, PLATFORM_AUTH_KEY, PLATFORM_AUTH_LOGIN_CLIENT_UNDELIVERED_SCOPE_KEY, PLATFORM_AUTH_LOGIN_CLIENT_USER_SCOPE_KEY, PLATFORM_AUTH_OWNER_INVITE_SCOPE_KEY, PLATFORM_AUTH_SMTP_SCOPE_KEY, PLATFORM_AUTH_ZITADEL_APPLIED_IMAGE_TAG_SCOPE_KEY, PLATFORM_AUTH_ZITADEL_RUN_OPTIONS_SCOPE_KEY, PLATFORM_BACKUP_DATABASE_SCOPE_KEYS, PLATFORM_BACKUP_SERVICE_KEYS, PLATFORM_FILES_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_FILES_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_FILES_INTERNAL_URL_SCOPE_KEY, PLATFORM_FILES_INTROSPECTION_SCOPE_KEY, PLATFORM_FILES_INTROSPECTION_UNDELIVERED_SCOPE_KEY, PLATFORM_FILES_KEY, PLATFORM_FILES_MANAGEMENT_KEY_SCOPE_KEY, PLATFORM_FILES_MANAGEMENT_UNDELIVERED_SCOPE_KEY, PLATFORM_FILES_ZITADEL_PROJECT_SCOPE_KEY, PLATFORM_INTEGRATIONS_APPLIED_GITREF_SCOPE_KEY, PLATFORM_INTEGRATIONS_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_INTEGRATIONS_KEY, PLATFORM_INTEGRATIONS_KEY_ID_SCOPE_KEY, PLATFORM_INTEGRATIONS_KEY_ROTATION_PREVIOUS_SCOPE_KEY, PLATFORM_INTEGRATIONS_KEY_ROTATION_REQUESTED_SCOPE_KEY, PLATFORM_INTEGRATIONS_KEY_ROTATION_UNDELIVERED_SCOPE_KEY, PLATFORM_INTEGRATIONS_KEY_SET_SCOPE_KEY, PLATFORM_INTEGRATIONS_KEY_UNDELIVERED_SCOPE_KEY, PLATFORM_INTEGRATIONS_RUNNER_KEYRING_DEPLOYED_SCOPE_KEY, PLATFORM_MAIL_ADMIN_KEY_SCOPE_KEY, PLATFORM_MAIL_ADMIN_UNDELIVERED_SCOPE_KEY, PLATFORM_MAIL_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_MAIL_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_MAIL_CONNECTOR_APP_SCOPE_KEY, PLATFORM_MAIL_INTERNAL_URL_SCOPE_KEY, PLATFORM_MAIL_KEY, PLATFORM_MAIL_ZITADEL_PROJECT_SCOPE_KEY, archonBootstrapOperator, parseBrokerDeployRecord, parseLoginClientRecord, platformMailMxHost, } from './platform-diff.js';
10
11
  import { applyCallsSfuPlatformStep } from './platform-sfu.js';
11
12
  import { CreateInstanceError, createZitadelInstance, generateSystemApiKeypair, instanceWithoutBearerMessage, systemApiUsersEnv, } from './zitadel-system-api.js';
12
13
  /** The key `runPlatformSteps` records the apex zone's id under (`system:kind:key`). */
@@ -258,7 +259,10 @@ export class FoundryPlatformStepExecutor {
258
259
  recordedIds[`coolify:app:${PLATFORM_ARCHON_KEY}`]) {
259
260
  const accessDrifted = await archonAccessDrifted(this.archon.cloudflare, platform.archon.hostname, this.archon.accessEmails);
260
261
  const brokerDrifted = !accessDrifted && (await archonBrokerRegistrationDrifted(this.archon.coolify, recordedIds));
261
- if (accessDrifted || brokerDrifted)
262
+ const bootstrapOperatorDrifted = !accessDrifted &&
263
+ !brokerDrifted &&
264
+ (await archonBootstrapOperatorDrifted(this.archon.coolify, recordedIds[`coolify:app:${PLATFORM_ARCHON_KEY}`], recordedIds));
265
+ if (accessDrifted || brokerDrifted || bootstrapOperatorDrifted)
262
266
  result.archon = true;
263
267
  }
264
268
  if (platform.auth.enabled && this.auth) {
@@ -276,7 +280,10 @@ export class FoundryPlatformStepExecutor {
276
280
  if (platform.files.enabled && this.files && recordedIds[`coolify:app:${PLATFORM_FILES_KEY}`]) {
277
281
  // A Coolify read error must not fail the whole plan: treat it as not drifted.
278
282
  try {
279
- if (await filesRunnerEnvDrifted(this.files.coolify, recordedIds))
283
+ if (await filesRunnerEnvDrifted(this.files.coolify, recordedIds, {
284
+ publicHosts: this.files.publicHosts,
285
+ shareDomains: filesShareDomains(await this.declaredOrgStore.list()),
286
+ }))
280
287
  result.files = true;
281
288
  }
282
289
  catch {
@@ -297,6 +304,25 @@ export class FoundryPlatformStepExecutor {
297
304
  }
298
305
  return result;
299
306
  }
307
+ /**
308
+ * Gives the files runner a changed share-domain set and nothing else: its
309
+ * Coolify `domains`, `FILES_PUBLIC_HOSTS` and one redeploy of the runner.
310
+ * Returns the scope ids to record, or `null` when nothing changed
311
+ * (or the runner was never applied).
312
+ */
313
+ async refreshFilesShare(_platform, recordedIds) {
314
+ const runnerAppId = recordedIds[`coolify:app:${PLATFORM_FILES_KEY}`];
315
+ if (!this.files || !runnerAppId)
316
+ return null;
317
+ const wanted = filesShareDomains(await this.declaredOrgStore.list());
318
+ if (!shareDomainsChanged(recordedIds, wanted))
319
+ return null;
320
+ const { coolify } = this.files;
321
+ await coolify.updateApplication(runnerAppId, filesShareDomainsPatch(recordedIds, wanted));
322
+ await coolify.bulkUpsertEnvs(runnerAppId, filesPublicHostsEnvs(recordedIds, this.files.publicHosts, wanted));
323
+ await coolify.deploy(runnerAppId);
324
+ return { [PLATFORM_FILES_SHARE_DOMAINS_SCOPE_KEY]: normalHosts(wanted).join(',') };
325
+ }
300
326
  async execute(action, context) {
301
327
  if (action.system === 'coolify' && action.kind === 'app' && action.key === PLATFORM_AUTH_KEY) {
302
328
  if (!this.auth) {
@@ -309,7 +335,7 @@ export class FoundryPlatformStepExecutor {
309
335
  if (!this.files) {
310
336
  throw new Error('platform:files needs FoundryPlatformStepExecutor to be constructed with its files dependencies (#171)');
311
337
  }
312
- return provisionPlatformFiles(context, this.files);
338
+ return provisionPlatformFiles(context, this.files, filesShareDomains(await this.declaredOrgStore.list()));
313
339
  }
314
340
  if (action.system === 'coolify' &&
315
341
  action.kind === 'app' &&
@@ -1061,6 +1087,15 @@ async function provisionPlatformAuth(context, deps, owners = []) {
1061
1087
  },
1062
1088
  { key: 'OTEL_NODE_RESOURCE_DETECTORS', value: 'none' },
1063
1089
  { key: 'API_CACHE_CONFIG', value: '{"defaultMinutes":2,"longMinutes":5}' },
1090
+ // #405: the login app's own requestId-free fallback (`resolveRedirectUri`
1091
+ // in `wtfalch/auth`'s `login/src/lib/client.ts`) is what a user without an
1092
+ // OIDC request in flight lands on after `/signedin` -- exactly the owner
1093
+ // invite's own path (set a password, then nowhere to go). Only set once
1094
+ // Archon is declared, and only ever to Archon: the platform's one landing
1095
+ // app for anyone who reaches the login app outside an app's own sign-in.
1096
+ ...(platform.archon.enabled
1097
+ ? [{ key: 'DEFAULT_REDIRECT_URI', value: `https://${platform.archon.hostname}` }]
1098
+ : []),
1064
1099
  ];
1065
1100
  // #210: read back now, not inside the deploy-decision block below -- this
1066
1101
  // push has to land before `missingBrokerInputs` is read a few lines down,
@@ -1446,11 +1481,13 @@ async function provisionPlatformAuth(context, deps, owners = []) {
1446
1481
  await record(PLATFORM_ZITADEL_ORG_SCOPE_KEY, ownerOrgId);
1447
1482
  let invitedUserId;
1448
1483
  try {
1484
+ // ZITADEL's default invite link is `/ui/v2/login/verify`, which 404s on the login app;
1485
+ // it serves `/verify` at the root.
1449
1486
  ({ userId: invitedUserId } = await mailZitadel.inviteOrgOwner(ownerOrgId, {
1450
1487
  givenName: owner.givenName,
1451
1488
  familyName: owner.familyName,
1452
1489
  email: owner.email,
1453
- }));
1490
+ }, `https://${hostname}/verify?userId={{.UserID}}&code={{.Code}}&organization={{.OrgID}}&invite=true`));
1454
1491
  }
1455
1492
  catch (error) {
1456
1493
  // The cause, with the owner's email and the Lettermint key redacted, so a failure says why.
@@ -1791,7 +1828,8 @@ async function provisionPlatformMail(context, deps) {
1791
1828
  const declaredChanged = recorded[PLATFORM_MAIL_DEPLOYED_HOSTNAME_SCOPE_KEY] !== hostname ||
1792
1829
  recorded[PLATFORM_MAIL_DEPLOYED_GIT_REF_SCOPE_KEY] !== imageTag;
1793
1830
  // A new storage binds into the container only at its next start.
1794
- if (app.justCreated || !app.running || envsChanged || declaredChanged || storagesAdded) {
1831
+ const mailDeployedNow = app.justCreated || !app.running || envsChanged || declaredChanged || storagesAdded;
1832
+ if (mailDeployedNow) {
1795
1833
  // A container still up from before would answer the readiness probe with
1796
1834
  // the old deployment's health. Stop it and see it stopped first, so the
1797
1835
  // wait below only ever reads the new deployment.
@@ -1799,7 +1837,16 @@ async function provisionPlatformMail(context, deps) {
1799
1837
  await deps.coolify.raw('POST', `/applications/${appId}/stop`);
1800
1838
  await waitForMailStopped(deps.coolify, appId, runtime, sleep);
1801
1839
  }
1802
- await deps.coolify.deploy(appId);
1840
+ // #252: phase 1 only -- Coolify's own deployment record reaching
1841
+ // `'finished'`. Phase 2 (the readiness probe, `ensureMailReady` below)
1842
+ // runs right after the tunnel and DNS wiring that follows, which is what
1843
+ // `hostname` needs before it answers anything at all; running it here
1844
+ // instead would just poll a hostname that cannot resolve yet on a first
1845
+ // apply. Either way it runs before any applied ref is recorded.
1846
+ // The old container was stopped above (or never ran), so nothing stays live.
1847
+ await waitForDeploymentFinished(deps.coolify, appId, runtime, 'platform:mail', app.justCreated || !app.running
1848
+ ? `mail stays at ${recorded[PLATFORM_MAIL_APPLIED_GIT_REF_SCOPE_KEY] ?? 'no recorded ref'}`
1849
+ : 'mail is stopped until the next apply');
1803
1850
  await record(PLATFORM_MAIL_DEPLOYED_SCOPE_KEY, appId);
1804
1851
  // Right after the deploy, so a later failure (tunnel, DNS, readiness,
1805
1852
  // settings, mint) does not make the next run stop and redeploy the app.
@@ -1873,6 +1920,31 @@ async function provisionPlatformMail(context, deps) {
1873
1920
  proxied: false,
1874
1921
  });
1875
1922
  }
1923
+ // #252: phase 2, now that the tunnel and DNS above actually route
1924
+ // `hostname`. `'ready'`, not `'version'`: Stalwart's `/healthz/ready`
1925
+ // carries no version field. Run once per apply, whichever of the three
1926
+ // things below needs it first -- a redeploy just above, the OIDC-audience
1927
+ // re-apply right after, or the admin-key mint further down, none of which
1928
+ // can reach an instance that is not up yet either. Combined with phase 1
1929
+ // above having already confirmed Coolify's own deployment record
1930
+ // finished, this is what actually closes the loophole named in #252 (an
1931
+ // old container staying healthy while the deployment itself fails) -- not
1932
+ // this probe alone, which a container that merely kept answering could
1933
+ // still pass.
1934
+ let mailReadyConfirmed = false;
1935
+ const ensureMailReady = async () => {
1936
+ if (mailReadyConfirmed)
1937
+ return;
1938
+ await waitForHealthyVersion(appId, { kind: 'ready', url: `https://${hostname}${STALWART_READY_PATH}` }, runtime, sleep, 'platform:mail', `mail stays at ${recorded[PLATFORM_MAIL_APPLIED_GIT_REF_SCOPE_KEY] ?? 'no recorded ref'}`);
1939
+ mailReadyConfirmed = true;
1940
+ };
1941
+ // Also probe when the applied ref lags the target: the deployed refs are
1942
+ // recorded before this probe, so a probe that timed out on the last run must
1943
+ // run again, not be skipped because the deploy itself is already recorded.
1944
+ const appliedLagsTarget = recorded[PLATFORM_MAIL_APPLIED_HOSTNAME_SCOPE_KEY] !== hostname ||
1945
+ recorded[PLATFORM_MAIL_APPLIED_GIT_REF_SCOPE_KEY] !== imageTag;
1946
+ if (mailDeployedNow || appliedLagsTarget)
1947
+ await ensureMailReady();
1876
1948
  // ---- the mail ZITADEL project, in the platform's sign-in -------------------
1877
1949
  // #346: found or created on every run, like the files/Archon steps' own
1878
1950
  // projects -- its id is the OIDC audience Stalwart's directory checks an
@@ -1898,7 +1970,7 @@ async function provisionPlatformMail(context, deps) {
1898
1970
  // run to retry. A project deleted out of band is noticed only here, on
1899
1971
  // the next plan/apply -- the same way the files/Archon steps' own
1900
1972
  // projects are.
1901
- await waitForMailInstance(`https://${hostname}${STALWART_READY_PATH}`, runtime, sleep);
1973
+ await ensureMailReady();
1902
1974
  const recoveryEnv = (await deps.coolify.listEnvs(appId)).find((e) => e.key === 'STALWART_RECOVERY_ADMIN' && !e.is_preview);
1903
1975
  if (!recoveryEnv) {
1904
1976
  throw new Error('platform:mail: STALWART_RECOVERY_ADMIN is not set on the app; cannot authenticate to re-apply the OIDC audience');
@@ -1938,7 +2010,7 @@ async function provisionPlatformMail(context, deps) {
1938
2010
  if (lost) {
1939
2011
  throw new Error(`platform:mail: an admin API key ("${lost}") was minted but never delivered; revoke it in Stalwart and remove the recorded key ${PLATFORM_MAIL_ADMIN_UNDELIVERED_SCOPE_KEY} to mint a new one`);
1940
2012
  }
1941
- await waitForMailInstance(`https://${hostname}${STALWART_READY_PATH}`, runtime, sleep);
2013
+ await ensureMailReady();
1942
2014
  const recoveryEnv = (await deps.coolify.listEnvs(appId)).find((e) => e.key === 'STALWART_RECOVERY_ADMIN' && !e.is_preview);
1943
2015
  if (!recoveryEnv) {
1944
2016
  throw new Error('platform:mail: STALWART_RECOVERY_ADMIN is not set on the app; cannot authenticate to mint the admin key');
@@ -2293,7 +2365,15 @@ export const PLATFORM_ARCHON_CLIENT_ID_SCOPE_KEY = 'zitadel_platform_archon_clie
2293
2365
  * does not parse fails the step without printing it.
2294
2366
  * - For each of files, ai, integrations and jobs that the platform declares
2295
2367
  * enabled: its `https://<hostname>` URL, and its management key when
2296
- * {@link PlatformArchonDeps.heldKeys} holds one.
2368
+ * {@link PlatformArchonDeps.heldKeys} holds one. Files also gets
2369
+ * `FILES_ORGANISATION_ID` (`platform.ownerOrgId`, Company Included's own
2370
+ * Files organisation -- `executor.ts`'s `filesOrganisation` provisions it
2371
+ * under that same id) and its signed-assertion identity
2372
+ * (`FILES_MANAGEMENT_ISSUER`, `_KEY_ID`, `_PRIVATE_KEY`; #436).
2373
+ * - `COOLIFY_URL`/`COOLIFY_READ_TOKEN` for `/backups`, once a human has
2374
+ * created a Coolify token scoped to read only and held it under
2375
+ * {@link PLATFORM_ARCHON_COOLIFY_READ_TOKEN_REF} -- that ref's own doc
2376
+ * comment says why foundry cannot mint one itself (#436).
2297
2377
  *
2298
2378
  * The OIDC client is ZITADEL's public web client (authorization code with
2299
2379
  * PKCE, no client secret: `createOidcApp` never returns one, see
@@ -2318,9 +2398,6 @@ export const PLATFORM_ARCHON_CLIENT_ID_SCOPE_KEY = 'zitadel_platform_archon_clie
2318
2398
  * needed so the two apps agree the moment the broker can run.
2319
2399
  *
2320
2400
  * Known gaps, left for review:
2321
- * - `AUTHZ_BOOTSTRAP_OPERATOR`: the first operator's ZITADEL subject id. A
2322
- * person has to exist in the new instance first; not something foundry
2323
- * mints.
2324
2401
  * - Foundry's own database URL. Archon runs foundry in its panel, but foundry
2325
2402
  * does not know its own connection string at plan/apply time: that is host
2326
2403
  * config, not something `DeclaredPlatform` or held keys carry.
@@ -2328,10 +2405,7 @@ export const PLATFORM_ARCHON_CLIENT_ID_SCOPE_KEY = 'zitadel_platform_archon_clie
2328
2405
  * steps are exported by those steps' own modules. A
2329
2406
  * missing one blocks the plan (`blockedSteps`); it is never skipped. The
2330
2407
  * env names `AI_*`, `INTEGRATIONS_*`, `JOBS_URL` and `JOBS_ADMIN_TOKEN` are
2331
- * this step's proposal; Archon reads only `FILES_URL` and
2332
- * `FILES_MANAGEMENT_KEY` of these today. Files' signed-assertion identity
2333
- * (`FILES_MANAGEMENT_ISSUER`, `_KEY_ID`, `_PRIVATE_KEY`, `_SCOPE`) and
2334
- * `FILES_ORGANISATION_ID` are not wired.
2408
+ * this step's proposal.
2335
2409
  * - The step plans again when the hostname, `gitRef` or the Access policy's
2336
2410
  * include list changes. Env for a service enabled later is not pushed until
2337
2411
  * one of those changes.
@@ -2347,6 +2421,8 @@ async function provisionPlatformArchon(context, deps) {
2347
2421
  const recorded = context.recordedIds();
2348
2422
  const issuer = `https://${platform.auth.hostname}`;
2349
2423
  const appliedGitRef = recorded[PLATFORM_ARCHON_APPLIED_GIT_REF_SCOPE_KEY];
2424
+ const runtime = deps.runtime ?? {};
2425
+ const sleep = runtime.sleep ?? defaultSleep;
2350
2426
  if (deps.accessEmails.length === 0) {
2351
2427
  throw new Error('platform:archon: no Access emails given; refusing to put Archon behind an Access app nobody can pass');
2352
2428
  }
@@ -2537,12 +2613,66 @@ async function provisionPlatformArchon(context, deps) {
2537
2613
  throw new Error(`platform:archon: app ${appId} lost its AUTH_APP_KEY between two reads`);
2538
2614
  }
2539
2615
  envs.push({ key: 'AUTH_APP_KEY', value: appKey });
2616
+ // The owner `platform:auth` invited is Archon's first operator.
2617
+ const bootstrapOperator = archonBootstrapOperator(recorded);
2618
+ if (bootstrapOperator)
2619
+ envs.push({ key: 'AUTHZ_BOOTSTRAP_OPERATOR', value: bootstrapOperator });
2620
+ // #406: the invited platform owner's ZITADEL subject id, once
2621
+ // `platform:auth` has recorded one -- the guard at the top of this
2622
+ // function already requires `platform:auth` to have been applied before
2623
+ // this step is reached, and `provisionPlatformAuth` records the owner
2624
+ // invite (or, with no owner declared, the literal string 'none') as part
2625
+ // of that same step, so by the time this runs the value here is whatever
2626
+ // is actually true. `bootstrapOperator` (archon's `src/lib/authz/
2627
+ // bootstrap.ts`) compares this verbatim against a signed-in user's
2628
+ // subject id, so 'none' is skipped rather than written -- it would simply
2629
+ // never match, but it is not a real id and should not look like one.
2630
+ const ownerInviteUserId = recorded[PLATFORM_AUTH_OWNER_INVITE_SCOPE_KEY];
2631
+ if (ownerInviteUserId && ownerInviteUserId !== 'none') {
2632
+ envs.push({ key: 'AUTHZ_BOOTSTRAP_OPERATOR', value: ownerInviteUserId });
2633
+ }
2540
2634
  // A value with a line break (the PEM management keys) must be flagged:
2541
2635
  // unflagged, Coolify writes it into `.env` unquoted and the container never
2542
2636
  // starts.
2543
2637
  envs.push(...inputs.envs.map((e) => (e.value.includes('\n') ? { ...e, multiline: true } : e)));
2638
+ // #436: /backups needs a Coolify token scoped to read only, never the
2639
+ // root token `putArchonMasterKeys` above already held. Nothing mints that
2640
+ // token over Coolify's API (this ref's own doc comment), so until a human
2641
+ // creates one and holds it under PLATFORM_ARCHON_COOLIFY_READ_TOKEN_REF,
2642
+ // this pushes nothing and /backups keeps showing its own "set COOLIFY_URL
2643
+ // and COOLIFY_READ_TOKEN" message -- never a reason to block the rest of
2644
+ // this step.
2645
+ const coolifyReadToken = await deps.heldKeys.resolve(PLATFORM_ARCHON_COOLIFY_READ_TOKEN_REF);
2646
+ if (coolifyReadToken) {
2647
+ const coolifyUrl = await deps.heldKeys.resolve(PLATFORM_ARCHON_MASTER_COOLIFY_URL_REF);
2648
+ if (coolifyUrl) {
2649
+ envs.push({ key: 'COOLIFY_URL', value: coolifyUrl }, { key: 'COOLIFY_READ_TOKEN', value: coolifyReadToken });
2650
+ }
2651
+ }
2544
2652
  const changed = await envsDiffer(deps.coolify, appId, envs);
2545
2653
  await deps.coolify.bulkUpsertEnvs(appId, envs);
2654
+ // #436 round 2: `bulkUpsertEnvs` only adds or updates, so the
2655
+ // `FILES_MANAGEMENT_KEY` this step used to write before the fix above
2656
+ // would otherwise survive on an Archon already deployed with it --
2657
+ // `filesManagementFetch` checks that var first, so a stale copy kept
2658
+ // steering it into the issued-key path this issue's wiring fixes, and it
2659
+ // left a second copy of the private key sitting in Coolify. Every row
2660
+ // under this key, including `listEnvs`'s own preview duplicate, is
2661
+ // deleted outright -- never overwritten, which would still leave a row.
2662
+ //
2663
+ // #462: the deletes happen after `changed` is already computed above, so
2664
+ // a retry that finds the envs already matching (`changed` false) but a
2665
+ // stale `FILES_MANAGEMENT_KEY` row still present must still redeploy --
2666
+ // the running container otherwise keeps reading the stale key out of its
2667
+ // own process env until someone redeploys by hand. `staleKeyDeleted`
2668
+ // feeds `archonDeployedNow` below for exactly that case.
2669
+ let staleKeyDeleted = false;
2670
+ for (const stale of await deps.coolify.listEnvs(appId)) {
2671
+ if (stale.key === 'FILES_MANAGEMENT_KEY' && stale.uuid) {
2672
+ await deps.coolify.deleteEnv(appId, stale.uuid);
2673
+ staleKeyDeleted = true;
2674
+ }
2675
+ }
2546
2676
  // ---- register with the broker (#204) ---------------------------------------
2547
2677
  // After Archon's own `AUTH_APP_KEY` is durably written above, so a throw
2548
2678
  // earlier in this function (a bad database URL, a Cloudflare failure) never
@@ -2645,12 +2775,33 @@ async function provisionPlatformArchon(context, deps) {
2645
2775
  await record(PLATFORM_ARCHON_ACCESS_SCOPE_KEY, accessAppId);
2646
2776
  }
2647
2777
  // ---- deploy ---------------------------------------------------------------
2648
- if (app.justCreated ||
2778
+ // #252: phase 1 of the shared deploy-health helper only -- Coolify's own
2779
+ // deployment record reaching `'finished'`, which is enough on its own to
2780
+ // catch the bug this closes (the old container stays healthy regardless
2781
+ // of whether the deployment record itself fails). Phase 2, the health
2782
+ // check, has to wait until after the tunnel and DNS below are wired: on a
2783
+ // first apply `hostname` answers nothing until then, and splitting the two
2784
+ // phases (`waitForDeploymentFinished` then `waitForHealthyVersion`,
2785
+ // instead of one `waitForNewDeployment` call) is exactly for a step shaped
2786
+ // like this one.
2787
+ // #398: the integrations key id this run wrote into Archon's env. Archon
2788
+ // counts as synced to it only after a deployment finishes and the health
2789
+ // check passes, so a still-unsynced id forces both even when nothing else
2790
+ // changed (a run that died between the env write and the deploy).
2791
+ const integrationsKeyId = inputs.envs.find((e) => e.key === 'INTEGRATIONS_MANAGEMENT_KEY_ID')?.value;
2792
+ // Not forced on a platform recorded before #398, which has no recorded id.
2793
+ const keyUnsynced = Boolean(integrationsKeyId) &&
2794
+ Boolean(recorded[PLATFORM_INTEGRATIONS_KEY_ID_SCOPE_KEY]) &&
2795
+ recorded[PLATFORM_ARCHON_INTEGRATIONS_KEY_SYNCED_SCOPE_KEY] !== integrationsKeyId;
2796
+ const archonDeployedNow = keyUnsynced ||
2797
+ app.justCreated ||
2649
2798
  !app.running ||
2650
2799
  changed ||
2800
+ staleKeyDeleted ||
2651
2801
  appliedGitRef !== gitRef ||
2652
- recorded[PLATFORM_ARCHON_APPLIED_HOSTNAME_SCOPE_KEY] !== hostname) {
2653
- await deps.coolify.deploy(appId);
2802
+ recorded[PLATFORM_ARCHON_APPLIED_HOSTNAME_SCOPE_KEY] !== hostname;
2803
+ if (archonDeployedNow) {
2804
+ await waitForDeploymentFinished(deps.coolify, appId, runtime, 'platform:archon', `archon stays at ${appliedGitRef ?? 'no recorded ref'} on ${recorded[PLATFORM_ARCHON_APPLIED_HOSTNAME_SCOPE_KEY] ?? 'no recorded hostname'}`);
2654
2805
  }
2655
2806
  // ---- tunnel and DNS ---------------------------------------------------------
2656
2807
  const existingTunnel = await deps.cloudflare.findTunnelByName(PLATFORM_ARCHON_TUNNEL_NAME);
@@ -2704,6 +2855,24 @@ async function provisionPlatformArchon(context, deps) {
2704
2855
  proxied: true,
2705
2856
  });
2706
2857
  }
2858
+ // #252: phase 2, now that the tunnel and DNS above actually route
2859
+ // `hostname`. `'ready'`, not `'version'`: archon's `/api/health`
2860
+ // (`@wtfalch/reporting`'s `healthHandler`, the same route `executor.ts`'s
2861
+ // `healthUrls` already assumes every hosted app serves) carries no
2862
+ // version field this package knows to check, so a plain 2xx is the mode
2863
+ // available -- combined with phase 1 above having already confirmed
2864
+ // Coolify's own deployment record finished, which is what actually closes
2865
+ // the loophole named in #252 (an old container staying healthy while the
2866
+ // deployment itself fails).
2867
+ if (archonDeployedNow) {
2868
+ await waitForHealthyVersion(appId, { kind: 'ready', url: `https://${hostname}/api/health` }, runtime, sleep, 'platform:archon', `archon stays at ${appliedGitRef ?? 'no recorded ref'} on ${recorded[PLATFORM_ARCHON_APPLIED_HOSTNAME_SCOPE_KEY] ?? 'no recorded hostname'}`);
2869
+ }
2870
+ // #398: the deployment finished and answers health, so Archon now runs with
2871
+ // the key id its env carries. `provisionPlatformIntegrations` drops the
2872
+ // previous public key only on this marker.
2873
+ if (keyUnsynced && integrationsKeyId) {
2874
+ await record(PLATFORM_ARCHON_INTEGRATIONS_KEY_SYNCED_SCOPE_KEY, integrationsKeyId);
2875
+ }
2707
2876
  // Last, after every write the applied values depend on, so a run that failed
2708
2877
  // halfway plans again.
2709
2878
  await record(PLATFORM_ARCHON_APPLIED_HOSTNAME_SCOPE_KEY, hostname);
@@ -2713,6 +2882,7 @@ async function provisionPlatformArchon(context, deps) {
2713
2882
  // (whose AUTH_APPS never carried this registration) from one still holding
2714
2883
  // it.
2715
2884
  await record(PLATFORM_ARCHON_APPLIED_BROKER_SCOPE_KEY, brokerAppId);
2885
+ await record(PLATFORM_ARCHON_APPLIED_OPERATOR_SCOPE_KEY, bootstrapOperator ?? 'none');
2716
2886
  return { resourceId: appId };
2717
2887
  }
2718
2888
  // #211 round 6: mirrors `wtfalch/auth/broker/src/apps.ts`'s own `isValidFrom`
@@ -3088,6 +3258,32 @@ async function archonBrokerRegistrationDrifted(coolify, recordedIds) {
3088
3258
  return false;
3089
3259
  }
3090
3260
  }
3261
+ /**
3262
+ * #406: whether the platform owner invite recorded under
3263
+ * {@link PLATFORM_AUTH_OWNER_INVITE_SCOPE_KEY} is a real user id (not
3264
+ * absent, not the literal `'none'` {@link provisionPlatformAuth} records
3265
+ * when no owner is declared) that Archon's live env does not yet carry as
3266
+ * `AUTHZ_BOOTSTRAP_OPERATOR`. `provisionPlatformAuth`'s own invite can
3267
+ * record after Archon's `coolify:app` id already has -- the owner invite
3268
+ * happens inside `platform:auth` only once its Lettermint token is held,
3269
+ * which Archon's own apply does not wait on -- so without this, an owner
3270
+ * invited after Archon's first deploy would need a redeploy by hand, the
3271
+ * way companyincluded.com's did. A failed read is not drift, the same rule
3272
+ * {@link archonAccessDrifted} and {@link archonBrokerRegistrationDrifted}
3273
+ * follow: the next plan reads again.
3274
+ */
3275
+ async function archonBootstrapOperatorDrifted(coolify, appId, recordedIds) {
3276
+ const invitedUserId = recordedIds[PLATFORM_AUTH_OWNER_INVITE_SCOPE_KEY];
3277
+ if (!appId || !invitedUserId || invitedUserId === 'none')
3278
+ return false;
3279
+ try {
3280
+ const live = await currentEnvValue(coolify, appId, 'AUTHZ_BOOTSTRAP_OPERATOR');
3281
+ return live !== invitedUserId;
3282
+ }
3283
+ catch {
3284
+ return false;
3285
+ }
3286
+ }
3091
3287
  /**
3092
3288
  * What Archon reads from each enabled service, gathered before anything is
3093
3289
  * built. Either everything is there and the env values come back, or the first
@@ -3105,6 +3301,23 @@ async function collectArchonServiceInputs(platform, deps, recorded) {
3105
3301
  if (services.length > 0 && !deps.heldKeys) {
3106
3302
  return block(`the host built the platform executor without a held-key reader, and ${services.join(', ')} ${services.length === 1 ? 'is' : 'are'} enabled`);
3107
3303
  }
3304
+ const rotationUndelivered = recorded[PLATFORM_INTEGRATIONS_KEY_ROTATION_UNDELIVERED_SCOPE_KEY];
3305
+ if (services.includes('integrations') &&
3306
+ rotationUndelivered &&
3307
+ rotationUndelivered !== recorded[PLATFORM_INTEGRATIONS_KEY_ID_SCOPE_KEY]) {
3308
+ // #398: the held key may already be the new pair while the recorded id is
3309
+ // still the old one; writing both would sign with a key id the runner
3310
+ // does not match.
3311
+ return block(`the integrations rotation key ${rotationUndelivered} is minted but not confirmed delivered and pinned`);
3312
+ }
3313
+ const activeKeyId = recorded[PLATFORM_INTEGRATIONS_KEY_ID_SCOPE_KEY];
3314
+ if (services.includes('integrations') &&
3315
+ activeKeyId &&
3316
+ recorded[PLATFORM_INTEGRATIONS_RUNNER_KEYRING_DEPLOYED_SCOPE_KEY] !== activeKeyId) {
3317
+ // #398: the runner must run with the new public key before Archon signs
3318
+ // with the new private key.
3319
+ return block(`the integrations runner is not yet deployed with management key ${activeKeyId}`);
3320
+ }
3108
3321
  for (const service of services) {
3109
3322
  const needs = service === 'jobs'
3110
3323
  ? {
@@ -3138,11 +3351,31 @@ async function collectArchonServiceInputs(platform, deps, recorded) {
3138
3351
  const held = (ref) => heldByRef.get(ref);
3139
3352
  for (const service of services) {
3140
3353
  if (service === 'files') {
3141
- envs.push({
3142
- key: 'FILES_URL',
3143
- value: recorded[PLATFORM_FILES_INTERNAL_URL_SCOPE_KEY],
3144
- }, {
3145
- key: 'FILES_MANAGEMENT_KEY',
3354
+ envs.push(
3355
+ // #436 round 2: not `PLATFORM_FILES_INTERNAL_URL_SCOPE_KEY`'s
3356
+ // internal `http://<name>:3000` -- Archon's signed path needs an
3357
+ // exact HTTPS origin (`filesOrigin()`/`filesBaseUrl()`,
3358
+ // `wtfalch/archon`'s `src/lib/authz/files-management.ts` and
3359
+ // `src/lib/files-admin/client.ts`), and that origin is also the
3360
+ // signing audience, so it must equal Files' own `APP_URL` exactly
3361
+ // (this file, the files runner env above: `https://${platform.
3362
+ // files.hostname}`).
3363
+ { key: 'FILES_URL', value: `https://${platform.files.hostname}` },
3364
+ // #436: Company Included's own Files organisation, provisioned by
3365
+ // the ordinary per-org `planFilesOrganisation` step under its own
3366
+ // org id -- `executor.ts`'s `filesOrganisation` calls
3367
+ // `provisionOrganisation({ organisationId: org.id, ... })`, so
3368
+ // `platform.ownerOrgId` is already that organisation's id, with no
3369
+ // separate platform-level mint needed.
3370
+ { key: 'FILES_ORGANISATION_ID', value: platform.ownerOrgId },
3371
+ // #436: the signed-assertion identity `filesSigner()` reads
3372
+ // (`archon`'s `src/lib/authz/files-management.ts`), not the bearer
3373
+ // token `FILES_MANAGEMENT_KEY` used to name here -- that name made
3374
+ // Archon take the issued-key branch of `filesManagementFetch`,
3375
+ // which then failed on the `FILES_MANAGEMENT_SCOPE` this step never
3376
+ // set, surfacing as "Files is not configured" (foundry#436).
3377
+ { key: 'FILES_MANAGEMENT_ISSUER', value: FILES_MANAGEMENT_ISSUER }, { key: 'FILES_MANAGEMENT_KEY_ID', value: FILES_MANAGEMENT_KEY_ID }, {
3378
+ key: 'FILES_MANAGEMENT_PRIVATE_KEY',
3146
3379
  value: held(ARCHON_SERVICE_INPUTS.files.refs[0]),
3147
3380
  });
3148
3381
  }
@@ -3168,7 +3401,10 @@ async function collectArchonServiceInputs(platform, deps, recorded) {
3168
3401
  value: INTEGRATIONS_MANAGEMENT_ISSUER,
3169
3402
  }, {
3170
3403
  key: 'INTEGRATIONS_MANAGEMENT_KEY_ID',
3171
- value: INTEGRATIONS_MANAGEMENT_KEY_ID,
3404
+ // #398: the active id, which a rotation moves past `foundry-1` --
3405
+ // falls back to the fixed constant for a platform that applied
3406
+ // integrations before this scope key existed.
3407
+ value: recorded[PLATFORM_INTEGRATIONS_KEY_ID_SCOPE_KEY] ?? INTEGRATIONS_MANAGEMENT_KEY_ID,
3172
3408
  }, { key: 'INTEGRATIONS_MANAGEMENT_AUDIENCE', value: url });
3173
3409
  }
3174
3410
  else {
@@ -3717,9 +3953,15 @@ async function provisionPlatformAi(context, deps) {
3717
3953
  await coolify.raw('POST', `/applications/${migrate.appId}/stop`);
3718
3954
  await waitForStopped(coolify, migrate.appId, runtime, sleep);
3719
3955
  }
3720
- await coolify.deploy(migrate.appId);
3956
+ const migrateDeployed = await coolify.deploy(migrate.appId);
3721
3957
  try {
3722
3958
  await waitForAiMigration(coolify, migrate.appId, runtime, sleep);
3959
+ // #399: the migrate job can read healthy well before Coolify's own
3960
+ // deployment does -- it is still waiting out the image's HEALTHCHECK
3961
+ // start period -- so stopping it below waits for Coolify's own record
3962
+ // too, or it can find the container already gone and mark a deployment
3963
+ // that actually migrated as `failed`.
3964
+ await waitForMigrateDeployment(coolify, migrate.appId, queuedDeploymentUuid(migrateDeployed) ?? 'untracked', 'platform:ai', 'ai-migrate', runtime.now ?? Date.now, runtime.migrateTimeoutMs ?? 600_000, runtime.migrateIntervalMs ?? 5_000, sleep);
3723
3965
  }
3724
3966
  catch (error) {
3725
3967
  if (error instanceof MigrationNotHealthyError) {
@@ -3732,49 +3974,44 @@ async function provisionPlatformAi(context, deps) {
3732
3974
  }
3733
3975
  await coolify.raw('POST', `/applications/${migrate.appId}/stop`);
3734
3976
  }
3735
- // `deploy` only queues a build that takes minutes, so the id of the host's
3736
- // deployment is kept for the mint below to wait on.
3737
- const runnerDeployment = deployRunner ? await coolify.deploy(runner.appId) : undefined;
3977
+ // #252: confirmed before anything below -- the applied ref recorded just
3978
+ // after, and the mint further down -- depends on the new version actually
3979
+ // being live. Never read the old container's healthy status as the new
3980
+ // deployment's (`CLAUDE.md`'s platform-step rule 5): `'deployment-only'`,
3981
+ // because the runner's own doc comment above says why there is no url to
3982
+ // check instead ("Nothing public is wired: no Coolify `domains`, tunnel,
3983
+ // DNS record or Access app"). This replaces the old deploy-then-defer-the-
3984
+ // wait-until-the-mint-needs-it shape (#375) with the one every other
3985
+ // platform step uses: the wait happens once, right here, so a failed
3986
+ // deployment is caught before either thing downstream trusts it.
3738
3987
  if (deployRunner) {
3739
- await record(PLATFORM_AI_RUNNER_DEPLOYMENT_SCOPE_KEY, queuedDeploymentUuid(runnerDeployment) ?? 'untracked');
3988
+ await waitForNewDeployment(coolify, runner.appId, { kind: 'deployment-only' }, runtime, sleep, 'platform:ai', `the runner stays at ${recorded[PLATFORM_AI_APPLIED_GITREF_SCOPE_KEY] ?? 'no recorded ref'}`);
3989
+ }
3990
+ if (deployWorker) {
3991
+ await waitForNewDeployment(coolify, worker.appId, { kind: 'deployment-only' }, runtime, sleep, 'platform:ai', `the worker stays at ${recorded[PLATFORM_AI_APPLIED_GITREF_SCOPE_KEY] ?? 'no recorded ref'}`);
3740
3992
  }
3741
- if (deployWorker)
3742
- await coolify.deploy(worker.appId);
3743
3993
  await record(PLATFORM_AI_DEPLOY_PENDING_SCOPE_KEY, '');
3744
- // The deploys are done, so record what they deployed now. A later failure
3745
- // at the mint or delivery must not make the next apply see a changed
3746
- // hostname or gitRef and redeploy the live host again. The planner still
3747
- // plans the step until the resource id is recorded, once the key is delivered.
3994
+ // The deploys are confirmed, so record what they deployed now. A later
3995
+ // failure at the mint or delivery must not make the next apply see a
3996
+ // changed hostname or gitRef and redeploy the live host again. The
3997
+ // planner still plans the step until the resource id is recorded, once
3998
+ // the key is delivered.
3748
3999
  await record(PLATFORM_AI_APPLIED_HOSTNAME_SCOPE_KEY, hostname);
3749
4000
  await record(PLATFORM_AI_APPLIED_GITREF_SCOPE_KEY, gitRef);
3750
4001
  const internalUrl = `http://${PLATFORM_AI_RUNNER.internalName}:${PLATFORM_AI_PORT}`;
3751
4002
  await record(PLATFORM_AI_INTERNAL_URL_SCOPE_KEY, internalUrl);
3752
4003
  // ---- Archon's AI management key, once ------------------------------------
4004
+ // #252: nothing left to wait on here -- the deployment above is already
4005
+ // confirmed finished by the time this runs. The host's own readiness
4006
+ // beyond that (a 503 for a moment after `AI_WORKER_TIMEOUT_MS` warms up)
4007
+ // is unchanged from before this change: a single mint attempt, not
4008
+ // retried here.
3753
4009
  const now = context.recordedIds();
3754
4010
  if (!now[PLATFORM_AI_MANAGEMENT_KEY_SCOPE_KEY]) {
3755
4011
  const lost = now[PLATFORM_AI_MANAGEMENT_KEY_UNDELIVERED_SCOPE_KEY];
3756
4012
  if (lost) {
3757
4013
  throw new Error(`platform:ai: AI management key ${lost} was minted but never delivered; revoke it and remove the recorded key ${PLATFORM_AI_MANAGEMENT_KEY_UNDELIVERED_SCOPE_KEY} to mint a new one`);
3758
4014
  }
3759
- // The host answers 503 (or nothing) until the deployment that carries its
3760
- // management settings has finished.
3761
- const queued = context.recordedIds()[PLATFORM_AI_RUNNER_DEPLOYMENT_SCOPE_KEY];
3762
- if (queued) {
3763
- try {
3764
- await waitForAiDeployment(coolify, runner.appId, queued, runtime, sleep);
3765
- }
3766
- catch (error) {
3767
- // Only a build still running is worth waiting on again. A deployment
3768
- // that failed, was cancelled or cannot be read is not: mark the host
3769
- // pending so the next apply queues a new one instead of polling it.
3770
- if (!(error instanceof AiDeploymentTimeoutError)) {
3771
- await record(PLATFORM_AI_DEPLOY_PENDING_SCOPE_KEY, '1');
3772
- await record(PLATFORM_AI_RUNNER_DEPLOYMENT_SCOPE_KEY, '');
3773
- }
3774
- throw error;
3775
- }
3776
- await record(PLATFORM_AI_RUNNER_DEPLOYMENT_SCOPE_KEY, '');
3777
- }
3778
4015
  // #201: a test injects its own minter; without one, build the real one
3779
4016
  // over ai's own POST /v1/platform/keys, signed with the key pinned above.
3780
4017
  const minter = deps.managementKeyMinter ??
@@ -3914,22 +4151,23 @@ function queuedDeploymentUuid(result) {
3914
4151
  return typeof uuid === 'string' && uuid ? uuid : undefined;
3915
4152
  }
3916
4153
  /**
3917
- * Waits until the host's queued deployment has `finished`; throws on
3918
- * `failed`/`cancelled-by-user` or once `migrateTimeoutMs` has passed. With no
3919
- * deployment id (`untracked`) the only signal left is the app's own status,
3920
- * which an old container can already satisfy, so that is best effort.
4154
+ * Waits until `deploymentUuid`'s Coolify deployment has `finished`; throws on
4155
+ * `failed`/`cancelled-by-user` (via `onEnded`) or once `timeoutMs` has passed
4156
+ * (via `onTimeout`). With no deployment id (`untracked`) the only signal left
4157
+ * is the app's own status, which an old container can already satisfy, so
4158
+ * that case is best effort and returns as soon as the app itself reads
4159
+ * healthy. Shared by every caller that must not act on an app's status
4160
+ * until Coolify's own deployment record agrees it is done (#399: Coolify's
4161
+ * own `HEALTHCHECK` start period can outlive the app already reading
4162
+ * `running:healthy`; acting then finds the container already gone, and
4163
+ * Coolify marks its own deployment `failed` -- "Removing the new version" --
4164
+ * though whatever the app was doing had already finished).
3921
4165
  */
3922
- /** The host's deployment was still running when the wait gave up. */
3923
- class AiDeploymentTimeoutError extends Error {
3924
- }
3925
- async function waitForAiDeployment(coolify, appId, deploymentUuid, runtime, sleep) {
3926
- const now = runtime.now ?? Date.now;
3927
- const timeout = runtime.migrateTimeoutMs ?? 600_000;
3928
- const interval = runtime.migrateIntervalMs ?? 5_000;
3929
- const deadline = now() + timeout;
4166
+ async function waitForCoolifyDeployment(coolify, appId, deploymentUuid, now, timeoutMs, intervalMs, sleep, onEnded, onTimeout) {
4167
+ const deadline = now() + timeoutMs;
3930
4168
  let status;
3931
4169
  for (;;) {
3932
- await sleep(interval);
4170
+ await sleep(intervalMs);
3933
4171
  if (deploymentUuid === 'untracked') {
3934
4172
  status = (await coolify.getApplication(appId)).status;
3935
4173
  if (status?.startsWith('running:healthy'))
@@ -3940,15 +4178,25 @@ async function waitForAiDeployment(coolify, appId, deploymentUuid, runtime, slee
3940
4178
  status = typeof found?.status === 'string' ? found.status : undefined;
3941
4179
  if (status === 'finished')
3942
4180
  return;
3943
- if (status === 'failed' || status === 'cancelled-by-user') {
3944
- throw new Error(`platform:ai: the host's deployment for ${appId} ended ${status}; the AI management key was not minted. Check its build log in Coolify, then apply again`);
3945
- }
3946
- }
3947
- if (now() >= deadline) {
3948
- throw new AiDeploymentTimeoutError(`platform:ai: the host's deployment for ${appId} did not finish within ${Math.round(timeout / 1000)}s (last status ${status ?? 'unknown'}); the AI management key was not minted. Apply again`);
4181
+ if (status === 'failed' || status === 'cancelled-by-user')
4182
+ throw onEnded(status);
3949
4183
  }
4184
+ if (now() >= deadline)
4185
+ throw onTimeout(status ?? 'unknown');
3950
4186
  }
3951
4187
  }
4188
+ /**
4189
+ * Waits until a migrate job's own queued deployment has `finished`, so stopping a migrate job never races Coolify's own
4190
+ * `HEALTHCHECK` start period (#399: every platform migrate job idles rather
4191
+ * than exits once its migration succeeds, so it reports `running:healthy`
4192
+ * well before Coolify's own deployment does, and stopping it on that signal
4193
+ * alone can make Coolify mark a deployment that actually migrated `failed`).
4194
+ * `label` and `jobName` build an error naming the step and the job, never
4195
+ * just "the deployment".
4196
+ */
4197
+ async function waitForMigrateDeployment(coolify, appId, deploymentUuid, label, jobName, now, timeoutMs, intervalMs, sleep) {
4198
+ await waitForCoolifyDeployment(coolify, appId, deploymentUuid, now, timeoutMs, intervalMs, sleep, (status) => new Error(`${label}: ${jobName} (${appId})'s deployment ended ${status}; check its build log in Coolify, then apply again`), (status) => new Error(`${label}: ${jobName} (${appId})'s deployment did not finish within ${Math.round(timeoutMs / 1000)}s (last status ${status}); apply again`));
4199
+ }
3952
4200
  /**
3953
4201
  * Why a mint failed, safe to show: the status ai answered (the minter's own
3954
4202
  * fixed message), or a network error's `name` and `cause.code`. Never a
@@ -4404,7 +4652,9 @@ const PLATFORM_FILES_RUNNER_SCOPE_KEY = 'coolify_platform_files_runner_app';
4404
4652
  * Coolify network, so the host must run there to use it for management
4405
4653
  * calls.
4406
4654
  */
4407
- async function provisionPlatformFiles(context, deps) {
4655
+ async function provisionPlatformFiles(context, deps,
4656
+ /** {@link filesShareDomains} of every declared org: each is a runner domain and a public host. */
4657
+ shareDomains) {
4408
4658
  const { platform } = context;
4409
4659
  const gitRef = platform.files.gitRef;
4410
4660
  const recorded = context.recordedIds();
@@ -4506,7 +4756,11 @@ async function provisionPlatformFiles(context, deps) {
4506
4756
  dockerfile_target_build: 'runner',
4507
4757
  custom_internal_name: FILES_RUNNER_INTERNAL_NAME,
4508
4758
  git_branch: gitRef,
4759
+ // Coolify wants full URLs. Written only when the set changed, so a runner
4760
+ // nobody gave a share host keeps no `domains`.
4761
+ ...filesShareDomainsPatch(recorded, shareDomains),
4509
4762
  });
4763
+ context.recordScopeId(PLATFORM_FILES_SHARE_DOMAINS_SCOPE_KEY, normalHosts(shareDomains).join(','));
4510
4764
  // ---- database env ----------------------------------------------------------
4511
4765
  // The migrate app sets `files_rt`'s password from DATABASE_URL on every
4512
4766
  // boot (files' `scripts/db-runtime-role.sh`), and the runner connects with
@@ -4557,11 +4811,7 @@ async function provisionPlatformFiles(context, deps) {
4557
4811
  value: deps.clamd.port,
4558
4812
  });
4559
4813
  }
4560
- if (deps.publicHosts)
4561
- runnerOptionalEnvs.push({
4562
- key: 'FILES_PUBLIC_HOSTS',
4563
- value: deps.publicHosts,
4564
- });
4814
+ runnerOptionalEnvs.push(...filesPublicHostsEnvs(recorded, deps.publicHosts, shareDomains));
4565
4815
  if (deps.jobsSigningSecrets)
4566
4816
  runnerOptionalEnvs.push({
4567
4817
  key: 'JOBS_SIGNING_SECRETS',
@@ -4642,10 +4892,22 @@ async function provisionPlatformFiles(context, deps) {
4642
4892
  // deploy. Stopped first, so the wait below cannot pass on the old
4643
4893
  // container before the new migration has run.
4644
4894
  await stopIfRunning(deps.coolify, migrateApp.appId);
4645
- await deps.coolify.deploy(migrateApp.appId);
4895
+ const migrateDeployed = await deps.coolify.deploy(migrateApp.appId);
4646
4896
  await waitForMigration(deps.coolify, migrateApp.appId, runtime, sleep);
4897
+ // #399: files-migrate idles rather than exits once it has migrated, so it
4898
+ // reads healthy well before Coolify's own deployment does; stopping it on
4899
+ // that signal alone can find the container already gone and make Coolify
4900
+ // mark a deployment that actually migrated as `failed`.
4901
+ await waitForMigrateDeployment(deps.coolify, migrateApp.appId, queuedDeploymentUuid(migrateDeployed) ?? 'untracked', 'platform:files', 'files-migrate', runtime.now ?? Date.now, runtime.migrateTimeoutMs ?? 900_000, runtime.migrateIntervalMs ?? 10_000, sleep);
4647
4902
  await deps.coolify.raw('POST', `/applications/${migrateApp.appId}/stop`);
4648
- await deps.coolify.deploy(runnerApp.appId);
4903
+ // #252: `'deployment-only'` -- the runner's own doc comment above says why
4904
+ // ("The runner is created without `domains`... Nothing public is wired"):
4905
+ // there is no url this executor, running outside files' own Docker
4906
+ // network, could ever reach, so `coolify.deployAndWait` reaching Coolify's
4907
+ // own deployment record is the whole check. Still enough to catch the bug
4908
+ // this closes: the applied ref recorded below while the *old* container
4909
+ // (never read here) stayed healthy through a failed deploy.
4910
+ await waitForNewDeployment(deps.coolify, runnerApp.appId, { kind: 'deployment-only' }, runtime, sleep, 'platform:files', `the runner stays at ${recorded[PLATFORM_FILES_APPLIED_GIT_REF_SCOPE_KEY] ?? 'no recorded ref'}`);
4649
4911
  await record(PLATFORM_FILES_INTERNAL_URL_SCOPE_KEY, `http://${FILES_RUNNER_INTERNAL_NAME}:${FILES_PORT}`);
4650
4912
  // Last: a run that failed anywhere above is planned again.
4651
4913
  context.recordScopeId(PLATFORM_FILES_APPLIED_HOSTNAME_SCOPE_KEY, platform.files.hostname);
@@ -4722,11 +4984,21 @@ function filesDepsInvalid(deps) {
4722
4984
  * nothing ever re-checked whether its env could actually pass a health
4723
4985
  * check.
4724
4986
  */
4725
- async function filesRunnerEnvDrifted(coolify, recordedIds) {
4987
+ async function filesRunnerEnvDrifted(coolify, recordedIds, hosts = {}) {
4726
4988
  const runnerAppId = recordedIds[`coolify:app:${PLATFORM_FILES_KEY}`];
4727
4989
  if (!runnerAppId)
4728
4990
  return false;
4729
- const present = new Set((await coolify.listEnvs(runnerAppId)).filter((e) => !e.is_preview && e.value).map((e) => e.key));
4991
+ // The share set the runner was last given differs from the one now wanted.
4992
+ if (shareDomainsChanged(recordedIds, hosts.shareDomains))
4993
+ return true;
4994
+ const live = (await coolify.listEnvs(runnerAppId)).filter((e) => !e.is_preview);
4995
+ const envs = live.filter((e) => e.value);
4996
+ const present = new Set(envs.map((e) => e.key));
4997
+ // An unset or blank live value equals an empty wanted one.
4998
+ const wantedHosts = filesPublicHostsValue(hosts.publicHosts, hosts.shareDomains);
4999
+ if ((envs.find((e) => e.key === 'FILES_PUBLIC_HOSTS')?.value ?? '') !== wantedHosts) {
5000
+ return true;
5001
+ }
4730
5002
  const required = [
4731
5003
  'R2_BUCKET',
4732
5004
  'R2_ENDPOINT',
@@ -4890,11 +5162,19 @@ export const INTEGRATIONS_MANAGEMENT_KEY_ID = 'foundry-1';
4890
5162
  * 3. The migrate app first, once per `gitRef`: stop it if running, deploy it,
4891
5163
  * wait until it reports healthy, stop it, and record the ref it migrated.
4892
5164
  * Only then are the runner and worker deployed.
4893
- * 4. The management key, once: an Ed25519 pair minted here, its id recorded
4894
- * before delivery, its private half handed to
4895
- * {@link PlatformIntegrationsDeps.deliverSecret} and its public half pinned
4896
- * on the runner as `MANAGEMENT_PUBLIC_KEYS`, with `MANAGEMENT_ISSUER`. The
4897
- * guard is recorded after both.
5165
+ * 4. The management key, once, and rotated on request (#398): an Ed25519
5166
+ * pair minted here, its id recorded before delivery, its private half
5167
+ * handed to {@link PlatformIntegrationsDeps.deliverSecret} and its public
5168
+ * half pinned on the runner as `MANAGEMENT_PUBLIC_KEYS`, with
5169
+ * `MANAGEMENT_ISSUER`. The guard is recorded after both.
5170
+ * A rotation, requested by recording
5171
+ * {@link PLATFORM_INTEGRATIONS_KEY_ROTATION_REQUESTED_SCOPE_KEY}, mints a second
5172
+ * pair under a new key id the same way, pinned *beside* the first rather
5173
+ * than replacing it; the first is dropped only once
5174
+ * {@link PLATFORM_ARCHON_INTEGRATIONS_KEY_SYNCED_SCOPE_KEY} shows Archon
5175
+ * holding the new id. Archon reads the active id back from
5176
+ * {@link PLATFORM_INTEGRATIONS_KEY_ID_SCOPE_KEY}, so a rotation reaches it
5177
+ * without this step touching Archon directly.
4898
5178
  * 5. The applied `hostname` and `gitRef` are recorded last. A later change
4899
5179
  * plans the step again (`planIntegrations`), which patches `git_branch`
4900
5180
  * and the runner's `domains`, reruns the migration for a new ref, and
@@ -5067,21 +5347,28 @@ async function provisionPlatformIntegrations(action, context, deps) {
5067
5347
  // container's healthy status is never read as the new deployment's.
5068
5348
  if (migrateApp.running)
5069
5349
  await coolify.raw('POST', `/applications/${migrateApp.appId}/stop`);
5070
- await coolify.deploy(migrateApp.appId);
5350
+ const migrateDeployed = await coolify.deploy(migrateApp.appId);
5071
5351
  await waitForIntegrationsMigration(coolify, migrateApp.appId, runtime, sleep);
5352
+ // #399: integrations-migrate idles rather than exits once it has
5353
+ // migrated (the Dockerfile's own comment, below), so it reads healthy
5354
+ // well before Coolify's own deployment does; stopping it on that signal
5355
+ // alone can find the container already gone and make Coolify mark a
5356
+ // deployment that actually migrated as `failed`.
5357
+ await waitForMigrateDeployment(coolify, migrateApp.appId, queuedDeploymentUuid(migrateDeployed) ?? 'untracked', 'platform:integrations', 'integrations-migrate', runtime.now ?? Date.now, runtime.migrateTimeoutMs ?? 20 * 60_000, runtime.migratePollMs ?? 10_000, sleep);
5072
5358
  // It idles after migrating rather than exiting (the Dockerfile's own
5073
5359
  // comment), so it is stopped here; a person starts it again for
5074
5360
  // maintenance commands.
5075
5361
  await coolify.raw('POST', `/applications/${migrateApp.appId}/stop`);
5076
5362
  await record(INTEGRATIONS_MIGRATED_SCOPE_KEY, gitRef);
5077
5363
  }
5078
- // ---- the management key, once --------------------------------------------------
5364
+ // ---- the management key, once, and rotated on request (#398) -------------
5079
5365
  // The pair's id is recorded and persisted before the private half is
5080
5366
  // delivered. The guard is recorded only after delivery and the write of the
5081
5367
  // public half. A run that dies in between leaves the id without the guard:
5082
5368
  // the planner then reports the key and mints nothing, so a second pair can
5083
5369
  // never replace one the host may already hold.
5084
5370
  let keyJustSet = false;
5371
+ let rotated = false;
5085
5372
  if (!context.recordedIds()[PLATFORM_INTEGRATIONS_KEY_SET_SCOPE_KEY]) {
5086
5373
  if (context.recordedIds()[PLATFORM_INTEGRATIONS_KEY_UNDELIVERED_SCOPE_KEY]) {
5087
5374
  throw new Error(`platform:integrations: management key ${INTEGRATIONS_MANAGEMENT_KEY_ID} was minted but not confirmed delivered and pinned; not minting another. Check the held key and the runner's MANAGEMENT_PUBLIC_KEYS, then remove the recorded key ${PLATFORM_INTEGRATIONS_KEY_UNDELIVERED_SCOPE_KEY} to mint a new one`);
@@ -5091,20 +5378,7 @@ async function provisionPlatformIntegrations(action, context, deps) {
5091
5378
  privateKeyEncoding: { type: 'pkcs8', format: 'pem' },
5092
5379
  });
5093
5380
  await record(PLATFORM_INTEGRATIONS_KEY_UNDELIVERED_SCOPE_KEY, INTEGRATIONS_MANAGEMENT_KEY_ID);
5094
- const attempts = runtime.deliverAttempts ?? 3;
5095
- for (let attempt = 1;; attempt += 1) {
5096
- try {
5097
- await deps.deliverSecret(INTEGRATIONS_MANAGEMENT_KEY_REF, privateKey);
5098
- break;
5099
- }
5100
- catch {
5101
- // Retried; the error is dropped in case it echoes the key.
5102
- if (attempt >= attempts) {
5103
- throw new Error(`platform:integrations: management key ${INTEGRATIONS_MANAGEMENT_KEY_ID} was minted but could not be delivered after ${attempts} attempts`);
5104
- }
5105
- await sleep(1000 * attempt);
5106
- }
5107
- }
5381
+ await deliverManagementKey(deps, runtime, sleep, privateKey, INTEGRATIONS_MANAGEMENT_KEY_ID);
5108
5382
  await coolify.bulkUpsertEnvs(runnerApp.appId, [
5109
5383
  { key: 'MANAGEMENT_ISSUER', value: INTEGRATIONS_MANAGEMENT_ISSUER },
5110
5384
  {
@@ -5112,28 +5386,190 @@ async function provisionPlatformIntegrations(action, context, deps) {
5112
5386
  value: JSON.stringify({ [INTEGRATIONS_MANAGEMENT_KEY_ID]: publicKey }),
5113
5387
  },
5114
5388
  ]);
5389
+ await record(PLATFORM_INTEGRATIONS_KEY_ID_SCOPE_KEY, INTEGRATIONS_MANAGEMENT_KEY_ID);
5115
5390
  await record(PLATFORM_INTEGRATIONS_KEY_SET_SCOPE_KEY, runnerApp.appId);
5116
5391
  keyJustSet = true;
5117
5392
  }
5393
+ else if (context.recordedIds()[PLATFORM_INTEGRATIONS_KEY_ROTATION_REQUESTED_SCOPE_KEY] &&
5394
+ !context.recordedIds()[PLATFORM_INTEGRATIONS_KEY_ROTATION_PREVIOUS_SCOPE_KEY]) {
5395
+ // A rotation, requested by the recorded scope id
5396
+ // `PLATFORM_INTEGRATIONS_KEY_ROTATION_REQUESTED_SCOPE_KEY`.
5397
+ // Mints a new pair under the next key id and pins it *beside* the
5398
+ // current one -- never replacing it outright, so a request signed with
5399
+ // the current key during the overlap window still verifies.
5400
+ // `rotationUndelivered` is never cleared on success (the once-only
5401
+ // mint's own `UNDELIVERED` key never is either) -- a completed rotation
5402
+ // leaves it equal to the now-active id, which is what lets the *next*
5403
+ // rotation request read "no attempt in flight" without this step ever
5404
+ // clearing it. It differs from the active id only while a mint is
5405
+ // unconfirmed: minted and recorded, but not yet delivered and pinned.
5406
+ const previousKeyId = context.recordedIds()[PLATFORM_INTEGRATIONS_KEY_ID_SCOPE_KEY] ??
5407
+ INTEGRATIONS_MANAGEMENT_KEY_ID;
5408
+ const rotationUndelivered = context.recordedIds()[PLATFORM_INTEGRATIONS_KEY_ROTATION_UNDELIVERED_SCOPE_KEY];
5409
+ if (rotationUndelivered && rotationUndelivered !== previousKeyId) {
5410
+ throw new Error(`platform:integrations: rotation key ${rotationUndelivered} was minted but not confirmed delivered and pinned; not minting another. Check the held key and the runner's MANAGEMENT_PUBLIC_KEYS, then remove the recorded key ${PLATFORM_INTEGRATIONS_KEY_ROTATION_UNDELIVERED_SCOPE_KEY} and set ${PLATFORM_INTEGRATIONS_KEY_ROTATION_REQUESTED_SCOPE_KEY} again to mint a new one`);
5411
+ }
5412
+ const keyring = await currentManagementPublicKeys(coolify, runnerApp.appId);
5413
+ const nextKeyId = nextManagementKeyId(previousKeyId, keyring);
5414
+ const { publicKey, privateKey } = generateKeyPairSync('ed25519', {
5415
+ publicKeyEncoding: { type: 'spki', format: 'pem' },
5416
+ privateKeyEncoding: { type: 'pkcs8', format: 'pem' },
5417
+ });
5418
+ await record(PLATFORM_INTEGRATIONS_KEY_ROTATION_UNDELIVERED_SCOPE_KEY, nextKeyId);
5419
+ await deliverManagementKey(deps, runtime, sleep, privateKey, nextKeyId);
5420
+ keyring[nextKeyId] = publicKey;
5421
+ await coolify.bulkUpsertEnvs(runnerApp.appId, [
5422
+ { key: 'MANAGEMENT_PUBLIC_KEYS', value: JSON.stringify(keyring) },
5423
+ ]);
5424
+ // The previous id and the new active id go down in one persist: a crash
5425
+ // can never leave the new id active with the previous one unrecorded, or
5426
+ // the previous one recorded while still active.
5427
+ context.recordScopeId(PLATFORM_INTEGRATIONS_KEY_ROTATION_PREVIOUS_SCOPE_KEY, previousKeyId);
5428
+ context.recordScopeId(PLATFORM_INTEGRATIONS_KEY_ID_SCOPE_KEY, nextKeyId);
5429
+ await context.persist?.();
5430
+ // One request, one rotation: cleared only now the new key is delivered
5431
+ // and pinned. A failure before this leaves the request set and the
5432
+ // undelivered id ahead of the active id, which the planner reports as
5433
+ // blocked and the next run refuses to mint past.
5434
+ await record(PLATFORM_INTEGRATIONS_KEY_ROTATION_REQUESTED_SCOPE_KEY, '');
5435
+ rotated = true;
5436
+ keyJustSet = true;
5437
+ }
5438
+ // A request that was not honoured this run (no key minted before it, or an
5439
+ // earlier rotation still waiting to drop its previous key) is cleared, not
5440
+ // kept to fire later; the plan item says so.
5441
+ if (!rotated && context.recordedIds()[PLATFORM_INTEGRATIONS_KEY_ROTATION_REQUESTED_SCOPE_KEY]) {
5442
+ await record(PLATFORM_INTEGRATIONS_KEY_ROTATION_REQUESTED_SCOPE_KEY, '');
5443
+ }
5444
+ // ---- drop the previous key once Archon is confirmed on the new one (#398) -
5445
+ // Dropping it any earlier would refuse a request signed by an Archon
5446
+ // container that is still running the previous key while its own redeploy
5447
+ // catches up.
5448
+ const rotationPrevious = context.recordedIds()[PLATFORM_INTEGRATIONS_KEY_ROTATION_PREVIOUS_SCOPE_KEY];
5449
+ const activeKeyId = context.recordedIds()[PLATFORM_INTEGRATIONS_KEY_ID_SCOPE_KEY];
5450
+ if (rotationPrevious && rotationPrevious === activeKeyId) {
5451
+ // Corrupt record: the previous id is the active one. Never delete it.
5452
+ await record(PLATFORM_INTEGRATIONS_KEY_ROTATION_PREVIOUS_SCOPE_KEY, '');
5453
+ }
5454
+ else if (rotationPrevious &&
5455
+ // The last delivered mint (the held key) is the active id.
5456
+ context.recordedIds()[PLATFORM_INTEGRATIONS_KEY_ROTATION_UNDELIVERED_SCOPE_KEY] ===
5457
+ activeKeyId &&
5458
+ // No Archon to wait for when it is not enabled: nothing else holds the key.
5459
+ (!context.platform.archon.enabled ||
5460
+ context.recordedIds()[PLATFORM_ARCHON_INTEGRATIONS_KEY_SYNCED_SCOPE_KEY] === activeKeyId)) {
5461
+ const keyring = await currentManagementPublicKeys(coolify, runnerApp.appId);
5462
+ // The active key must be pinned before any other is removed. The planner
5463
+ // cannot see the live env, so a missing active key is an error here, not
5464
+ // a silent no-op on every apply.
5465
+ if (!activeKeyId || !keyring[activeKeyId]) {
5466
+ throw new Error(`platform:integrations: the runner's MANAGEMENT_PUBLIC_KEYS does not hold the active key ${activeKeyId ?? ''}; not removing ${rotationPrevious}. Pin the active public key on the runner, then apply again`);
5467
+ }
5468
+ delete keyring[rotationPrevious];
5469
+ await coolify.bulkUpsertEnvs(runnerApp.appId, [
5470
+ { key: 'MANAGEMENT_PUBLIC_KEYS', value: JSON.stringify(keyring) },
5471
+ ]);
5472
+ // One persist: the previous id is gone and the runner is marked as not
5473
+ // yet deployed with this keyring, so a failed deploy below is retried by
5474
+ // the next plan (`runnerKeyStale`) until the live runner stops accepting
5475
+ // the old key.
5476
+ context.recordScopeId(PLATFORM_INTEGRATIONS_KEY_ROTATION_PREVIOUS_SCOPE_KEY, '');
5477
+ context.recordScopeId(PLATFORM_INTEGRATIONS_RUNNER_KEYRING_DEPLOYED_SCOPE_KEY, '');
5478
+ await context.persist?.();
5479
+ keyJustSet = true;
5480
+ }
5481
+ const activeKeyForRunner = context.recordedIds()[PLATFORM_INTEGRATIONS_KEY_ID_SCOPE_KEY];
5482
+ const runnerKeyStale = Boolean(activeKeyForRunner) &&
5483
+ context.recordedIds()[PLATFORM_INTEGRATIONS_RUNNER_KEYRING_DEPLOYED_SCOPE_KEY] !==
5484
+ activeKeyForRunner;
5118
5485
  // ---- then the runner and the worker ------------------------------------------------
5119
5486
  // Decided from the live state: an env change only reaches a container on its
5120
5487
  // next deploy, and a stopped runner is deployed so an earlier failed deploy
5121
5488
  // is retried. The worker exits after one pass by design, so "not running"
5122
5489
  // says nothing about it and it is deployed only when created or changed.
5490
+ // #252: `'deployment-only'` -- like `coolify:jobs`, the runner is reached
5491
+ // through its Coolify `domains` binding, not a tunnel/DNS record this
5492
+ // executor set up itself, and no Cloudflare wiring is created for
5493
+ // `hostname` (this function's own doc comment, "Known gaps"). With no
5494
+ // reachability confirmed from here, `coolify.deployAndWait` reaching
5495
+ // Coolify's own deployment record is the whole check -- still enough to
5496
+ // catch the bug this closes: the applied ref recorded below while the
5497
+ // *old* container (never read here) stayed healthy through a failed
5498
+ // deploy.
5123
5499
  if (runnerApp.justCreated ||
5124
5500
  !runnerApp.running ||
5125
5501
  runnerEnvsChanged ||
5126
5502
  keyJustSet ||
5503
+ runnerKeyStale ||
5127
5504
  configChanged) {
5128
- await coolify.deploy(runnerApp.appId);
5505
+ await waitForNewDeployment(coolify, runnerApp.appId, { kind: 'deployment-only' }, runtime, sleep, 'platform:integrations', `the runner stays at ${recorded[PLATFORM_INTEGRATIONS_APPLIED_GITREF_SCOPE_KEY] ?? 'no recorded ref'}`);
5506
+ }
5507
+ // #398: the runner is up with the keyring that holds the active id; only
5508
+ // now may Archon move to it.
5509
+ if (runnerKeyStale && activeKeyForRunner) {
5510
+ await record(PLATFORM_INTEGRATIONS_RUNNER_KEYRING_DEPLOYED_SCOPE_KEY, activeKeyForRunner);
5129
5511
  }
5130
5512
  if (workerApp.justCreated || workerEnvsChanged || configChanged) {
5131
- await coolify.deploy(workerApp.appId);
5513
+ await waitForNewDeployment(coolify, workerApp.appId, { kind: 'deployment-only' }, runtime, sleep, 'platform:integrations', `the worker stays at ${recorded[PLATFORM_INTEGRATIONS_APPLIED_GITREF_SCOPE_KEY] ?? 'no recorded ref'}`);
5132
5514
  }
5133
5515
  await record(PLATFORM_INTEGRATIONS_APPLIED_HOSTNAME_SCOPE_KEY, hostname);
5134
5516
  await record(PLATFORM_INTEGRATIONS_APPLIED_GITREF_SCOPE_KEY, gitRef);
5135
5517
  return { resourceId: runnerApp.appId };
5136
5518
  }
5519
+ /**
5520
+ * #398: `deliverSecret` with the once-only mint's own retry discipline,
5521
+ * shared with the rotation path so both mint the same way. The error never
5522
+ * carries the key, only its id.
5523
+ */
5524
+ async function deliverManagementKey(deps, runtime, sleep, privateKey, keyId) {
5525
+ const attempts = runtime.deliverAttempts ?? 3;
5526
+ for (let attempt = 1;; attempt += 1) {
5527
+ try {
5528
+ await deps.deliverSecret(INTEGRATIONS_MANAGEMENT_KEY_REF, privateKey);
5529
+ return;
5530
+ }
5531
+ catch {
5532
+ // Retried; the error is dropped in case it echoes the key.
5533
+ if (attempt >= attempts) {
5534
+ throw new Error(`platform:integrations: management key ${keyId} was minted but could not be delivered after ${attempts} attempts`);
5535
+ }
5536
+ await sleep(1000 * attempt);
5537
+ }
5538
+ }
5539
+ }
5540
+ /**
5541
+ * #398: the runner's live `MANAGEMENT_PUBLIC_KEYS`, parsed -- `{}` when the
5542
+ * env is absent (no key minted yet). The rotation path reads this rather
5543
+ * than keeping its own copy, so it edits whatever the runner actually holds,
5544
+ * including a key id a person pinned by hand.
5545
+ */
5546
+ async function currentManagementPublicKeys(coolify, runnerAppId) {
5547
+ const raw = (await coolify.listEnvs(runnerAppId)).find((e) => e.key === 'MANAGEMENT_PUBLIC_KEYS' && !e.is_preview)?.value;
5548
+ if (!raw)
5549
+ return {};
5550
+ try {
5551
+ return JSON.parse(raw);
5552
+ }
5553
+ catch {
5554
+ throw new Error("platform:integrations: the runner's MANAGEMENT_PUBLIC_KEYS is not valid JSON; not touching the keyring");
5555
+ }
5556
+ }
5557
+ /**
5558
+ * #398: the key id a rotation mints next -- `foundry-1` to `foundry-2`, and
5559
+ * so on, by incrementing the trailing number. A current id with no trailing
5560
+ * number (hand-pinned, never by this step) gets `-2` appended rather than
5561
+ * refusing: the host's keyring only requires `^[A-Za-z0-9_-]{1,100}$`, and
5562
+ * this never collides with a number the current id does not already carry.
5563
+ */
5564
+ function nextManagementKeyId(currentKeyId, keyring) {
5565
+ const match = /^(.*-)(\d+)$/.exec(currentKeyId);
5566
+ const prefix = match ? match[1] : `${currentKeyId}-`;
5567
+ let n = match ? Number(match[2]) + 1 : 2;
5568
+ // Never overwrite an id already pinned on the runner.
5569
+ while (keyring[`${prefix}${n}`])
5570
+ n += 1;
5571
+ return `${prefix}${n}`;
5572
+ }
5137
5573
  /**
5138
5574
  * The runtime `DATABASE_URL` already on the migrate app, or a new one:
5139
5575
  * {@link INTEGRATIONS_RUNTIME_ROLE} with a password minted once from a
@@ -5773,3 +6209,50 @@ const ARCHON_SERVICE_INPUTS = {
5773
6209
  refs: [INTEGRATIONS_MANAGEMENT_KEY_REF],
5774
6210
  },
5775
6211
  };
6212
+ /** Scope id (no `:`): the share domains the runner's Coolify `domains` was last set to, comma-joined. */
6213
+ export const PLATFORM_FILES_SHARE_DOMAINS_SCOPE_KEY = 'platform_files_share_domains';
6214
+ /** Lowercased, trimmed, deduplicated and sorted, so the same set always gives the same value. */
6215
+ function normalHosts(hosts) {
6216
+ return [...new Set((hosts ?? []).map((h) => h.trim().toLowerCase()).filter(Boolean))].sort();
6217
+ }
6218
+ /** The runner's Coolify `domains`, written only when the share set changed (a runner nobody gave a share host keeps none). */
6219
+ function filesShareDomainsPatch(recorded, shareDomains) {
6220
+ return shareDomainsChanged(recorded, shareDomains)
6221
+ ? { domains: coolifyDomains(normalHosts(shareDomains)) }
6222
+ : {};
6223
+ }
6224
+ /**
6225
+ * The runner's `FILES_PUBLIC_HOSTS` env. Always written when an earlier apply
6226
+ * recorded a share set: files reads an empty value as no public host
6227
+ * (`filesPublicHosts()`), so an emptied set blanks the old value (the port has
6228
+ * no env delete). A runner never given a host keeps no such env.
6229
+ */
6230
+ function filesPublicHostsEnvs(recorded, publicHosts, shareDomains) {
6231
+ const value = filesPublicHostsValue(publicHosts, shareDomains);
6232
+ return value || recorded[PLATFORM_FILES_SHARE_DOMAINS_SCOPE_KEY]
6233
+ ? [{ key: 'FILES_PUBLIC_HOSTS', value }]
6234
+ : [];
6235
+ }
6236
+ function shareDomainsChanged(recorded, shareDomains) {
6237
+ return ((recorded[PLATFORM_FILES_SHARE_DOMAINS_SCOPE_KEY] ?? '') !== normalHosts(shareDomains).join(','));
6238
+ }
6239
+ /**
6240
+ * The runner's `FILES_PUBLIC_HOSTS`: the host-supplied `publicHosts` first, in
6241
+ * their given order, then the share domains sorted, with no host twice.
6242
+ * Empty when there is neither.
6243
+ */
6244
+ export function filesPublicHostsValue(publicHosts, shareDomains) {
6245
+ const given = (publicHosts ?? '')
6246
+ .split(',')
6247
+ .map((h) => h.trim())
6248
+ .filter(Boolean);
6249
+ return [...new Set([...given, ...normalHosts(shareDomains)])].join(',');
6250
+ }
6251
+ /**
6252
+ * The share domain of every org that has a domain and an entry declaring
6253
+ * `uses.files.shareHost`: `share.<domain>`. The platform files step derives it
6254
+ * from the declared orgs, for its apply and its drift check alike.
6255
+ */
6256
+ export function filesShareDomains(orgs) {
6257
+ return normalHosts(orgs.map((org) => filesShareHostname(org) ?? ''));
6258
+ }