@wtfalch/foundry 0.61.0 → 0.63.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/dist/bootstrap/declare.js +4 -1
  2. package/dist/bootstrap/versions.d.ts +2 -0
  3. package/dist/index.d.ts +9 -1
  4. package/dist/index.js +3 -0
  5. package/dist/migrations/0065_native_build_runs.sql +32 -0
  6. package/dist/migrations/0066_org_kosmos.sql +10 -0
  7. package/dist/migrations/0067_requested_changes_kosmos.sql +6 -0
  8. package/dist/migrations/0068_platform_registry.sql +17 -0
  9. package/dist/modules/sign-in.js +1 -1
  10. package/dist/native-build.d.ts +50 -0
  11. package/dist/native-build.js +72 -0
  12. package/dist/org.d.ts +22 -3
  13. package/dist/org.js +25 -2
  14. package/dist/plan/agora.d.ts +1 -1
  15. package/dist/plan/agora.js +4 -0
  16. package/dist/plan/archon-blue-green.d.ts +5 -0
  17. package/dist/plan/archon-blue-green.js +2 -0
  18. package/dist/plan/blue-green-apply.d.ts +5 -0
  19. package/dist/plan/blue-green-apply.js +197 -22
  20. package/dist/plan/blue-green.d.ts +80 -9
  21. package/dist/plan/blue-green.js +115 -14
  22. package/dist/plan/boule.d.ts +1 -1
  23. package/dist/plan/boule.js +2 -0
  24. package/dist/plan/broker-entries.d.ts +9 -0
  25. package/dist/plan/broker-entries.js +29 -0
  26. package/dist/plan/deploy-health.js +9 -2
  27. package/dist/plan/desktop-callback.js +4 -1
  28. package/dist/plan/diff.d.ts +5 -0
  29. package/dist/plan/diff.js +184 -63
  30. package/dist/plan/edge-access.d.ts +2 -3
  31. package/dist/plan/edge-access.js +8 -0
  32. package/dist/plan/executor.d.ts +111 -19
  33. package/dist/plan/executor.js +578 -72
  34. package/dist/plan/index.d.ts +2 -0
  35. package/dist/plan/index.js +1 -0
  36. package/dist/plan/jobs-scheduler-http.d.ts +2 -0
  37. package/dist/plan/jobs-scheduler-http.js +11 -0
  38. package/dist/plan/kosmos.d.ts +56 -0
  39. package/dist/plan/kosmos.js +84 -0
  40. package/dist/plan/locked-hostnames.js +5 -2
  41. package/dist/plan/org-names.d.ts +3 -3
  42. package/dist/plan/org-names.js +3 -1
  43. package/dist/plan/org-postgres-migrate.js +1 -0
  44. package/dist/plan/own-limits.d.ts +12 -0
  45. package/dist/plan/own-limits.js +2 -0
  46. package/dist/plan/plan-apply.d.ts +51 -1
  47. package/dist/plan/plan-apply.js +298 -25
  48. package/dist/plan/platform-diff.d.ts +6 -0
  49. package/dist/plan/platform-diff.js +74 -0
  50. package/dist/plan/platform-org.d.ts +2 -0
  51. package/dist/plan/platform-providers.d.ts +46 -1
  52. package/dist/plan/platform-providers.js +162 -1
  53. package/dist/plan/platform-registry-keys.d.ts +58 -0
  54. package/dist/plan/platform-registry-keys.js +69 -0
  55. package/dist/plan/platform-registry.d.ts +67 -0
  56. package/dist/plan/platform-registry.js +298 -0
  57. package/dist/plan/post-logout-uris.d.ts +1 -1
  58. package/dist/plan/post-logout-uris.js +5 -2
  59. package/dist/plan/providers-readers.js +20 -1
  60. package/dist/plan/release-host.d.ts +31 -2
  61. package/dist/plan/release-host.js +114 -6
  62. package/dist/plan/stage-copy-host.d.ts +104 -0
  63. package/dist/plan/stage-copy-host.js +259 -0
  64. package/dist/plan/stage-copy.js +44 -0
  65. package/dist/plan/stage-env.d.ts +9 -0
  66. package/dist/plan/stage-env.js +38 -0
  67. package/dist/plan/stage-guards.d.ts +57 -1
  68. package/dist/plan/stage-guards.js +106 -0
  69. package/dist/plan/stage-isolation.js +56 -0
  70. package/dist/plan/stage-services.d.ts +93 -0
  71. package/dist/plan/stage-services.js +153 -0
  72. package/dist/plan/stage-signin.d.ts +152 -0
  73. package/dist/plan/stage-signin.js +133 -0
  74. package/dist/plan/stage-sink.d.ts +2 -0
  75. package/dist/plan/stage-sink.js +6 -1
  76. package/dist/platform.d.ts +7 -1
  77. package/dist/platform.js +1 -0
  78. package/dist/release/decide.d.ts +12 -0
  79. package/dist/release/decide.js +37 -2
  80. package/dist/release/migration-compatibility.d.ts +66 -0
  81. package/dist/release/migration-compatibility.js +340 -0
  82. package/dist/release/run.d.ts +71 -1
  83. package/dist/release/run.js +92 -8
  84. package/dist/release/state.d.ts +14 -1
  85. package/dist/release/state.js +43 -0
  86. package/dist/release/types.d.ts +20 -0
  87. package/dist/release/types.js +2 -0
  88. package/dist/requested-changes.d.ts +11 -2
  89. package/dist/requested-changes.js +13 -1
  90. package/dist/slug.d.ts +1 -1
  91. package/dist/slug.js +11 -2
  92. package/dist/store/postgres.js +21 -0
  93. package/dist/store/schema.d.ts +87 -0
  94. package/dist/store/schema.js +10 -0
  95. package/package.json +3 -2
  96. package/versions.digests.json +1 -0
  97. package/versions.json +1 -0
@@ -4,6 +4,7 @@ import { VERSIONS, floatingTagBlocked } from '../bootstrap/versions.js';
4
4
  import { sfuSlotPorts } from '../calls-sfu.js';
5
5
  import { ORG_SERVICE_SCHEMAS, heldEnvRef, ownsServices, validateHosting } from '../org.js';
6
6
  import { PLATFORM_SERVICE_KEYS } from '../platform.js';
7
+ import { recordedFrom, releaseKeys } from '../release/state.js';
7
8
  import { AlreadyRunningError, StepFailedError } from '../runner/errors.js';
8
9
  import { StaleLeaseError, } from '../store.js';
9
10
  import { entryHasDatabase } from '../worker-app-config.js';
@@ -36,8 +37,9 @@ import { DEFAULT_ORG_PLATFORM_READINESS, ORG_PLATFORM_KEY_REFS, agoraHostValuesF
36
37
  import { PLATFORM_AUTH_BROKER_MAIL_KEY_REF, PLATFORM_BACKUP_S3_STORAGE_SCOPE_KEY, } from './platform-providers.js';
37
38
  import { privatePageGateKey } from './private-pages.js';
38
39
  import { CLOUDFLARE_TUNNEL_APPS_HOSTNAMES_SCOPE_KEY, CLOUDFLARE_TUNNEL_ENTRY_ROUTES_SCOPE_KEY, CLOUDFLARE_TUNNEL_OIKOS_HOSTNAMES_SCOPE_KEY, } from './proxy-labels.js';
39
- import { runSubjectRelease } from './release-host.js';
40
+ import { finishStageCopy, runSubjectRelease, } from './release-host.js';
40
41
  import { remoteDatabaseFactsFrom, remoteDatabaseKeys } from './remote-database.js';
42
+ import { STAGE_COPY_LOCK_MS, claimStageCopyLock, parseStageCopyLock, stageCopyLockValue, } from './stage-copy-host.js';
41
43
  import { liveWorkerApps, workerAppTraceScopeKeys } from './worker-app.js';
42
44
  import { liveWorkers, workerTraceScopeKeys } from './worker.js';
43
45
  /** #441: the lease scope that serializes every mutation of one company, and the one for the shared platform. */
@@ -45,6 +47,8 @@ const companyScope = (orgId) => `company:${orgId}`;
45
47
  const PLATFORM_SCOPE = 'platform';
46
48
  /** A lease nobody renews: long enough for one apply, short enough that a dead worker's scope frees itself. The fence stops a slow worker that outlives it. */
47
49
  const DEFAULT_LEASE_TTL_MS = 30 * 60 * 1000;
50
+ /** Whether any of the subjects has a release stepping. */
51
+ const releaseRunning = (subjects, ids) => subjects.some((s) => recordedFrom(ids, s.id)?.release != null);
48
52
  /** #441: another apply, destroy, adopt or release holds the lease for this company or the platform. Nothing ran; try again when it finishes. */
49
53
  export class MutationInProgressError extends Error {
50
54
  scope;
@@ -195,6 +199,7 @@ export class FoundryPlanApply {
195
199
  onlySteps;
196
200
  now;
197
201
  leaseTtlMs;
202
+ copyLeaseRetry;
198
203
  appliedVersions;
199
204
  backupGate;
200
205
  platformDeps;
@@ -238,6 +243,7 @@ export class FoundryPlanApply {
238
243
  this.onlySteps = options.onlySteps;
239
244
  this.now = options.now ?? (() => new Date());
240
245
  this.leaseTtlMs = options.leaseTtlMs ?? DEFAULT_LEASE_TTL_MS;
246
+ this.copyLeaseRetry = options.copyLeaseRetry ?? { tries: 10, waitMs: 30_000 };
241
247
  this.appliedVersions = options.appliedVersions;
242
248
  this.backupGate = options.backupGate;
243
249
  this.platformDeps = options.platform;
@@ -1163,22 +1169,28 @@ export class FoundryPlanApply {
1163
1169
  * (the live ids of the run that holds the lease). A weight the runner changed reaches every server's balancer
1164
1170
  * before this returns.
1165
1171
  */
1166
- async runOrgReleases(org, context, fence) {
1172
+ async runOrgReleases(org, context, fence, copies = 'defer') {
1173
+ const out = { deferred: false, claimed: [] };
1167
1174
  const store = this.releaseStore;
1168
1175
  const primaryUuid = context.recordedIds()[PRIMARY_SERVER_SCOPE_KEY];
1169
1176
  if (!store || !this.executor.releaseHost || !primaryUuid)
1170
- return;
1171
- for (const subject of blueGreenSubjects(org, context.recordedIds())) {
1177
+ return out;
1178
+ const subjects = blueGreenSubjects(org, context.recordedIds());
1179
+ // A copy waits while any subject of the org has a release running: the runner must stay free to roll back.
1180
+ if (copies === 'claim' && releaseRunning(subjects, context.recordedIds()))
1181
+ return out;
1182
+ for (const subject of subjects) {
1172
1183
  // A built-in app's names reach its balancer only once the tunnel was last written with it: until then a
1173
1184
  // pinned check would answer from blue whatever color it names.
1174
1185
  if (subject.builtin &&
1175
1186
  !(context.recordedIds()[CLOUDFLARE_TUNNEL_ENTRY_ROUTES_SCOPE_KEY] ?? '').includes(`"service":"${lbOrigin(subject)}"`)) {
1176
1187
  continue;
1177
1188
  }
1189
+ await this.executor.syncStageGateMembers?.(org, subject, context);
1178
1190
  const host = this.executor.releaseHost(org, subject, context, () => this.renewLease(fence));
1179
1191
  if (!host)
1180
1192
  continue;
1181
- await runSubjectRelease({
1193
+ const result = await runSubjectRelease({
1182
1194
  store,
1183
1195
  scopeOwnerId: org.id,
1184
1196
  subject,
@@ -1186,21 +1198,40 @@ export class FoundryPlanApply {
1186
1198
  context,
1187
1199
  now: this.now(),
1188
1200
  primaryUuid,
1201
+ ...(copies === 'defer' ? { deferCopies: true } : {}),
1189
1202
  });
1203
+ out.deferred ||= result.deferredCopy;
1204
+ if (copies === 'claim' && result.copy) {
1205
+ // The copy's own lock is taken here, under the lease; the copy then runs without it.
1206
+ const runId = randomUUID();
1207
+ const claim = await claimStageCopyLock({
1208
+ subjectId: subject.id,
1209
+ context,
1210
+ requestId: result.copy.requestId,
1211
+ runId,
1212
+ });
1213
+ if (claim === 'claimed')
1214
+ out.claimed.push({ subject, job: { ...result.copy, runId } });
1215
+ }
1190
1216
  }
1217
+ return out;
1191
1218
  }
1192
1219
  /** One release run for platform-archon, on the platform's live ids; see {@link runOrgReleases}. */
1193
- async runPlatformReleases(platform, context, fence) {
1220
+ async runPlatformReleases(platform, context, fence, copies = 'defer') {
1221
+ const out = { deferred: false, claimed: [] };
1194
1222
  const store = this.releaseStore;
1195
1223
  const executor = this.platformDeps?.executor;
1196
1224
  const subject = archonSubject(platform, ARCHON_REPO);
1197
1225
  const primaryUuid = context.recordedIds()[PRIMARY_SERVER_SCOPE_KEY];
1198
1226
  if (!store || !executor?.releaseHost || !subject || !primaryUuid)
1199
- return;
1227
+ return out;
1228
+ if (copies === 'claim' && releaseRunning([subject], context.recordedIds()))
1229
+ return out;
1230
+ await executor.syncStageGateMembers?.(platform, subject, context);
1200
1231
  const host = executor.releaseHost(platform, subject, context, () => this.renewLease(fence));
1201
1232
  if (!host)
1202
- return;
1203
- await runSubjectRelease({
1233
+ return out;
1234
+ const result = await runSubjectRelease({
1204
1235
  store,
1205
1236
  scopeOwnerId: 'platform',
1206
1237
  subject,
@@ -1208,21 +1239,35 @@ export class FoundryPlanApply {
1208
1239
  context,
1209
1240
  now: this.now(),
1210
1241
  primaryUuid,
1242
+ ...(copies === 'defer' ? { deferCopies: true } : {}),
1211
1243
  });
1244
+ out.deferred = result.deferredCopy;
1245
+ if (copies === 'claim' && result.copy) {
1246
+ const runId = randomUUID();
1247
+ const claim = await claimStageCopyLock({
1248
+ subjectId: subject.id,
1249
+ context,
1250
+ requestId: result.copy.requestId,
1251
+ runId,
1252
+ });
1253
+ if (claim === 'claimed')
1254
+ out.claimed.push({ subject, job: { ...result.copy, runId } });
1255
+ }
1256
+ return out;
1212
1257
  }
1213
- async runPlatformReleaseTick() {
1258
+ async runPlatformReleaseTick(copies) {
1214
1259
  const deps = this.platformDeps;
1215
1260
  const platform = await deps?.store.get();
1216
1261
  if (!deps || !platform)
1217
- return;
1262
+ return { deferred: false, claimed: [] };
1218
1263
  const empty = { orgId: platform.ownerOrgId, steps: [], unmanaged: [] };
1219
1264
  if ((await this.authz.canApplyPlatform?.(this.actor)) !== true) {
1220
1265
  await this.refusePlatform(empty, platform.ownerOrgId, 'not authorized to run the release tick');
1221
1266
  throw new NotAuthorizedError(this.actor, platform.ownerOrgId);
1222
1267
  }
1223
- await this.withMutationLease(PLATFORM_SCOPE, (reason) => this.refusePlatform(empty, platform.ownerOrgId, reason), async (fence) => {
1268
+ return this.withMutationLease(PLATFORM_SCOPE, (reason) => this.refusePlatform(empty, platform.ownerOrgId, reason), async (fence) => {
1224
1269
  const serviceIds = { ...(await deps.store.getServiceIds()) };
1225
- await this.runPlatformReleases(platform, {
1270
+ return this.runPlatformReleases(platform, {
1226
1271
  recordedIds: () => ({ ...serviceIds }),
1227
1272
  recordScopeId: (key, value) => {
1228
1273
  serviceIds[key] = value;
@@ -1231,7 +1276,7 @@ export class FoundryPlanApply {
1231
1276
  delete serviceIds[key];
1232
1277
  },
1233
1278
  persist: () => deps.store.putServiceIds({ ...serviceIds }, fence),
1234
- }, fence);
1279
+ }, fence, copies);
1235
1280
  });
1236
1281
  }
1237
1282
  /**
@@ -1243,29 +1288,216 @@ export class FoundryPlanApply {
1243
1288
  */
1244
1289
  async runReleaseTick() {
1245
1290
  const results = [];
1291
+ const settle = (target, error) => {
1292
+ const entry = {
1293
+ target,
1294
+ ...(error === undefined
1295
+ ? {}
1296
+ : { error: error instanceof Error ? error.message : String(error) }),
1297
+ };
1298
+ const at = results.findIndex((r) => r.target === target);
1299
+ if (at === -1)
1300
+ results.push(entry);
1301
+ else
1302
+ results[at] = entry;
1303
+ };
1304
+ // Phase 1: every org's and the platform's release, with stage copy requests left open. A copy takes up to
1305
+ // ~45 minutes, so no copy runs before every other release of this tick has run.
1306
+ const copying = [];
1246
1307
  for (const org of await this.store.list()) {
1247
1308
  if (!declaresBlueGreen(org))
1248
1309
  continue;
1249
1310
  try {
1250
- await this.runOrgReleaseTick(org.id);
1251
- results.push({ target: org.id });
1311
+ if ((await this.runOrgReleaseTick(org.id, 'defer')).deferred) {
1312
+ copying.push({ target: org.id, run: () => this.runOrgStageCopies(org.id) });
1313
+ }
1314
+ settle(org.id, undefined);
1252
1315
  }
1253
1316
  catch (e) {
1254
- results.push({ target: org.id, error: e instanceof Error ? e.message : String(e) });
1317
+ settle(org.id, e);
1255
1318
  }
1256
1319
  }
1257
1320
  const platform = await this.platformDeps?.store.get();
1258
1321
  if (platform?.archon.enabled && platform.archon.blueGreen) {
1259
1322
  try {
1260
- await this.runPlatformReleaseTick();
1261
- results.push({ target: 'platform' });
1323
+ if ((await this.runPlatformReleaseTick('defer')).deferred) {
1324
+ copying.push({ target: 'platform', run: () => this.runPlatformStageCopies() });
1325
+ }
1326
+ settle('platform', undefined);
1327
+ }
1328
+ catch (e) {
1329
+ settle('platform', e);
1330
+ }
1331
+ }
1332
+ // Phase 2: the copies, one subject at a time. A copy takes its lock under the org's lease and runs without it.
1333
+ for (const { target, run } of copying) {
1334
+ try {
1335
+ await run();
1336
+ settle(target, undefined);
1262
1337
  }
1263
1338
  catch (e) {
1264
- results.push({ target: 'platform', error: e instanceof Error ? e.message : String(e) });
1339
+ settle(target, e);
1265
1340
  }
1266
1341
  }
1267
1342
  return results;
1268
1343
  }
1344
+ /**
1345
+ * Phase 2 for one org: takes the copy lock of each stage copy request under the org's lease, then runs the
1346
+ * copies after the lease is released. While a copy runs, every other tick and apply for the org goes on, a
1347
+ * hard-fail rollback included; the copy records through short leases ({@link copyContext}).
1348
+ */
1349
+ async runOrgStageCopies(id) {
1350
+ const claimed = (await this.runOrgReleaseTick(id, 'claim')).claimed;
1351
+ if (claimed.length === 0)
1352
+ return;
1353
+ const org = await this.store.get(id);
1354
+ if (!org)
1355
+ return;
1356
+ const target = {
1357
+ scope: companyScope(id),
1358
+ refuse: (reason) => this.refuse({ orgId: id, steps: [], unmanaged: [] }, reason),
1359
+ read: async () => {
1360
+ const state = await this.store.getAppliedState(id);
1361
+ if (!state)
1362
+ return null;
1363
+ return {
1364
+ ids: { ...state.serviceIds },
1365
+ write: (ids, fence) => this.store.putAppliedState({ ...state, serviceIds: { ...ids } }, fence),
1366
+ };
1367
+ },
1368
+ host: (subject, context, renew) => this.executor.releaseHost?.(org, subject, context, renew),
1369
+ };
1370
+ for (const copy of claimed)
1371
+ await this.runClaimedCopy(target, copy);
1372
+ }
1373
+ /** Phase 2 for platform-archon; see {@link runOrgStageCopies}. */
1374
+ async runPlatformStageCopies() {
1375
+ const claimed = (await this.runPlatformReleaseTick('claim')).claimed;
1376
+ if (claimed.length === 0)
1377
+ return;
1378
+ const deps = this.platformDeps;
1379
+ const platform = await deps?.store.get();
1380
+ if (!deps || !platform)
1381
+ return;
1382
+ const target = {
1383
+ scope: PLATFORM_SCOPE,
1384
+ refuse: (reason) => this.refusePlatform({ orgId: platform.ownerOrgId, steps: [], unmanaged: [] }, platform.ownerOrgId, reason),
1385
+ read: async () => ({
1386
+ ids: { ...(await deps.store.getServiceIds()) },
1387
+ write: (ids, fence) => deps.store.putServiceIds({ ...ids }, fence),
1388
+ }),
1389
+ host: (subject, context, renew) => deps.executor.releaseHost?.(platform, subject, context, renew),
1390
+ };
1391
+ for (const copy of claimed)
1392
+ await this.runClaimedCopy(target, copy);
1393
+ }
1394
+ /**
1395
+ * Runs one claimed copy with no lease held, then records its outcome and consumes the request. A lost lock
1396
+ * throws (`StaleLeaseError`) and leaves the request open.
1397
+ */
1398
+ async runClaimedCopy(target, claimed) {
1399
+ const store = this.releaseStore;
1400
+ const { subject, job } = claimed;
1401
+ const first = await target.read();
1402
+ if (!store || !first)
1403
+ return;
1404
+ const lockKey = releaseKeys(subject.id).stageCopyLock;
1405
+ const context = this.copyContext(target, first.ids, lockKey, job.requestId, job.runId ?? '');
1406
+ await finishStageCopy({
1407
+ store,
1408
+ host: target.host(subject, context, () => context.extendLock()),
1409
+ subject,
1410
+ context,
1411
+ now: this.now(),
1412
+ job,
1413
+ });
1414
+ }
1415
+ /**
1416
+ * The recorded ids a copy reads and writes, without the org's lease. It reads a snapshot taken when the lock was
1417
+ * claimed; each `persist` takes the lease for a moment, re-reads the stored ids and writes only the keys the
1418
+ * copy changed, so a tick that ran in between loses nothing. The lease may be busy: it waits and tries again.
1419
+ */
1420
+ copyContext(target, initial, lockKey, requestId, runId) {
1421
+ const ids = { ...initial };
1422
+ // When the lock this run holds lapses; a beat that cannot renew it by then fails the run as stale.
1423
+ let released = false;
1424
+ let lockExpiry = parseStageCopyLock(initial[lockKey]).expiry;
1425
+ const dirty = new Map();
1426
+ let queue = Promise.resolve();
1427
+ const serial = (fn) => {
1428
+ const run = queue.then(fn, fn);
1429
+ queue = run.catch(() => undefined);
1430
+ return run;
1431
+ };
1432
+ const write = async (changes, tries, mustHoldLock) => {
1433
+ for (let attempt = 1;; attempt++) {
1434
+ const lease = await this.store.acquireMutationLease(target.scope, randomUUID(), this.leaseTtlMs);
1435
+ if ('token' in lease) {
1436
+ try {
1437
+ const stored = await target.read();
1438
+ if (!stored)
1439
+ throw new Error('stage copy: the recorded state is gone');
1440
+ // Another run's lock means this run lapsed and was replaced: write nothing.
1441
+ // No lock is fine only after this run removed it itself (its result, or its failure path).
1442
+ const heldBy = stored.ids[lockKey];
1443
+ if (heldBy ? parseStageCopyLock(heldBy).runId !== runId : mustHoldLock || !released) {
1444
+ throw new StaleLeaseError(target.scope);
1445
+ }
1446
+ for (const [key, value] of changes) {
1447
+ if (value === null)
1448
+ delete stored.ids[key];
1449
+ else
1450
+ stored.ids[key] = value;
1451
+ }
1452
+ await stored.write(stored.ids, lease);
1453
+ if (changes.get(lockKey) === null)
1454
+ released = true;
1455
+ return true;
1456
+ }
1457
+ finally {
1458
+ await this.store.releaseMutationLease(lease).catch(() => undefined);
1459
+ }
1460
+ }
1461
+ if (attempt >= tries) {
1462
+ if (tries === 1)
1463
+ return false;
1464
+ throw new MutationInProgressError(target.scope, lease.heldUntil);
1465
+ }
1466
+ await new Promise((resolve) => setTimeout(resolve, this.copyLeaseRetry.waitMs));
1467
+ }
1468
+ };
1469
+ return {
1470
+ recordedIds: () => ({ ...ids }),
1471
+ recordScopeId: (key, value) => {
1472
+ ids[key] = value;
1473
+ dirty.set(key, value);
1474
+ },
1475
+ forgetId: (key) => {
1476
+ delete ids[key];
1477
+ dirty.set(key, null);
1478
+ },
1479
+ persist: () => serial(async () => {
1480
+ const batch = new Map(dirty);
1481
+ await write(batch, this.copyLeaseRetry.tries, false);
1482
+ for (const [key, value] of batch)
1483
+ if (dirty.get(key) === value)
1484
+ dirty.delete(key);
1485
+ }),
1486
+ // Keeps the copy's lock alive. A lock another run took over throws. A busy lease is skipped while the lock
1487
+ // still stands; once it would lapse, a beat that cannot renew it fails the run as stale.
1488
+ extendLock: () => serial(async () => {
1489
+ const expiry = Date.now() + STAGE_COPY_LOCK_MS;
1490
+ const value = stageCopyLockValue(expiry, requestId, runId);
1491
+ if (await write(new Map([[lockKey, value]]), 1, true)) {
1492
+ ids[lockKey] = value;
1493
+ lockExpiry = expiry;
1494
+ }
1495
+ else if (Date.now() >= lockExpiry) {
1496
+ throw new StaleLeaseError(target.scope);
1497
+ }
1498
+ }),
1499
+ };
1500
+ }
1269
1501
  /**
1270
1502
  * The recurring audit sealing run (foundry#929): seals every ledger registered by an `audit:ledger` step with the
1271
1503
  * platform signer and has the timestamp authority stamp the new checkpoints. Foundry has no scheduler, so the host
@@ -1297,19 +1529,23 @@ export class FoundryPlanApply {
1297
1529
  allowed: (org) => this.authz.canApply(this.actor, org.id),
1298
1530
  });
1299
1531
  }
1300
- async runOrgReleaseTick(id) {
1532
+ /**
1533
+ * One org's release run. `defer`: stage copy requests stay open and `deferred` says whether there were any.
1534
+ * `claim`: the copies this run decided on, each with its lock taken, for {@link runClaimedCopy}.
1535
+ */
1536
+ async runOrgReleaseTick(id, copies) {
1301
1537
  const empty = { orgId: id, steps: [], unmanaged: [] };
1302
1538
  if (!(await this.authz.canApply(this.actor, id))) {
1303
1539
  await this.refuse(empty, 'not authorized to run the release tick');
1304
1540
  throw new NotAuthorizedError(this.actor, id);
1305
1541
  }
1306
- await this.withMutationLease(companyScope(id), (reason) => this.refuse(empty, reason), async (fence) => {
1542
+ return this.withMutationLease(companyScope(id), (reason) => this.refuse(empty, reason), async (fence) => {
1307
1543
  const org = await this.store.get(id);
1308
1544
  const previous = await this.store.getAppliedState(id);
1309
1545
  if (!org || !previous)
1310
- return;
1546
+ return { deferred: false, claimed: [] };
1311
1547
  const serviceIds = { ...previous.serviceIds };
1312
- await this.runOrgReleases(org, {
1548
+ return this.runOrgReleases(org, {
1313
1549
  recordedIds: () => ({ ...serviceIds }),
1314
1550
  recordScopeId: (key, value) => {
1315
1551
  serviceIds[key] = value;
@@ -1318,7 +1554,7 @@ export class FoundryPlanApply {
1318
1554
  delete serviceIds[key];
1319
1555
  },
1320
1556
  persist: () => this.store.putAppliedState({ ...previous, serviceIds: { ...serviceIds } }, fence),
1321
- }, fence);
1557
+ }, fence, copies);
1322
1558
  });
1323
1559
  }
1324
1560
  /** An own org with calls on: the host's answer about its slot's ports. A failure or a slow answer is "not open". */
@@ -1405,6 +1641,9 @@ export class FoundryPlanApply {
1405
1641
  bouleArchonCredential: this.executor.bouleArchonCredentialAvailable?.() ?? false,
1406
1642
  ...(blueGreenServers ? { blueGreenServers } : {}),
1407
1643
  ...(stageSinkDown ? { stageSinkDown } : {}),
1644
+ ...(this.executor.stageGuardPorts
1645
+ ? { stageGuardPorts: this.executor.stageGuardPorts() }
1646
+ : {}),
1408
1647
  ...(blueGreenServers && org.hosting.entries.some((e) => e.blueGreen && e.database)
1409
1648
  ? {
1410
1649
  remoteDatabase: remoteDatabaseFactsFrom({
@@ -1783,12 +2022,16 @@ export class FoundryPlanApply {
1783
2022
  const archonBlueGreen = platform.archon.enabled && platform.archon.blueGreen && executor.archonBlueGreen
1784
2023
  ? await executor.archonBlueGreen().catch(() => ({ servers: undefined }))
1785
2024
  : null;
2025
+ const archonGuards = archonBlueGreen
2026
+ ? (executor.archonStageGuardNames?.(platform, serviceIds, owner) ?? [])
2027
+ : [];
1786
2028
  const detail = computePlatformPlanDetail(platform, live, {
1787
2029
  owner,
1788
2030
  ...(archonBlueGreen
1789
2031
  ? {
1790
2032
  archonBlueGreen: {
1791
2033
  servers: archonBlueGreen.servers,
2034
+ ...(archonGuards.length > 0 ? { guards: archonGuards } : {}),
1792
2035
  ...('blocked' in archonBlueGreen && archonBlueGreen.blocked
1793
2036
  ? { blocked: archonBlueGreen.blocked }
1794
2037
  : {}),
@@ -1816,6 +2059,7 @@ export class FoundryPlanApply {
1816
2059
  signIn: ownerSignInFrom(owner, recorded),
1817
2060
  files: ownerFilesFrom(owner, recorded),
1818
2061
  chat: ownerChatFrom(owner, recorded),
2062
+ emails: owner.owners.map((o) => o.email),
1819
2063
  };
1820
2064
  }
1821
2065
  /**
@@ -2270,6 +2514,11 @@ function destroyStepDescriptions(targets) {
2270
2514
  /** Scope keys of the Coolify apps blue/green made: `bg.<entry>.app.<server>.<part>`, a database sidecar, a connector on another server, the company's stage mail sink (`stage_sink_app`). */
2271
2515
  const BLUE_GREEN_APP_KEY = /^(bg\.[^.:]+\.app\.[^.:]+\.[a-z]+|bgdb\.[^.:]+\.sidecar\.[^.:]+|cloudflare_tunnel_connector_app\.[^.:]+|stage_sink_app)$/;
2272
2516
  const BLUE_GREEN_DATABASE_KEY = /^bg\.[^.:]+\.(stagedb|stagekept)$/;
2517
+ /** Stage's own sign-in: its broker entry (with the key) goes first, then its client. */
2518
+ const BLUE_GREEN_BROKER_KEY = /^bg\.[^.:]+\.stagebroker$/;
2519
+ const BLUE_GREEN_CLIENT_KEY = /^bg\.[^.:]+\.stageclient$/;
2520
+ /** Stage's own files and ai organisations and jobs project (`bg.<entry>.stage<service>org`), and the keys in them (`…keys`, revoked first). */
2521
+ const BLUE_GREEN_SERVICE_KEY = /^bg\.[^.:]+\.stage(files|ai|jobs)(org|keys)$/;
2273
2522
  /** Stage's Access gate (`bg.<entry>.stagegate`), and the database route's Cloudflare parts: its Access app, its service token (also one minted and never delivered) and its DNS record (recorded as the route). */
2274
2523
  const BLUE_GREEN_EDGE_KEYS = [
2275
2524
  [/^bgdb\.[^.:]+\.access$/, 'bg-access'],
@@ -2291,6 +2540,30 @@ function blueGreenDestroyTargets(serviceIds) {
2291
2540
  if (id)
2292
2541
  targets.push({ system: 'coolify', kind: 'bg-database', key, id });
2293
2542
  }
2543
+ for (const key of keys.filter((k) => BLUE_GREEN_BROKER_KEY.test(k))) {
2544
+ const id = serviceIds[key];
2545
+ if (id)
2546
+ targets.push({ system: 'coolify', kind: 'bg-broker', key, id });
2547
+ }
2548
+ for (const key of keys.filter((k) => BLUE_GREEN_CLIENT_KEY.test(k))) {
2549
+ const id = serviceIds[key];
2550
+ if (id)
2551
+ targets.push({ system: 'zitadel', kind: 'bg-client', key, id });
2552
+ }
2553
+ for (const part of ['keys', 'org']) {
2554
+ for (const key of keys) {
2555
+ const match = BLUE_GREEN_SERVICE_KEY.exec(key);
2556
+ const id = serviceIds[key];
2557
+ if (!match || match[2] !== part || !id)
2558
+ continue;
2559
+ targets.push({
2560
+ system: match[1],
2561
+ kind: part === 'keys' ? 'bg-keys' : 'bg-org',
2562
+ key,
2563
+ id,
2564
+ });
2565
+ }
2566
+ }
2294
2567
  for (const [pattern, kind] of BLUE_GREEN_EDGE_KEYS) {
2295
2568
  for (const key of keys.filter((k) => pattern.test(k))) {
2296
2569
  const id = serviceIds[key];
@@ -1188,6 +1188,8 @@ export interface PlanPlatformOptions {
1188
1188
  archonBlueGreen?: {
1189
1189
  servers: readonly BlueGreenServer[] | undefined;
1190
1190
  blocked?: string;
1191
+ /** The stage guards that can be made now ({@link stageGuardInputs}); they go into the applied hash. */
1192
+ guards?: readonly string[];
1191
1193
  };
1192
1194
  /** Whether the host supplies a Coolify reader for the platform (default `true`). */
1193
1195
  jobsObservable?: boolean;
@@ -1255,6 +1257,8 @@ export interface PlanPlatformOptions {
1255
1257
  backupExecutorMissing?: boolean;
1256
1258
  /** #540: why `hc` is blocked, or absent when it is not. */
1257
1259
  hc?: string;
1260
+ /** Why the image registry is blocked, or absent when it is not. */
1261
+ registry?: string;
1258
1262
  /**
1259
1263
  * #540: true once `hc` is applied but the host built the platform
1260
1264
  * executor with no heartbeats dependencies -- the heartbeat-checks step
@@ -1377,6 +1381,8 @@ export interface PlanPlatformOptions {
1377
1381
  authApps?: boolean;
1378
1382
  edge?: boolean;
1379
1383
  files?: boolean;
1384
+ /** A usable, reachable Coolify server has no registry login for the current host and passwords. */
1385
+ registryLogins?: boolean;
1380
1386
  backupStorage?: boolean;
1381
1387
  backupStorageBlocked?: string;
1382
1388
  };
@@ -11,6 +11,7 @@ import { legacyAppHostname } from './hostnames.js';
11
11
  import { PLATFORM_MAIL_RELAY_KEY, planMailRelay } from './mail-relay.js';
12
12
  import { MAIL_DATABASE_SCOPE_KEY, mailStoreIsLegacy, mailStoreSwitchChange, mailStoreSwitchDescription, } from './mail-store.js';
13
13
  import { planOikosIngress } from './oikos-ingress.js';
14
+ import { PLATFORM_REGISTRY_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_REGISTRY_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_REGISTRY_KEY, PLATFORM_REGISTRY_LOGINS_APPLIED_SCOPE_KEY, PLATFORM_REGISTRY_LOGINS_KEY, PLATFORM_REGISTRY_PASSWORDS_SET_SCOPE_KEY, PLATFORM_REGISTRY_PASSWORDS_UNDELIVERED_SCOPE_KEY, registryLoginVersion, } from './platform-registry-keys.js';
14
15
  import { callsSfuAppliedFingerprint } from './platform-sfu.js';
15
16
  import { postLogoutUpdateOwed, postLogoutUrisFor } from './post-logout-uris.js';
16
17
  import { archonInAppHost } from './sign-in-archon.js';
@@ -1418,6 +1419,12 @@ export function planPlatform(platform, live, options = {}) {
1418
1419
  const hc = planHc(platform, recordedIds, blocked.hc ?? dnsBlocked?.('heartbeat server'));
1419
1420
  if (hc.step)
1420
1421
  plannedSteps.push(hc.step);
1422
+ // The image registry: edge's DNS and tunnel like hc. The login step follows the app step.
1423
+ const registry = planRegistry(platform, recordedIds, blocked.registry ?? dnsBlocked?.('image registry'), drifted.registryLogins ?? false);
1424
+ if (registry.step)
1425
+ plannedSteps.push(registry.step);
1426
+ if (registry.loginsStep)
1427
+ plannedSteps.push(registry.loginsStep);
1421
1428
  // #540: the `authReady` pattern, for hc -- true once hc is either already
1422
1429
  // applied or about to run earlier in this very plan, so the heartbeats
1423
1430
  // reconcile below can be planned in the *same* round hc first comes up,
@@ -1494,6 +1501,7 @@ export function planPlatform(platform, live, options = {}) {
1494
1501
  recorded: recordedIds,
1495
1502
  servers: archonBlueGreen.servers,
1496
1503
  executorBlocked: archonBlueGreen.blocked,
1504
+ ...(archonBlueGreen.guards ? { guards: archonBlueGreen.guards } : {}),
1497
1505
  })
1498
1506
  : { step: null, unmanaged: [] };
1499
1507
  if (archonBg.step) {
@@ -1637,6 +1645,8 @@ export function planPlatform(platform, live, options = {}) {
1637
1645
  }
1638
1646
  if (hc.blocked)
1639
1647
  unmanaged.push({ description: hc.blocked, key: 'hc' });
1648
+ if (registry.blocked)
1649
+ unmanaged.push({ description: registry.blocked, key: 'registry' });
1640
1650
  if (archonOrg.blocked)
1641
1651
  unmanaged.push({ description: archonOrg.blocked, key: 'archon-org' });
1642
1652
  if (archon.blocked)
@@ -2887,6 +2897,70 @@ function planAccessProvider(platform, recordedIds, executorBlocked, ownerSignIn)
2887
2897
  * Blocked, not planned: `executorBlocked`, from `dnsBlocked` (edge neither
2888
2898
  * applied nor planned) or the host building no `hc` dependencies.
2889
2899
  */
2900
+ /**
2901
+ * The platform's image registry (C5): `coolify:app` keyed {@link PLATFORM_REGISTRY_KEY}, then
2902
+ * `coolify:registry-logins` keyed {@link PLATFORM_REGISTRY_LOGINS_KEY}, which logs every Coolify
2903
+ * server in as `pull`. The logins are planned while the host and passwords they last ran with
2904
+ * differ from the current ones, and again when a server has no login (`drifted`).
2905
+ */
2906
+ function planRegistry(platform, recordedIds, executorBlocked, loginsDrifted) {
2907
+ const none = { step: null, loginsStep: null, blocked: null };
2908
+ if (!platform.registry.enabled)
2909
+ return none;
2910
+ const { hostname, gitRef } = platform.registry;
2911
+ const undelivered = recordedIds[PLATFORM_REGISTRY_PASSWORDS_UNDELIVERED_SCOPE_KEY];
2912
+ if (undelivered && !recordedIds[PLATFORM_REGISTRY_PASSWORDS_SET_SCOPE_KEY]) {
2913
+ return {
2914
+ ...none,
2915
+ blocked: `the image registry is blocked, not planned: its passwords (${undelivered}) were made but never confirmed delivered and cannot be shown again; remove the recorded id ${PLATFORM_REGISTRY_PASSWORDS_UNDELIVERED_SCOPE_KEY} to make new ones`,
2916
+ };
2917
+ }
2918
+ const applied = Boolean(recordedIds[resourceKey('coolify', 'app', PLATFORM_REGISTRY_KEY)]) &&
2919
+ Boolean(recordedIds[PLATFORM_REGISTRY_PASSWORDS_SET_SCOPE_KEY]) &&
2920
+ recordedIds[PLATFORM_REGISTRY_APPLIED_HOSTNAME_SCOPE_KEY] === hostname &&
2921
+ recordedIds[PLATFORM_REGISTRY_APPLIED_GIT_REF_SCOPE_KEY] === gitRef;
2922
+ if (executorBlocked && !applied)
2923
+ return { ...none, blocked: executorBlocked };
2924
+ const step = applied
2925
+ ? null
2926
+ : {
2927
+ action: {
2928
+ system: 'coolify',
2929
+ kind: 'app',
2930
+ key: PLATFORM_REGISTRY_KEY,
2931
+ desired: { hostname, gitRef },
2932
+ },
2933
+ step: {
2934
+ description: `provision the image registry at ${hostname}: the build workflow pushes company app images there and the company servers pull them`,
2935
+ destroys: false,
2936
+ migrates: false,
2937
+ },
2938
+ };
2939
+ const loginsCurrent = recordedIds[PLATFORM_REGISTRY_LOGINS_APPLIED_SCOPE_KEY] ===
2940
+ registryLoginVersion(hostname, recordedIds[PLATFORM_REGISTRY_PASSWORDS_SET_SCOPE_KEY]) &&
2941
+ Boolean(recordedIds[PLATFORM_REGISTRY_PASSWORDS_SET_SCOPE_KEY]);
2942
+ const loginsStep = (loginsCurrent && !loginsDrifted) || executorBlocked
2943
+ ? null
2944
+ : {
2945
+ action: {
2946
+ system: 'coolify',
2947
+ kind: 'registry-logins',
2948
+ key: PLATFORM_REGISTRY_LOGINS_KEY,
2949
+ desired: { hostname },
2950
+ },
2951
+ step: {
2952
+ description: `log every Coolify server in to the image registry at ${hostname} so it can pull company app images`,
2953
+ destroys: false,
2954
+ migrates: false,
2955
+ },
2956
+ };
2957
+ const loginsNeeded = !loginsCurrent || loginsDrifted;
2958
+ return {
2959
+ step,
2960
+ loginsStep,
2961
+ blocked: executorBlocked && !step && loginsNeeded ? executorBlocked : null,
2962
+ };
2963
+ }
2890
2964
  function planHc(platform, recordedIds, executorBlocked) {
2891
2965
  if (!platform.hc.enabled)
2892
2966
  return { step: null, blocked: null };
@@ -373,6 +373,8 @@ export interface PlatformOwnerInput {
373
373
  signIn?: OwnerSignIn | null;
374
374
  files?: OwnerFiles | null;
375
375
  chat?: OwnerChat | null;
376
+ /** The owner company's owners' emails: who passes the Access gate over Archon's stage. */
377
+ emails?: readonly string[];
376
378
  }
377
379
  /**
378
380
  * The owner's sign-in from its declared org and the ids its own steps recorded.