@wtfalch/foundry 0.62.0 → 0.64.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apps/index.d.ts +1 -0
- package/dist/apps/index.js +1 -0
- package/dist/apps/server-entry.d.ts +43 -0
- package/dist/apps/server-entry.js +99 -0
- package/dist/bootstrap/declare.js +4 -1
- package/dist/bootstrap/versions.d.ts +2 -0
- package/dist/build-json.d.ts +25 -0
- package/dist/build-json.js +61 -0
- package/dist/catalogue/copy-en.js +10 -0
- package/dist/catalogue/integrations.js +10 -0
- package/dist/index.d.ts +8 -4
- package/dist/index.js +3 -2
- package/dist/migrations/0067_requested_changes_kosmos.sql +6 -0
- package/dist/migrations/0068_platform_registry.sql +17 -0
- package/dist/modules/customer-grants.d.ts +3 -0
- package/dist/modules/customer-grants.js +28 -0
- package/dist/modules/index.js +49 -1
- package/dist/modules/repository.d.ts +6 -0
- package/dist/modules/repository.js +40 -0
- package/dist/modules/types.d.ts +8 -0
- package/dist/native-build.d.ts +7 -27
- package/dist/native-build.js +7 -27
- package/dist/org.d.ts +19 -1
- package/dist/org.js +45 -3
- package/dist/plan/archon-blue-green.d.ts +5 -0
- package/dist/plan/archon-blue-green.js +2 -0
- package/dist/plan/blue-green-apply.d.ts +5 -0
- package/dist/plan/blue-green-apply.js +197 -22
- package/dist/plan/blue-green.d.ts +61 -2
- package/dist/plan/blue-green.js +80 -6
- package/dist/plan/boule.d.ts +1 -1
- package/dist/plan/boule.js +2 -0
- package/dist/plan/broker-entries.d.ts +9 -0
- package/dist/plan/broker-entries.js +29 -0
- package/dist/plan/diff.d.ts +5 -0
- package/dist/plan/diff.js +21 -2
- package/dist/plan/edge-access.d.ts +2 -3
- package/dist/plan/edge-access.js +7 -0
- package/dist/plan/executor.d.ts +71 -1
- package/dist/plan/executor.js +491 -56
- package/dist/plan/index.d.ts +2 -0
- package/dist/plan/index.js +1 -0
- package/dist/plan/jobs-scheduler-http.d.ts +2 -0
- package/dist/plan/jobs-scheduler-http.js +11 -0
- package/dist/plan/own-limits.d.ts +6 -0
- package/dist/plan/own-limits.js +1 -0
- package/dist/plan/plan-apply.d.ts +51 -1
- package/dist/plan/plan-apply.js +298 -25
- package/dist/plan/platform-diff.d.ts +23 -1
- package/dist/plan/platform-diff.js +96 -0
- package/dist/plan/platform-org.d.ts +2 -0
- package/dist/plan/platform-providers.d.ts +46 -1
- package/dist/plan/platform-providers.js +156 -1
- package/dist/plan/platform-registry-keys.d.ts +58 -0
- package/dist/plan/platform-registry-keys.js +69 -0
- package/dist/plan/platform-registry.d.ts +67 -0
- package/dist/plan/platform-registry.js +298 -0
- package/dist/plan/release-host.d.ts +31 -2
- package/dist/plan/release-host.js +114 -6
- package/dist/plan/stage-copy-host.d.ts +104 -0
- package/dist/plan/stage-copy-host.js +259 -0
- package/dist/plan/stage-copy.js +44 -0
- package/dist/plan/stage-env.d.ts +9 -0
- package/dist/plan/stage-env.js +38 -0
- package/dist/plan/stage-guards.d.ts +57 -1
- package/dist/plan/stage-guards.js +106 -0
- package/dist/plan/stage-isolation.js +56 -0
- package/dist/plan/stage-services.d.ts +93 -0
- package/dist/plan/stage-services.js +153 -0
- package/dist/plan/stage-signin.d.ts +152 -0
- package/dist/plan/stage-signin.js +133 -0
- package/dist/plan/stage-sink.d.ts +2 -0
- package/dist/plan/stage-sink.js +6 -1
- package/dist/platform.d.ts +7 -1
- package/dist/platform.js +1 -0
- package/dist/release/decide.d.ts +12 -0
- package/dist/release/decide.js +37 -2
- package/dist/release/run.d.ts +71 -1
- package/dist/release/run.js +92 -8
- package/dist/release/state.d.ts +14 -1
- package/dist/release/state.js +43 -0
- package/dist/release/types.d.ts +20 -0
- package/dist/release/types.js +2 -0
- package/dist/requested-changes.d.ts +11 -2
- package/dist/requested-changes.js +11 -0
- package/dist/store/postgres.js +8 -0
- package/dist/store/schema.d.ts +51 -0
- package/dist/store/schema.js +4 -0
- package/package.json +3 -2
- package/versions.digests.json +1 -0
- package/versions.json +1 -0
package/dist/plan/plan-apply.js
CHANGED
|
@@ -4,6 +4,7 @@ import { VERSIONS, floatingTagBlocked } from '../bootstrap/versions.js';
|
|
|
4
4
|
import { sfuSlotPorts } from '../calls-sfu.js';
|
|
5
5
|
import { ORG_SERVICE_SCHEMAS, heldEnvRef, ownsServices, validateHosting } from '../org.js';
|
|
6
6
|
import { PLATFORM_SERVICE_KEYS } from '../platform.js';
|
|
7
|
+
import { recordedFrom, releaseKeys } from '../release/state.js';
|
|
7
8
|
import { AlreadyRunningError, StepFailedError } from '../runner/errors.js';
|
|
8
9
|
import { StaleLeaseError, } from '../store.js';
|
|
9
10
|
import { entryHasDatabase } from '../worker-app-config.js';
|
|
@@ -36,8 +37,9 @@ import { DEFAULT_ORG_PLATFORM_READINESS, ORG_PLATFORM_KEY_REFS, agoraHostValuesF
|
|
|
36
37
|
import { PLATFORM_AUTH_BROKER_MAIL_KEY_REF, PLATFORM_BACKUP_S3_STORAGE_SCOPE_KEY, } from './platform-providers.js';
|
|
37
38
|
import { privatePageGateKey } from './private-pages.js';
|
|
38
39
|
import { CLOUDFLARE_TUNNEL_APPS_HOSTNAMES_SCOPE_KEY, CLOUDFLARE_TUNNEL_ENTRY_ROUTES_SCOPE_KEY, CLOUDFLARE_TUNNEL_OIKOS_HOSTNAMES_SCOPE_KEY, } from './proxy-labels.js';
|
|
39
|
-
import { runSubjectRelease } from './release-host.js';
|
|
40
|
+
import { finishStageCopy, runSubjectRelease, } from './release-host.js';
|
|
40
41
|
import { remoteDatabaseFactsFrom, remoteDatabaseKeys } from './remote-database.js';
|
|
42
|
+
import { STAGE_COPY_LOCK_MS, claimStageCopyLock, parseStageCopyLock, stageCopyLockValue, } from './stage-copy-host.js';
|
|
41
43
|
import { liveWorkerApps, workerAppTraceScopeKeys } from './worker-app.js';
|
|
42
44
|
import { liveWorkers, workerTraceScopeKeys } from './worker.js';
|
|
43
45
|
/** #441: the lease scope that serializes every mutation of one company, and the one for the shared platform. */
|
|
@@ -45,6 +47,8 @@ const companyScope = (orgId) => `company:${orgId}`;
|
|
|
45
47
|
const PLATFORM_SCOPE = 'platform';
|
|
46
48
|
/** A lease nobody renews: long enough for one apply, short enough that a dead worker's scope frees itself. The fence stops a slow worker that outlives it. */
|
|
47
49
|
const DEFAULT_LEASE_TTL_MS = 30 * 60 * 1000;
|
|
50
|
+
/** Whether any of the subjects has a release stepping. */
|
|
51
|
+
const releaseRunning = (subjects, ids) => subjects.some((s) => recordedFrom(ids, s.id)?.release != null);
|
|
48
52
|
/** #441: another apply, destroy, adopt or release holds the lease for this company or the platform. Nothing ran; try again when it finishes. */
|
|
49
53
|
export class MutationInProgressError extends Error {
|
|
50
54
|
scope;
|
|
@@ -195,6 +199,7 @@ export class FoundryPlanApply {
|
|
|
195
199
|
onlySteps;
|
|
196
200
|
now;
|
|
197
201
|
leaseTtlMs;
|
|
202
|
+
copyLeaseRetry;
|
|
198
203
|
appliedVersions;
|
|
199
204
|
backupGate;
|
|
200
205
|
platformDeps;
|
|
@@ -238,6 +243,7 @@ export class FoundryPlanApply {
|
|
|
238
243
|
this.onlySteps = options.onlySteps;
|
|
239
244
|
this.now = options.now ?? (() => new Date());
|
|
240
245
|
this.leaseTtlMs = options.leaseTtlMs ?? DEFAULT_LEASE_TTL_MS;
|
|
246
|
+
this.copyLeaseRetry = options.copyLeaseRetry ?? { tries: 10, waitMs: 30_000 };
|
|
241
247
|
this.appliedVersions = options.appliedVersions;
|
|
242
248
|
this.backupGate = options.backupGate;
|
|
243
249
|
this.platformDeps = options.platform;
|
|
@@ -1163,22 +1169,28 @@ export class FoundryPlanApply {
|
|
|
1163
1169
|
* (the live ids of the run that holds the lease). A weight the runner changed reaches every server's balancer
|
|
1164
1170
|
* before this returns.
|
|
1165
1171
|
*/
|
|
1166
|
-
async runOrgReleases(org, context, fence) {
|
|
1172
|
+
async runOrgReleases(org, context, fence, copies = 'defer') {
|
|
1173
|
+
const out = { deferred: false, claimed: [] };
|
|
1167
1174
|
const store = this.releaseStore;
|
|
1168
1175
|
const primaryUuid = context.recordedIds()[PRIMARY_SERVER_SCOPE_KEY];
|
|
1169
1176
|
if (!store || !this.executor.releaseHost || !primaryUuid)
|
|
1170
|
-
return;
|
|
1171
|
-
|
|
1177
|
+
return out;
|
|
1178
|
+
const subjects = blueGreenSubjects(org, context.recordedIds());
|
|
1179
|
+
// A copy waits while any subject of the org has a release running: the runner must stay free to roll back.
|
|
1180
|
+
if (copies === 'claim' && releaseRunning(subjects, context.recordedIds()))
|
|
1181
|
+
return out;
|
|
1182
|
+
for (const subject of subjects) {
|
|
1172
1183
|
// A built-in app's names reach its balancer only once the tunnel was last written with it: until then a
|
|
1173
1184
|
// pinned check would answer from blue whatever color it names.
|
|
1174
1185
|
if (subject.builtin &&
|
|
1175
1186
|
!(context.recordedIds()[CLOUDFLARE_TUNNEL_ENTRY_ROUTES_SCOPE_KEY] ?? '').includes(`"service":"${lbOrigin(subject)}"`)) {
|
|
1176
1187
|
continue;
|
|
1177
1188
|
}
|
|
1189
|
+
await this.executor.syncStageGateMembers?.(org, subject, context);
|
|
1178
1190
|
const host = this.executor.releaseHost(org, subject, context, () => this.renewLease(fence));
|
|
1179
1191
|
if (!host)
|
|
1180
1192
|
continue;
|
|
1181
|
-
await runSubjectRelease({
|
|
1193
|
+
const result = await runSubjectRelease({
|
|
1182
1194
|
store,
|
|
1183
1195
|
scopeOwnerId: org.id,
|
|
1184
1196
|
subject,
|
|
@@ -1186,21 +1198,40 @@ export class FoundryPlanApply {
|
|
|
1186
1198
|
context,
|
|
1187
1199
|
now: this.now(),
|
|
1188
1200
|
primaryUuid,
|
|
1201
|
+
...(copies === 'defer' ? { deferCopies: true } : {}),
|
|
1189
1202
|
});
|
|
1203
|
+
out.deferred ||= result.deferredCopy;
|
|
1204
|
+
if (copies === 'claim' && result.copy) {
|
|
1205
|
+
// The copy's own lock is taken here, under the lease; the copy then runs without it.
|
|
1206
|
+
const runId = randomUUID();
|
|
1207
|
+
const claim = await claimStageCopyLock({
|
|
1208
|
+
subjectId: subject.id,
|
|
1209
|
+
context,
|
|
1210
|
+
requestId: result.copy.requestId,
|
|
1211
|
+
runId,
|
|
1212
|
+
});
|
|
1213
|
+
if (claim === 'claimed')
|
|
1214
|
+
out.claimed.push({ subject, job: { ...result.copy, runId } });
|
|
1215
|
+
}
|
|
1190
1216
|
}
|
|
1217
|
+
return out;
|
|
1191
1218
|
}
|
|
1192
1219
|
/** One release run for platform-archon, on the platform's live ids; see {@link runOrgReleases}. */
|
|
1193
|
-
async runPlatformReleases(platform, context, fence) {
|
|
1220
|
+
async runPlatformReleases(platform, context, fence, copies = 'defer') {
|
|
1221
|
+
const out = { deferred: false, claimed: [] };
|
|
1194
1222
|
const store = this.releaseStore;
|
|
1195
1223
|
const executor = this.platformDeps?.executor;
|
|
1196
1224
|
const subject = archonSubject(platform, ARCHON_REPO);
|
|
1197
1225
|
const primaryUuid = context.recordedIds()[PRIMARY_SERVER_SCOPE_KEY];
|
|
1198
1226
|
if (!store || !executor?.releaseHost || !subject || !primaryUuid)
|
|
1199
|
-
return;
|
|
1227
|
+
return out;
|
|
1228
|
+
if (copies === 'claim' && releaseRunning([subject], context.recordedIds()))
|
|
1229
|
+
return out;
|
|
1230
|
+
await executor.syncStageGateMembers?.(platform, subject, context);
|
|
1200
1231
|
const host = executor.releaseHost(platform, subject, context, () => this.renewLease(fence));
|
|
1201
1232
|
if (!host)
|
|
1202
|
-
return;
|
|
1203
|
-
await runSubjectRelease({
|
|
1233
|
+
return out;
|
|
1234
|
+
const result = await runSubjectRelease({
|
|
1204
1235
|
store,
|
|
1205
1236
|
scopeOwnerId: 'platform',
|
|
1206
1237
|
subject,
|
|
@@ -1208,21 +1239,35 @@ export class FoundryPlanApply {
|
|
|
1208
1239
|
context,
|
|
1209
1240
|
now: this.now(),
|
|
1210
1241
|
primaryUuid,
|
|
1242
|
+
...(copies === 'defer' ? { deferCopies: true } : {}),
|
|
1211
1243
|
});
|
|
1244
|
+
out.deferred = result.deferredCopy;
|
|
1245
|
+
if (copies === 'claim' && result.copy) {
|
|
1246
|
+
const runId = randomUUID();
|
|
1247
|
+
const claim = await claimStageCopyLock({
|
|
1248
|
+
subjectId: subject.id,
|
|
1249
|
+
context,
|
|
1250
|
+
requestId: result.copy.requestId,
|
|
1251
|
+
runId,
|
|
1252
|
+
});
|
|
1253
|
+
if (claim === 'claimed')
|
|
1254
|
+
out.claimed.push({ subject, job: { ...result.copy, runId } });
|
|
1255
|
+
}
|
|
1256
|
+
return out;
|
|
1212
1257
|
}
|
|
1213
|
-
async runPlatformReleaseTick() {
|
|
1258
|
+
async runPlatformReleaseTick(copies) {
|
|
1214
1259
|
const deps = this.platformDeps;
|
|
1215
1260
|
const platform = await deps?.store.get();
|
|
1216
1261
|
if (!deps || !platform)
|
|
1217
|
-
return;
|
|
1262
|
+
return { deferred: false, claimed: [] };
|
|
1218
1263
|
const empty = { orgId: platform.ownerOrgId, steps: [], unmanaged: [] };
|
|
1219
1264
|
if ((await this.authz.canApplyPlatform?.(this.actor)) !== true) {
|
|
1220
1265
|
await this.refusePlatform(empty, platform.ownerOrgId, 'not authorized to run the release tick');
|
|
1221
1266
|
throw new NotAuthorizedError(this.actor, platform.ownerOrgId);
|
|
1222
1267
|
}
|
|
1223
|
-
|
|
1268
|
+
return this.withMutationLease(PLATFORM_SCOPE, (reason) => this.refusePlatform(empty, platform.ownerOrgId, reason), async (fence) => {
|
|
1224
1269
|
const serviceIds = { ...(await deps.store.getServiceIds()) };
|
|
1225
|
-
|
|
1270
|
+
return this.runPlatformReleases(platform, {
|
|
1226
1271
|
recordedIds: () => ({ ...serviceIds }),
|
|
1227
1272
|
recordScopeId: (key, value) => {
|
|
1228
1273
|
serviceIds[key] = value;
|
|
@@ -1231,7 +1276,7 @@ export class FoundryPlanApply {
|
|
|
1231
1276
|
delete serviceIds[key];
|
|
1232
1277
|
},
|
|
1233
1278
|
persist: () => deps.store.putServiceIds({ ...serviceIds }, fence),
|
|
1234
|
-
}, fence);
|
|
1279
|
+
}, fence, copies);
|
|
1235
1280
|
});
|
|
1236
1281
|
}
|
|
1237
1282
|
/**
|
|
@@ -1243,29 +1288,216 @@ export class FoundryPlanApply {
|
|
|
1243
1288
|
*/
|
|
1244
1289
|
async runReleaseTick() {
|
|
1245
1290
|
const results = [];
|
|
1291
|
+
const settle = (target, error) => {
|
|
1292
|
+
const entry = {
|
|
1293
|
+
target,
|
|
1294
|
+
...(error === undefined
|
|
1295
|
+
? {}
|
|
1296
|
+
: { error: error instanceof Error ? error.message : String(error) }),
|
|
1297
|
+
};
|
|
1298
|
+
const at = results.findIndex((r) => r.target === target);
|
|
1299
|
+
if (at === -1)
|
|
1300
|
+
results.push(entry);
|
|
1301
|
+
else
|
|
1302
|
+
results[at] = entry;
|
|
1303
|
+
};
|
|
1304
|
+
// Phase 1: every org's and the platform's release, with stage copy requests left open. A copy takes up to
|
|
1305
|
+
// ~45 minutes, so no copy runs before every other release of this tick has run.
|
|
1306
|
+
const copying = [];
|
|
1246
1307
|
for (const org of await this.store.list()) {
|
|
1247
1308
|
if (!declaresBlueGreen(org))
|
|
1248
1309
|
continue;
|
|
1249
1310
|
try {
|
|
1250
|
-
await this.runOrgReleaseTick(org.id)
|
|
1251
|
-
|
|
1311
|
+
if ((await this.runOrgReleaseTick(org.id, 'defer')).deferred) {
|
|
1312
|
+
copying.push({ target: org.id, run: () => this.runOrgStageCopies(org.id) });
|
|
1313
|
+
}
|
|
1314
|
+
settle(org.id, undefined);
|
|
1252
1315
|
}
|
|
1253
1316
|
catch (e) {
|
|
1254
|
-
|
|
1317
|
+
settle(org.id, e);
|
|
1255
1318
|
}
|
|
1256
1319
|
}
|
|
1257
1320
|
const platform = await this.platformDeps?.store.get();
|
|
1258
1321
|
if (platform?.archon.enabled && platform.archon.blueGreen) {
|
|
1259
1322
|
try {
|
|
1260
|
-
await this.runPlatformReleaseTick()
|
|
1261
|
-
|
|
1323
|
+
if ((await this.runPlatformReleaseTick('defer')).deferred) {
|
|
1324
|
+
copying.push({ target: 'platform', run: () => this.runPlatformStageCopies() });
|
|
1325
|
+
}
|
|
1326
|
+
settle('platform', undefined);
|
|
1327
|
+
}
|
|
1328
|
+
catch (e) {
|
|
1329
|
+
settle('platform', e);
|
|
1330
|
+
}
|
|
1331
|
+
}
|
|
1332
|
+
// Phase 2: the copies, one subject at a time. A copy takes its lock under the org's lease and runs without it.
|
|
1333
|
+
for (const { target, run } of copying) {
|
|
1334
|
+
try {
|
|
1335
|
+
await run();
|
|
1336
|
+
settle(target, undefined);
|
|
1262
1337
|
}
|
|
1263
1338
|
catch (e) {
|
|
1264
|
-
|
|
1339
|
+
settle(target, e);
|
|
1265
1340
|
}
|
|
1266
1341
|
}
|
|
1267
1342
|
return results;
|
|
1268
1343
|
}
|
|
1344
|
+
/**
|
|
1345
|
+
* Phase 2 for one org: takes the copy lock of each stage copy request under the org's lease, then runs the
|
|
1346
|
+
* copies after the lease is released. While a copy runs, every other tick and apply for the org goes on, a
|
|
1347
|
+
* hard-fail rollback included; the copy records through short leases ({@link copyContext}).
|
|
1348
|
+
*/
|
|
1349
|
+
async runOrgStageCopies(id) {
|
|
1350
|
+
const claimed = (await this.runOrgReleaseTick(id, 'claim')).claimed;
|
|
1351
|
+
if (claimed.length === 0)
|
|
1352
|
+
return;
|
|
1353
|
+
const org = await this.store.get(id);
|
|
1354
|
+
if (!org)
|
|
1355
|
+
return;
|
|
1356
|
+
const target = {
|
|
1357
|
+
scope: companyScope(id),
|
|
1358
|
+
refuse: (reason) => this.refuse({ orgId: id, steps: [], unmanaged: [] }, reason),
|
|
1359
|
+
read: async () => {
|
|
1360
|
+
const state = await this.store.getAppliedState(id);
|
|
1361
|
+
if (!state)
|
|
1362
|
+
return null;
|
|
1363
|
+
return {
|
|
1364
|
+
ids: { ...state.serviceIds },
|
|
1365
|
+
write: (ids, fence) => this.store.putAppliedState({ ...state, serviceIds: { ...ids } }, fence),
|
|
1366
|
+
};
|
|
1367
|
+
},
|
|
1368
|
+
host: (subject, context, renew) => this.executor.releaseHost?.(org, subject, context, renew),
|
|
1369
|
+
};
|
|
1370
|
+
for (const copy of claimed)
|
|
1371
|
+
await this.runClaimedCopy(target, copy);
|
|
1372
|
+
}
|
|
1373
|
+
/** Phase 2 for platform-archon; see {@link runOrgStageCopies}. */
|
|
1374
|
+
async runPlatformStageCopies() {
|
|
1375
|
+
const claimed = (await this.runPlatformReleaseTick('claim')).claimed;
|
|
1376
|
+
if (claimed.length === 0)
|
|
1377
|
+
return;
|
|
1378
|
+
const deps = this.platformDeps;
|
|
1379
|
+
const platform = await deps?.store.get();
|
|
1380
|
+
if (!deps || !platform)
|
|
1381
|
+
return;
|
|
1382
|
+
const target = {
|
|
1383
|
+
scope: PLATFORM_SCOPE,
|
|
1384
|
+
refuse: (reason) => this.refusePlatform({ orgId: platform.ownerOrgId, steps: [], unmanaged: [] }, platform.ownerOrgId, reason),
|
|
1385
|
+
read: async () => ({
|
|
1386
|
+
ids: { ...(await deps.store.getServiceIds()) },
|
|
1387
|
+
write: (ids, fence) => deps.store.putServiceIds({ ...ids }, fence),
|
|
1388
|
+
}),
|
|
1389
|
+
host: (subject, context, renew) => deps.executor.releaseHost?.(platform, subject, context, renew),
|
|
1390
|
+
};
|
|
1391
|
+
for (const copy of claimed)
|
|
1392
|
+
await this.runClaimedCopy(target, copy);
|
|
1393
|
+
}
|
|
1394
|
+
/**
|
|
1395
|
+
* Runs one claimed copy with no lease held, then records its outcome and consumes the request. A lost lock
|
|
1396
|
+
* throws (`StaleLeaseError`) and leaves the request open.
|
|
1397
|
+
*/
|
|
1398
|
+
async runClaimedCopy(target, claimed) {
|
|
1399
|
+
const store = this.releaseStore;
|
|
1400
|
+
const { subject, job } = claimed;
|
|
1401
|
+
const first = await target.read();
|
|
1402
|
+
if (!store || !first)
|
|
1403
|
+
return;
|
|
1404
|
+
const lockKey = releaseKeys(subject.id).stageCopyLock;
|
|
1405
|
+
const context = this.copyContext(target, first.ids, lockKey, job.requestId, job.runId ?? '');
|
|
1406
|
+
await finishStageCopy({
|
|
1407
|
+
store,
|
|
1408
|
+
host: target.host(subject, context, () => context.extendLock()),
|
|
1409
|
+
subject,
|
|
1410
|
+
context,
|
|
1411
|
+
now: this.now(),
|
|
1412
|
+
job,
|
|
1413
|
+
});
|
|
1414
|
+
}
|
|
1415
|
+
/**
|
|
1416
|
+
* The recorded ids a copy reads and writes, without the org's lease. It reads a snapshot taken when the lock was
|
|
1417
|
+
* claimed; each `persist` takes the lease for a moment, re-reads the stored ids and writes only the keys the
|
|
1418
|
+
* copy changed, so a tick that ran in between loses nothing. The lease may be busy: it waits and tries again.
|
|
1419
|
+
*/
|
|
1420
|
+
copyContext(target, initial, lockKey, requestId, runId) {
|
|
1421
|
+
const ids = { ...initial };
|
|
1422
|
+
// When the lock this run holds lapses; a beat that cannot renew it by then fails the run as stale.
|
|
1423
|
+
let released = false;
|
|
1424
|
+
let lockExpiry = parseStageCopyLock(initial[lockKey]).expiry;
|
|
1425
|
+
const dirty = new Map();
|
|
1426
|
+
let queue = Promise.resolve();
|
|
1427
|
+
const serial = (fn) => {
|
|
1428
|
+
const run = queue.then(fn, fn);
|
|
1429
|
+
queue = run.catch(() => undefined);
|
|
1430
|
+
return run;
|
|
1431
|
+
};
|
|
1432
|
+
const write = async (changes, tries, mustHoldLock) => {
|
|
1433
|
+
for (let attempt = 1;; attempt++) {
|
|
1434
|
+
const lease = await this.store.acquireMutationLease(target.scope, randomUUID(), this.leaseTtlMs);
|
|
1435
|
+
if ('token' in lease) {
|
|
1436
|
+
try {
|
|
1437
|
+
const stored = await target.read();
|
|
1438
|
+
if (!stored)
|
|
1439
|
+
throw new Error('stage copy: the recorded state is gone');
|
|
1440
|
+
// Another run's lock means this run lapsed and was replaced: write nothing.
|
|
1441
|
+
// No lock is fine only after this run removed it itself (its result, or its failure path).
|
|
1442
|
+
const heldBy = stored.ids[lockKey];
|
|
1443
|
+
if (heldBy ? parseStageCopyLock(heldBy).runId !== runId : mustHoldLock || !released) {
|
|
1444
|
+
throw new StaleLeaseError(target.scope);
|
|
1445
|
+
}
|
|
1446
|
+
for (const [key, value] of changes) {
|
|
1447
|
+
if (value === null)
|
|
1448
|
+
delete stored.ids[key];
|
|
1449
|
+
else
|
|
1450
|
+
stored.ids[key] = value;
|
|
1451
|
+
}
|
|
1452
|
+
await stored.write(stored.ids, lease);
|
|
1453
|
+
if (changes.get(lockKey) === null)
|
|
1454
|
+
released = true;
|
|
1455
|
+
return true;
|
|
1456
|
+
}
|
|
1457
|
+
finally {
|
|
1458
|
+
await this.store.releaseMutationLease(lease).catch(() => undefined);
|
|
1459
|
+
}
|
|
1460
|
+
}
|
|
1461
|
+
if (attempt >= tries) {
|
|
1462
|
+
if (tries === 1)
|
|
1463
|
+
return false;
|
|
1464
|
+
throw new MutationInProgressError(target.scope, lease.heldUntil);
|
|
1465
|
+
}
|
|
1466
|
+
await new Promise((resolve) => setTimeout(resolve, this.copyLeaseRetry.waitMs));
|
|
1467
|
+
}
|
|
1468
|
+
};
|
|
1469
|
+
return {
|
|
1470
|
+
recordedIds: () => ({ ...ids }),
|
|
1471
|
+
recordScopeId: (key, value) => {
|
|
1472
|
+
ids[key] = value;
|
|
1473
|
+
dirty.set(key, value);
|
|
1474
|
+
},
|
|
1475
|
+
forgetId: (key) => {
|
|
1476
|
+
delete ids[key];
|
|
1477
|
+
dirty.set(key, null);
|
|
1478
|
+
},
|
|
1479
|
+
persist: () => serial(async () => {
|
|
1480
|
+
const batch = new Map(dirty);
|
|
1481
|
+
await write(batch, this.copyLeaseRetry.tries, false);
|
|
1482
|
+
for (const [key, value] of batch)
|
|
1483
|
+
if (dirty.get(key) === value)
|
|
1484
|
+
dirty.delete(key);
|
|
1485
|
+
}),
|
|
1486
|
+
// Keeps the copy's lock alive. A lock another run took over throws. A busy lease is skipped while the lock
|
|
1487
|
+
// still stands; once it would lapse, a beat that cannot renew it fails the run as stale.
|
|
1488
|
+
extendLock: () => serial(async () => {
|
|
1489
|
+
const expiry = Date.now() + STAGE_COPY_LOCK_MS;
|
|
1490
|
+
const value = stageCopyLockValue(expiry, requestId, runId);
|
|
1491
|
+
if (await write(new Map([[lockKey, value]]), 1, true)) {
|
|
1492
|
+
ids[lockKey] = value;
|
|
1493
|
+
lockExpiry = expiry;
|
|
1494
|
+
}
|
|
1495
|
+
else if (Date.now() >= lockExpiry) {
|
|
1496
|
+
throw new StaleLeaseError(target.scope);
|
|
1497
|
+
}
|
|
1498
|
+
}),
|
|
1499
|
+
};
|
|
1500
|
+
}
|
|
1269
1501
|
/**
|
|
1270
1502
|
* The recurring audit sealing run (foundry#929): seals every ledger registered by an `audit:ledger` step with the
|
|
1271
1503
|
* platform signer and has the timestamp authority stamp the new checkpoints. Foundry has no scheduler, so the host
|
|
@@ -1297,19 +1529,23 @@ export class FoundryPlanApply {
|
|
|
1297
1529
|
allowed: (org) => this.authz.canApply(this.actor, org.id),
|
|
1298
1530
|
});
|
|
1299
1531
|
}
|
|
1300
|
-
|
|
1532
|
+
/**
|
|
1533
|
+
* One org's release run. `defer`: stage copy requests stay open and `deferred` says whether there were any.
|
|
1534
|
+
* `claim`: the copies this run decided on, each with its lock taken, for {@link runClaimedCopy}.
|
|
1535
|
+
*/
|
|
1536
|
+
async runOrgReleaseTick(id, copies) {
|
|
1301
1537
|
const empty = { orgId: id, steps: [], unmanaged: [] };
|
|
1302
1538
|
if (!(await this.authz.canApply(this.actor, id))) {
|
|
1303
1539
|
await this.refuse(empty, 'not authorized to run the release tick');
|
|
1304
1540
|
throw new NotAuthorizedError(this.actor, id);
|
|
1305
1541
|
}
|
|
1306
|
-
|
|
1542
|
+
return this.withMutationLease(companyScope(id), (reason) => this.refuse(empty, reason), async (fence) => {
|
|
1307
1543
|
const org = await this.store.get(id);
|
|
1308
1544
|
const previous = await this.store.getAppliedState(id);
|
|
1309
1545
|
if (!org || !previous)
|
|
1310
|
-
return;
|
|
1546
|
+
return { deferred: false, claimed: [] };
|
|
1311
1547
|
const serviceIds = { ...previous.serviceIds };
|
|
1312
|
-
|
|
1548
|
+
return this.runOrgReleases(org, {
|
|
1313
1549
|
recordedIds: () => ({ ...serviceIds }),
|
|
1314
1550
|
recordScopeId: (key, value) => {
|
|
1315
1551
|
serviceIds[key] = value;
|
|
@@ -1318,7 +1554,7 @@ export class FoundryPlanApply {
|
|
|
1318
1554
|
delete serviceIds[key];
|
|
1319
1555
|
},
|
|
1320
1556
|
persist: () => this.store.putAppliedState({ ...previous, serviceIds: { ...serviceIds } }, fence),
|
|
1321
|
-
}, fence);
|
|
1557
|
+
}, fence, copies);
|
|
1322
1558
|
});
|
|
1323
1559
|
}
|
|
1324
1560
|
/** An own org with calls on: the host's answer about its slot's ports. A failure or a slow answer is "not open". */
|
|
@@ -1405,6 +1641,9 @@ export class FoundryPlanApply {
|
|
|
1405
1641
|
bouleArchonCredential: this.executor.bouleArchonCredentialAvailable?.() ?? false,
|
|
1406
1642
|
...(blueGreenServers ? { blueGreenServers } : {}),
|
|
1407
1643
|
...(stageSinkDown ? { stageSinkDown } : {}),
|
|
1644
|
+
...(this.executor.stageGuardPorts
|
|
1645
|
+
? { stageGuardPorts: this.executor.stageGuardPorts() }
|
|
1646
|
+
: {}),
|
|
1408
1647
|
...(blueGreenServers && org.hosting.entries.some((e) => e.blueGreen && e.database)
|
|
1409
1648
|
? {
|
|
1410
1649
|
remoteDatabase: remoteDatabaseFactsFrom({
|
|
@@ -1783,12 +2022,16 @@ export class FoundryPlanApply {
|
|
|
1783
2022
|
const archonBlueGreen = platform.archon.enabled && platform.archon.blueGreen && executor.archonBlueGreen
|
|
1784
2023
|
? await executor.archonBlueGreen().catch(() => ({ servers: undefined }))
|
|
1785
2024
|
: null;
|
|
2025
|
+
const archonGuards = archonBlueGreen
|
|
2026
|
+
? (executor.archonStageGuardNames?.(platform, serviceIds, owner) ?? [])
|
|
2027
|
+
: [];
|
|
1786
2028
|
const detail = computePlatformPlanDetail(platform, live, {
|
|
1787
2029
|
owner,
|
|
1788
2030
|
...(archonBlueGreen
|
|
1789
2031
|
? {
|
|
1790
2032
|
archonBlueGreen: {
|
|
1791
2033
|
servers: archonBlueGreen.servers,
|
|
2034
|
+
...(archonGuards.length > 0 ? { guards: archonGuards } : {}),
|
|
1792
2035
|
...('blocked' in archonBlueGreen && archonBlueGreen.blocked
|
|
1793
2036
|
? { blocked: archonBlueGreen.blocked }
|
|
1794
2037
|
: {}),
|
|
@@ -1816,6 +2059,7 @@ export class FoundryPlanApply {
|
|
|
1816
2059
|
signIn: ownerSignInFrom(owner, recorded),
|
|
1817
2060
|
files: ownerFilesFrom(owner, recorded),
|
|
1818
2061
|
chat: ownerChatFrom(owner, recorded),
|
|
2062
|
+
emails: owner.owners.map((o) => o.email),
|
|
1819
2063
|
};
|
|
1820
2064
|
}
|
|
1821
2065
|
/**
|
|
@@ -2270,6 +2514,11 @@ function destroyStepDescriptions(targets) {
|
|
|
2270
2514
|
/** Scope keys of the Coolify apps blue/green made: `bg.<entry>.app.<server>.<part>`, a database sidecar, a connector on another server, the company's stage mail sink (`stage_sink_app`). */
|
|
2271
2515
|
const BLUE_GREEN_APP_KEY = /^(bg\.[^.:]+\.app\.[^.:]+\.[a-z]+|bgdb\.[^.:]+\.sidecar\.[^.:]+|cloudflare_tunnel_connector_app\.[^.:]+|stage_sink_app)$/;
|
|
2272
2516
|
const BLUE_GREEN_DATABASE_KEY = /^bg\.[^.:]+\.(stagedb|stagekept)$/;
|
|
2517
|
+
/** Stage's own sign-in: its broker entry (with the key) goes first, then its client. */
|
|
2518
|
+
const BLUE_GREEN_BROKER_KEY = /^bg\.[^.:]+\.stagebroker$/;
|
|
2519
|
+
const BLUE_GREEN_CLIENT_KEY = /^bg\.[^.:]+\.stageclient$/;
|
|
2520
|
+
/** Stage's own files and ai organisations and jobs project (`bg.<entry>.stage<service>org`), and the keys in them (`…keys`, revoked first). */
|
|
2521
|
+
const BLUE_GREEN_SERVICE_KEY = /^bg\.[^.:]+\.stage(files|ai|jobs)(org|keys)$/;
|
|
2273
2522
|
/** Stage's Access gate (`bg.<entry>.stagegate`), and the database route's Cloudflare parts: its Access app, its service token (also one minted and never delivered) and its DNS record (recorded as the route). */
|
|
2274
2523
|
const BLUE_GREEN_EDGE_KEYS = [
|
|
2275
2524
|
[/^bgdb\.[^.:]+\.access$/, 'bg-access'],
|
|
@@ -2291,6 +2540,30 @@ function blueGreenDestroyTargets(serviceIds) {
|
|
|
2291
2540
|
if (id)
|
|
2292
2541
|
targets.push({ system: 'coolify', kind: 'bg-database', key, id });
|
|
2293
2542
|
}
|
|
2543
|
+
for (const key of keys.filter((k) => BLUE_GREEN_BROKER_KEY.test(k))) {
|
|
2544
|
+
const id = serviceIds[key];
|
|
2545
|
+
if (id)
|
|
2546
|
+
targets.push({ system: 'coolify', kind: 'bg-broker', key, id });
|
|
2547
|
+
}
|
|
2548
|
+
for (const key of keys.filter((k) => BLUE_GREEN_CLIENT_KEY.test(k))) {
|
|
2549
|
+
const id = serviceIds[key];
|
|
2550
|
+
if (id)
|
|
2551
|
+
targets.push({ system: 'zitadel', kind: 'bg-client', key, id });
|
|
2552
|
+
}
|
|
2553
|
+
for (const part of ['keys', 'org']) {
|
|
2554
|
+
for (const key of keys) {
|
|
2555
|
+
const match = BLUE_GREEN_SERVICE_KEY.exec(key);
|
|
2556
|
+
const id = serviceIds[key];
|
|
2557
|
+
if (!match || match[2] !== part || !id)
|
|
2558
|
+
continue;
|
|
2559
|
+
targets.push({
|
|
2560
|
+
system: match[1],
|
|
2561
|
+
kind: part === 'keys' ? 'bg-keys' : 'bg-org',
|
|
2562
|
+
key,
|
|
2563
|
+
id,
|
|
2564
|
+
});
|
|
2565
|
+
}
|
|
2566
|
+
}
|
|
2294
2567
|
for (const [pattern, kind] of BLUE_GREEN_EDGE_KEYS) {
|
|
2295
2568
|
for (const key of keys.filter((k) => pattern.test(k))) {
|
|
2296
2569
|
const id = serviceIds[key];
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { DeclaredOrg, EnvSharedSecret, HostingEntry, OrgId } from '../org.js';
|
|
1
|
+
import type { DeclaredOrg, EntryImage, EnvSharedSecret, HostingEntry, OrgId } from '../org.js';
|
|
2
2
|
import { type Plan, type PlanNote, type UnmanagedItem, type VersionChangeItem } from '../plan.js';
|
|
3
3
|
import type { DeclaredPlatform } from '../platform.js';
|
|
4
4
|
import { type BlueGreenServer } from './blue-green.js';
|
|
@@ -725,6 +725,22 @@ export declare const DEFAULT_APP_SETTINGS: AppSettings;
|
|
|
725
725
|
export declare function appSettingsAppliedScopeKey(entryKey: string): string;
|
|
726
726
|
/** The settings the app has: the marker, or the defaults when there is none. */
|
|
727
727
|
export declare function appliedAppSettings(recorded: Readonly<Record<string, string>>, entryKey: string): AppSettings;
|
|
728
|
+
/**
|
|
729
|
+
* The image an entry's app last ran, as `repository@digest` (no secret). `planApp` plans the step again, with a
|
|
730
|
+
* redeploy, when the entry's `image` differs from it.
|
|
731
|
+
*/
|
|
732
|
+
export declare function appImageAppliedScopeKey(entryKey: string): string;
|
|
733
|
+
/** An entry image as one pull reference. */
|
|
734
|
+
export declare function entryImageRef(image: EntryImage): string;
|
|
735
|
+
/**
|
|
736
|
+
* The two image fields Coolify accepts for an entry image. Its tag rule has no `@` or `:`, so the digest goes in
|
|
737
|
+
* the name as `<repository>@sha256` and the tag holds the bare 64 hex digits (Coolify pulls `<name>:<tag>`),
|
|
738
|
+
* the same way `coolifyImageFor` writes a pinned platform image.
|
|
739
|
+
*/
|
|
740
|
+
export declare function entryImageFields(image: EntryImage): {
|
|
741
|
+
docker_registry_image_name: string;
|
|
742
|
+
docker_registry_image_tag: string;
|
|
743
|
+
};
|
|
728
744
|
/**
|
|
729
745
|
* The URLs `coolifyApp` last put in the entry's Coolify `domains`
|
|
730
746
|
* (`tunnelDomainList`), as a JSON array (`[]` for an internal entry). `planApp`
|
|
@@ -1188,6 +1204,8 @@ export interface PlanPlatformOptions {
|
|
|
1188
1204
|
archonBlueGreen?: {
|
|
1189
1205
|
servers: readonly BlueGreenServer[] | undefined;
|
|
1190
1206
|
blocked?: string;
|
|
1207
|
+
/** The stage guards that can be made now ({@link stageGuardInputs}); they go into the applied hash. */
|
|
1208
|
+
guards?: readonly string[];
|
|
1191
1209
|
};
|
|
1192
1210
|
/** Whether the host supplies a Coolify reader for the platform (default `true`). */
|
|
1193
1211
|
jobsObservable?: boolean;
|
|
@@ -1255,6 +1273,8 @@ export interface PlanPlatformOptions {
|
|
|
1255
1273
|
backupExecutorMissing?: boolean;
|
|
1256
1274
|
/** #540: why `hc` is blocked, or absent when it is not. */
|
|
1257
1275
|
hc?: string;
|
|
1276
|
+
/** Why the image registry is blocked, or absent when it is not. */
|
|
1277
|
+
registry?: string;
|
|
1258
1278
|
/**
|
|
1259
1279
|
* #540: true once `hc` is applied but the host built the platform
|
|
1260
1280
|
* executor with no heartbeats dependencies -- the heartbeat-checks step
|
|
@@ -1377,6 +1397,8 @@ export interface PlanPlatformOptions {
|
|
|
1377
1397
|
authApps?: boolean;
|
|
1378
1398
|
edge?: boolean;
|
|
1379
1399
|
files?: boolean;
|
|
1400
|
+
/** A usable, reachable Coolify server has no registry login for the current host and passwords. */
|
|
1401
|
+
registryLogins?: boolean;
|
|
1380
1402
|
backupStorage?: boolean;
|
|
1381
1403
|
backupStorageBlocked?: string;
|
|
1382
1404
|
};
|