@wtfalch/foundry 0.20.0 → 0.21.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bootstrap/declare.d.ts +13 -0
- package/dist/bootstrap/declare.js +23 -1
- package/dist/bootstrap/versions.d.ts +1 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +1 -0
- package/dist/migrations/0046_platform_hc.sql +19 -0
- package/dist/plan/deploy-health.js +5 -1
- package/dist/plan/diff.js +23 -0
- package/dist/plan/executor.d.ts +34 -0
- package/dist/plan/executor.js +181 -28
- package/dist/plan/health-check-settings.d.ts +47 -0
- package/dist/plan/health-check-settings.js +47 -0
- package/dist/plan/health-targets.d.ts +86 -0
- package/dist/plan/health-targets.js +344 -0
- package/dist/plan/index.d.ts +2 -0
- package/dist/plan/index.js +1 -0
- package/dist/plan/live.d.ts +12 -1
- package/dist/plan/plan-apply.d.ts +4 -0
- package/dist/plan/platform-diff.d.ts +124 -0
- package/dist/plan/platform-diff.js +287 -3
- package/dist/plan/platform-providers.d.ts +150 -2
- package/dist/plan/platform-providers.js +498 -42
- package/dist/plan/providers-readers.js +32 -1
- package/dist/plan/rollout.d.ts +9 -8
- package/dist/platform.d.ts +14 -1
- package/dist/platform.js +1 -0
- package/dist/store/postgres.js +8 -0
- package/dist/store/schema.d.ts +51 -0
- package/dist/store/schema.js +4 -0
- package/package.json +1 -1
- package/versions.json +2 -1
|
@@ -6,8 +6,9 @@ import { redactSecrets } from '../calls-sfu-deploy-support.js';
|
|
|
6
6
|
import { ensureCloudflaredConnector } from './cloudflared-connector.js';
|
|
7
7
|
import { waitForDeploymentFinished, waitForHealthyVersion, waitForNewDeployment, } from './deploy-health.js';
|
|
8
8
|
import { ACCESS_POLICY_NAME, getDatabaseStarted, isNotFoundError, isRunningOrStarting, } from './executor.js';
|
|
9
|
+
import { healthCheckSettings } from './health-check-settings.js';
|
|
9
10
|
import { coolifyDomains, filesShareHostname } from './hostnames.js';
|
|
10
|
-
import { AUTH_ZITADEL_RUN_OPTIONS, JOBS_SCHEDULER_KEY, PLATFORM_AI_APPLIED_GITREF_SCOPE_KEY, PLATFORM_AI_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_AI_DEPLOY_PENDING_SCOPE_KEY, PLATFORM_AI_INTERNAL_URL_SCOPE_KEY, PLATFORM_AI_KEY, PLATFORM_AI_MANAGEMENT_KEY_CONFLICT_SCOPE_KEY, PLATFORM_AI_MANAGEMENT_KEY_SCOPE_KEY, PLATFORM_AI_MANAGEMENT_KEY_UNDELIVERED_SCOPE_KEY, PLATFORM_AI_MIGRATE_BLOCKED_SCOPE_KEY, PLATFORM_AI_ORGANISATION_SCOPE_KEY, PLATFORM_AI_SIGNING_KEY_SET_SCOPE_KEY, PLATFORM_AI_SIGNING_KEY_UNDELIVERED_SCOPE_KEY, PLATFORM_ARCHON_APPLIED_BROKER_SCOPE_KEY, PLATFORM_ARCHON_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_ARCHON_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_ARCHON_APPLIED_OPERATOR_SCOPE_KEY, PLATFORM_ARCHON_APPLIED_ROLLING_UPDATE_SCOPE_KEY, PLATFORM_ARCHON_CONNECTOR_APP_SCOPE_KEY, PLATFORM_ARCHON_COOLIFY_READ_TOKEN_REF, PLATFORM_ARCHON_ENVS_DEPLOYED_SCOPE_KEY, PLATFORM_ARCHON_GATE_CLOSED_SCOPE_KEY, PLATFORM_ARCHON_INTEGRATIONS_KEY_SYNCED_SCOPE_KEY, PLATFORM_ARCHON_KEY, PLATFORM_ARCHON_MASTER_CLOUDFLARE_TOKEN_REF, PLATFORM_ARCHON_MASTER_COOLIFY_TOKEN_REF, PLATFORM_ARCHON_MASTER_COOLIFY_URL_REF, PLATFORM_AUTH_ADMIN_UNDELIVERED_SCOPE_KEY, PLATFORM_AUTH_ADMIN_USER_SCOPE_KEY, PLATFORM_AUTH_BROKER_DEPLOYED_SCOPE_KEY, PLATFORM_AUTH_BROKER_SCOPE_KEY, PLATFORM_AUTH_CONNECTOR_APP_SCOPE_KEY, PLATFORM_AUTH_INSTANCE_SCOPE_KEY, PLATFORM_AUTH_INSTANCE_UNDELIVERED_SCOPE_KEY, PLATFORM_AUTH_KEY, PLATFORM_AUTH_LOGIN_CLIENT_UNDELIVERED_SCOPE_KEY, PLATFORM_AUTH_LOGIN_CLIENT_USER_SCOPE_KEY, PLATFORM_AUTH_OWNER_INVITE_SCOPE_KEY, PLATFORM_AUTH_SMTP_SCOPE_KEY, PLATFORM_AUTH_ZITADEL_APPLIED_IMAGE_TAG_SCOPE_KEY, PLATFORM_AUTH_ZITADEL_RUN_OPTIONS_SCOPE_KEY, PLATFORM_BACKUP_DATABASE_SCOPE_KEYS, PLATFORM_BACKUP_SERVICE_KEYS, PLATFORM_EDGE_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_EDGE_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_EDGE_CLOUDFLARE_TOKEN_REF, PLATFORM_EDGE_INTERNAL_URL_SCOPE_KEY, PLATFORM_EDGE_KEY, PLATFORM_EDGE_KEY_REF, PLATFORM_EDGE_PLATFORM_KEY_SET_SCOPE_KEY, PLATFORM_EDGE_PLATFORM_KEY_UNDELIVERED_SCOPE_KEY, PLATFORM_FILES_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_FILES_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_FILES_INTERNAL_URL_SCOPE_KEY, PLATFORM_FILES_INTROSPECTION_SCOPE_KEY, PLATFORM_FILES_INTROSPECTION_UNDELIVERED_SCOPE_KEY, PLATFORM_FILES_KEY, PLATFORM_FILES_MANAGEMENT_KEY_SCOPE_KEY, PLATFORM_FILES_MANAGEMENT_UNDELIVERED_SCOPE_KEY, PLATFORM_FILES_ZITADEL_PROJECT_SCOPE_KEY, PLATFORM_INTEGRATIONS_APPLIED_GITREF_SCOPE_KEY, PLATFORM_INTEGRATIONS_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_INTEGRATIONS_KEY, PLATFORM_INTEGRATIONS_KEY_ID_SCOPE_KEY, PLATFORM_INTEGRATIONS_KEY_ROTATION_PREVIOUS_SCOPE_KEY, PLATFORM_INTEGRATIONS_KEY_ROTATION_REQUESTED_SCOPE_KEY, PLATFORM_INTEGRATIONS_KEY_ROTATION_UNDELIVERED_SCOPE_KEY, PLATFORM_INTEGRATIONS_KEY_SET_SCOPE_KEY, PLATFORM_INTEGRATIONS_KEY_UNDELIVERED_SCOPE_KEY, PLATFORM_INTEGRATIONS_RUNNER_KEYRING_DEPLOYED_SCOPE_KEY, PLATFORM_MAIL_ADMIN_KEY_SCOPE_KEY, PLATFORM_MAIL_ADMIN_UNDELIVERED_SCOPE_KEY, PLATFORM_MAIL_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_MAIL_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_MAIL_CONNECTOR_APP_SCOPE_KEY, PLATFORM_MAIL_INTERNAL_URL_SCOPE_KEY, PLATFORM_MAIL_KEY, PLATFORM_MAIL_ZITADEL_PROJECT_SCOPE_KEY, archonBootstrapOperator, parseBrokerDeployRecord, parseLoginClientRecord, platformMailMxHost, } from './platform-diff.js';
|
|
11
|
+
import { AUTH_ZITADEL_RUN_OPTIONS, JOBS_DEPLOY_PENDING_SCOPE_KEY, JOBS_SCHEDULER_KEY, PLATFORM_AI_APPLIED_GITREF_SCOPE_KEY, PLATFORM_AI_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_AI_DEPLOY_PENDING_SCOPE_KEY, PLATFORM_AI_INTERNAL_URL_SCOPE_KEY, PLATFORM_AI_KEY, PLATFORM_AI_MANAGEMENT_KEY_CONFLICT_SCOPE_KEY, PLATFORM_AI_MANAGEMENT_KEY_SCOPE_KEY, PLATFORM_AI_MANAGEMENT_KEY_UNDELIVERED_SCOPE_KEY, PLATFORM_AI_MIGRATE_BLOCKED_SCOPE_KEY, PLATFORM_AI_ORGANISATION_SCOPE_KEY, PLATFORM_AI_SIGNING_KEY_SET_SCOPE_KEY, PLATFORM_AI_SIGNING_KEY_UNDELIVERED_SCOPE_KEY, PLATFORM_ARCHON_APPLIED_BROKER_SCOPE_KEY, PLATFORM_ARCHON_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_ARCHON_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_ARCHON_APPLIED_OPERATOR_SCOPE_KEY, PLATFORM_ARCHON_APPLIED_ROLLING_UPDATE_SCOPE_KEY, PLATFORM_ARCHON_CONNECTOR_APP_SCOPE_KEY, PLATFORM_ARCHON_COOLIFY_READ_TOKEN_REF, PLATFORM_ARCHON_ENVS_DEPLOYED_SCOPE_KEY, PLATFORM_ARCHON_GATE_CLOSED_SCOPE_KEY, PLATFORM_ARCHON_INTEGRATIONS_KEY_SYNCED_SCOPE_KEY, PLATFORM_ARCHON_KEY, PLATFORM_ARCHON_MASTER_CLOUDFLARE_TOKEN_REF, PLATFORM_ARCHON_MASTER_COOLIFY_TOKEN_REF, PLATFORM_ARCHON_MASTER_COOLIFY_URL_REF, PLATFORM_AUTH_ADMIN_UNDELIVERED_SCOPE_KEY, PLATFORM_AUTH_ADMIN_USER_SCOPE_KEY, PLATFORM_AUTH_BROKER_DEPLOYED_SCOPE_KEY, PLATFORM_AUTH_BROKER_SCOPE_KEY, PLATFORM_AUTH_CONNECTOR_APP_SCOPE_KEY, PLATFORM_AUTH_INSTANCE_SCOPE_KEY, PLATFORM_AUTH_INSTANCE_UNDELIVERED_SCOPE_KEY, PLATFORM_AUTH_KEY, PLATFORM_AUTH_LOGIN_CLIENT_UNDELIVERED_SCOPE_KEY, PLATFORM_AUTH_LOGIN_CLIENT_USER_SCOPE_KEY, PLATFORM_AUTH_OWNER_INVITE_SCOPE_KEY, PLATFORM_AUTH_SMTP_SCOPE_KEY, PLATFORM_AUTH_ZITADEL_APPLIED_IMAGE_TAG_SCOPE_KEY, PLATFORM_AUTH_ZITADEL_RUN_OPTIONS_SCOPE_KEY, PLATFORM_BACKUP_DATABASE_SCOPE_KEYS, PLATFORM_BACKUP_SERVICE_KEYS, PLATFORM_EDGE_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_EDGE_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_EDGE_CLOUDFLARE_TOKEN_REF, PLATFORM_EDGE_INTERNAL_URL_SCOPE_KEY, PLATFORM_EDGE_KEY, PLATFORM_EDGE_KEY_REF, PLATFORM_EDGE_PLATFORM_KEY_SET_SCOPE_KEY, PLATFORM_EDGE_PLATFORM_KEY_UNDELIVERED_SCOPE_KEY, PLATFORM_FILES_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_FILES_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_FILES_INTERNAL_URL_SCOPE_KEY, PLATFORM_FILES_INTROSPECTION_SCOPE_KEY, PLATFORM_FILES_INTROSPECTION_UNDELIVERED_SCOPE_KEY, PLATFORM_FILES_KEY, PLATFORM_FILES_MANAGEMENT_KEY_SCOPE_KEY, PLATFORM_FILES_MANAGEMENT_UNDELIVERED_SCOPE_KEY, PLATFORM_FILES_ZITADEL_PROJECT_SCOPE_KEY, PLATFORM_HC_API_KEY_REF, PLATFORM_HC_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_HC_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_HC_KEY, PLATFORM_HEARTBEATS_APPLIED_SCOPE_KEY, PLATFORM_INTEGRATIONS_APPLIED_GITREF_SCOPE_KEY, PLATFORM_INTEGRATIONS_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_INTEGRATIONS_KEY, PLATFORM_INTEGRATIONS_KEY_ID_SCOPE_KEY, PLATFORM_INTEGRATIONS_KEY_ROTATION_PREVIOUS_SCOPE_KEY, PLATFORM_INTEGRATIONS_KEY_ROTATION_REQUESTED_SCOPE_KEY, PLATFORM_INTEGRATIONS_KEY_ROTATION_UNDELIVERED_SCOPE_KEY, PLATFORM_INTEGRATIONS_KEY_SET_SCOPE_KEY, PLATFORM_INTEGRATIONS_KEY_UNDELIVERED_SCOPE_KEY, PLATFORM_INTEGRATIONS_RUNNER_KEYRING_DEPLOYED_SCOPE_KEY, PLATFORM_MAIL_ADMIN_KEY_SCOPE_KEY, PLATFORM_MAIL_ADMIN_UNDELIVERED_SCOPE_KEY, PLATFORM_MAIL_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_MAIL_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_MAIL_CONNECTOR_APP_SCOPE_KEY, PLATFORM_MAIL_INTERNAL_URL_SCOPE_KEY, PLATFORM_MAIL_KEY, PLATFORM_MAIL_ZITADEL_PROJECT_SCOPE_KEY, PLATFORM_UPTIME_HEARTBEAT_CHECK_NAME, archonBootstrapOperator, desiredHeartbeatCheckNames, heartbeatCheckUrlRef, parseBrokerDeployRecord, parseLoginClientRecord, platformBackupHeartbeatCheckName, platformMailMxHost, } from './platform-diff.js';
|
|
11
12
|
import { applyCallsSfuPlatformStep } from './platform-sfu.js';
|
|
12
13
|
import { CreateInstanceError, createZitadelInstance, generateSystemApiKeypair, instanceWithoutBearerMessage, systemApiUsersEnv, } from './zitadel-system-api.js';
|
|
13
14
|
/** The key `runPlatformSteps` records the apex zone's id under (`system:kind:key`). */
|
|
@@ -148,6 +149,8 @@ export class FoundryPlatformStepExecutor {
|
|
|
148
149
|
edge;
|
|
149
150
|
archon;
|
|
150
151
|
backup;
|
|
152
|
+
hc;
|
|
153
|
+
heartbeats;
|
|
151
154
|
constructor(cloudflare, declaredOrgStore, options = {}) {
|
|
152
155
|
this.cloudflare = cloudflare;
|
|
153
156
|
this.declaredOrgStore = declaredOrgStore;
|
|
@@ -162,6 +165,8 @@ export class FoundryPlatformStepExecutor {
|
|
|
162
165
|
this.edge = options.edge;
|
|
163
166
|
this.archon = options.archon;
|
|
164
167
|
this.backup = options.backup;
|
|
168
|
+
this.hc = options.hc;
|
|
169
|
+
this.heartbeats = options.heartbeats;
|
|
165
170
|
}
|
|
166
171
|
async blockedSteps(platform, recordedIds = {}) {
|
|
167
172
|
const blocked = {};
|
|
@@ -286,6 +291,33 @@ export class FoundryPlatformStepExecutor {
|
|
|
286
291
|
PLATFORM_BACKUP_SERVICE_KEYS.some((service) => Boolean(recordedIds[PLATFORM_BACKUP_DATABASE_SCOPE_KEYS[service]]))) {
|
|
287
292
|
blocked.backupExecutorMissing = true;
|
|
288
293
|
}
|
|
294
|
+
if (platform.hc.enabled && !this.hc) {
|
|
295
|
+
blocked.hc =
|
|
296
|
+
'the heartbeat server is blocked, not planned: the host built the platform executor without its hc dependencies';
|
|
297
|
+
}
|
|
298
|
+
else if (platform.hc.enabled && this.hc) {
|
|
299
|
+
const hcTagBlocked = hcImageTagBlocked(platform.hc.gitRef);
|
|
300
|
+
if (hcTagBlocked)
|
|
301
|
+
blocked.hc = hcTagBlocked;
|
|
302
|
+
}
|
|
303
|
+
// Only once `hc` is actually applied -- `planHeartbeats` itself already
|
|
304
|
+
// withholds the step while `hc` is not, so a host with no heartbeats
|
|
305
|
+
// deps wired carries no standing blocked note before there is anything
|
|
306
|
+
// for it to reconcile.
|
|
307
|
+
if (!this.heartbeats && Boolean(recordedIds[`coolify:app:${PLATFORM_HC_KEY}`])) {
|
|
308
|
+
blocked.heartbeatsExecutorMissing = true;
|
|
309
|
+
}
|
|
310
|
+
else if (platform.hc.enabled && this.heartbeats) {
|
|
311
|
+
// #540 round 2: checked whether or not `hc` is recorded applied yet --
|
|
312
|
+
// `hcReady` (`planPlatform`) goes true the same round `hc` is merely
|
|
313
|
+
// *planned*, so this has to hold the heartbeats step back from the
|
|
314
|
+
// very first round, before `coolify:app:platform:hc` is ever recorded.
|
|
315
|
+
// See `PLATFORM_HC_API_KEY_REF`'s own doc comment (`platform-diff.ts`)
|
|
316
|
+
// for why nothing here can mint this key itself.
|
|
317
|
+
const apiKey = await this.heartbeats.heldKeys?.resolve?.(PLATFORM_HC_API_KEY_REF);
|
|
318
|
+
if (!apiKey)
|
|
319
|
+
blocked.heartbeatsApiKeyMissing = true;
|
|
320
|
+
}
|
|
289
321
|
return blocked;
|
|
290
322
|
}
|
|
291
323
|
/**
|
|
@@ -424,6 +456,18 @@ export class FoundryPlatformStepExecutor {
|
|
|
424
456
|
}
|
|
425
457
|
return provisionPlatformMail(context, this.mail);
|
|
426
458
|
}
|
|
459
|
+
if (action.system === 'coolify' && action.kind === 'app' && action.key === PLATFORM_HC_KEY) {
|
|
460
|
+
if (!this.hc) {
|
|
461
|
+
throw new Error('platform:hc needs FoundryPlatformStepExecutor to be constructed with its hc dependencies (#540)');
|
|
462
|
+
}
|
|
463
|
+
return provisionPlatformHc(context, this.hc);
|
|
464
|
+
}
|
|
465
|
+
if (action.system === 'heartbeats' && action.kind === 'reconcile') {
|
|
466
|
+
if (!this.heartbeats) {
|
|
467
|
+
throw new Error('heartbeats:reconcile needs FoundryPlatformStepExecutor to be constructed with its heartbeats dependencies (#540)');
|
|
468
|
+
}
|
|
469
|
+
return provisionPlatformHeartbeats(context, this.heartbeats);
|
|
470
|
+
}
|
|
427
471
|
if (action.system === 'cloudflare' && action.kind === 'zone') {
|
|
428
472
|
return this.cloudflareZone(action, context);
|
|
429
473
|
}
|
|
@@ -552,7 +596,25 @@ export class FoundryPlatformStepExecutor {
|
|
|
552
596
|
if (!db.internal_db_url) {
|
|
553
597
|
throw new Error(`platform step coolify:jobs: database ${dbId} has no internal_db_url yet; not deploying the scheduler without a database`);
|
|
554
598
|
}
|
|
555
|
-
|
|
599
|
+
// The scheduler migrates as `DATABASE_URL_OWNER` and creates its own
|
|
600
|
+
// `jobs_rt` role from `DATABASE_URL` (jobs README, "Deploying"). An app
|
|
601
|
+
// with no owner URL still holds the owner's URL in `DATABASE_URL`, so
|
|
602
|
+
// that is never reused: a runtime URL is minted, once, and kept after.
|
|
603
|
+
const hasOwnerUrl = (await coolify.listEnvs(appId)).some((e) => e.key === 'DATABASE_URL_OWNER' && !e.is_preview && e.value);
|
|
604
|
+
const runtimeUrl = await ensureAiRuntimeUrl(coolify, hasOwnerUrl ? [appId] : [], db.internal_db_url, 'platform step coolify:jobs');
|
|
605
|
+
const envs = [
|
|
606
|
+
{ key: 'DATABASE_URL_OWNER', value: db.internal_db_url },
|
|
607
|
+
{ key: 'DATABASE_URL', value: runtimeUrl },
|
|
608
|
+
];
|
|
609
|
+
// Whether the scheduler runs on envs other than the ones written below.
|
|
610
|
+
// Marked and persisted before the first write, and cleared only after the
|
|
611
|
+
// deploy, so a run that dies in between leaves the mark and the next run
|
|
612
|
+
// deploys again (`CLAUDE.md` rule 4).
|
|
613
|
+
const pending = Boolean(recorded[JOBS_DEPLOY_PENDING_SCOPE_KEY]) || (await envsDiffer(coolify, appId, envs));
|
|
614
|
+
if (pending) {
|
|
615
|
+
context.recordScopeId(JOBS_DEPLOY_PENDING_SCOPE_KEY, '1');
|
|
616
|
+
await context.persist?.();
|
|
617
|
+
}
|
|
556
618
|
// SCHEDULER_ADMIN_TOKEN gates the admin API the org registrations use
|
|
557
619
|
// (jobs README, "The admin API"); unset, every admin route answers 401.
|
|
558
620
|
// Delivered to the host before it is written to the app, so a failed
|
|
@@ -585,8 +647,12 @@ export class FoundryPlatformStepExecutor {
|
|
|
585
647
|
// a redeploy, so deploy after a create or a new token. Also when it is not
|
|
586
648
|
// running, so a deploy that failed on an earlier run is retried.
|
|
587
649
|
const running = (await coolify.getApplication(appId)).status?.startsWith('running') ?? false;
|
|
588
|
-
if (!existingApp || tokenJustSet || !running)
|
|
650
|
+
if (!existingApp || tokenJustSet || pending || !running)
|
|
589
651
|
await coolify.deploy(appId);
|
|
652
|
+
if (pending) {
|
|
653
|
+
context.recordScopeId(JOBS_DEPLOY_PENDING_SCOPE_KEY, '');
|
|
654
|
+
await context.persist?.();
|
|
655
|
+
}
|
|
590
656
|
return { resourceId: appId };
|
|
591
657
|
}
|
|
592
658
|
}
|
|
@@ -1849,9 +1915,14 @@ async function provisionPlatformMail(context, deps) {
|
|
|
1849
1915
|
await record(PLATFORM_MAIL_APP_SCOPE_KEY, appId);
|
|
1850
1916
|
// Set on every run, not only after a create -- same rule `provisionPlatformAuth` follows for its own apps.
|
|
1851
1917
|
// The image tag too: a declared release change reaches the app here.
|
|
1918
|
+
// #543: the health-check quartet too, the same reason Archon's own doc
|
|
1919
|
+
// comment gives -- without it Coolify's rolling update stops the old
|
|
1920
|
+
// container before the new one can answer `waitForHealthyVersion`'s own
|
|
1921
|
+
// `'ready'` probe below. Stalwart's own readiness route and HTTP port.
|
|
1852
1922
|
await deps.coolify.updateApplication(appId, {
|
|
1853
1923
|
custom_internal_name: PLATFORM_MAIL_INTERNAL_NAME,
|
|
1854
1924
|
docker_registry_image_tag: imageTag,
|
|
1925
|
+
...healthCheckSettings(STALWART_READY_PATH, Number(STALWART_HTTP_PORT)),
|
|
1855
1926
|
});
|
|
1856
1927
|
// #177: org steps (an org's own `coolify:mail`/`coolify:agora`) read this
|
|
1857
1928
|
// back instead of a literal, the same way files/ai's own internal
|
|
@@ -2511,10 +2582,14 @@ export const PLATFORM_ARCHON_CLIENT_ID_SCOPE_KEY = 'zitadel_platform_archon_clie
|
|
|
2511
2582
|
* by hand outside foundry. {@link archonRollingUpdateDrifted} re-reads
|
|
2512
2583
|
* these settings (and the #475 ones) at plan time, so a human turning any
|
|
2513
2584
|
* of them back off in the Coolify UI is noticed and repaired, not just a
|
|
2514
|
-
* platform that applied before this patch existed.
|
|
2515
|
-
*
|
|
2516
|
-
*
|
|
2517
|
-
*
|
|
2585
|
+
* platform that applied before this patch existed. #543 lifted the quartet
|
|
2586
|
+
* into the shared `healthCheckSettings` helper and applied it to every
|
|
2587
|
+
* other platform app with a confirmed health route (mail, edge, ai's and
|
|
2588
|
+
* files' runners) plus every company app `executor.ts` deploys; the
|
|
2589
|
+
* calls/chat Matrix stacks' Synapse and MAS stay off on purpose (neither
|
|
2590
|
+
* can answer a health check before the other is up) and `platform:auth`
|
|
2591
|
+
* and `platform:integrations` are left for a further follow-up, since
|
|
2592
|
+
* #543 did not establish a reachable url for either.
|
|
2518
2593
|
*/
|
|
2519
2594
|
async function provisionPlatformArchon(context, deps) {
|
|
2520
2595
|
const { platform } = context;
|
|
@@ -2641,10 +2716,10 @@ async function provisionPlatformArchon(context, deps) {
|
|
|
2641
2716
|
await deps.coolify.updateApplication(appId, {
|
|
2642
2717
|
custom_network_aliases: PLATFORM_ARCHON_INTERNAL_NAME,
|
|
2643
2718
|
is_consistent_container_name_enabled: false,
|
|
2644
|
-
|
|
2645
|
-
|
|
2646
|
-
|
|
2647
|
-
|
|
2719
|
+
// #543: the quartet below is the shared `healthCheckSettings` helper
|
|
2720
|
+
// every other app foundry creates now also goes through -- Archon's own
|
|
2721
|
+
// fields, unchanged in value, just no longer hand-rolled here.
|
|
2722
|
+
...healthCheckSettings(ARCHON_HEALTH_CHECK_PATH, Number(ARCHON_PORT), ARCHON_HEALTH_CHECK_START_PERIOD_SECONDS),
|
|
2648
2723
|
git_branch: gitRef,
|
|
2649
2724
|
});
|
|
2650
2725
|
// ---- sign-in client in the platform's own ZITADEL -------------------------
|
|
@@ -4116,8 +4191,13 @@ async function provisionPlatformAi(context, deps) {
|
|
|
4116
4191
|
// Every run, so a changed gitRef reaches an app that already exists.
|
|
4117
4192
|
git_branch: gitRef,
|
|
4118
4193
|
};
|
|
4119
|
-
|
|
4194
|
+
// #543: the health-check quartet too, runner only -- the worker exposes
|
|
4195
|
+
// no port at all (`ports_exposes` is never set on it, just below), so
|
|
4196
|
+
// there is nothing for Coolify's own container probe to reach.
|
|
4197
|
+
if (spec === PLATFORM_AI_RUNNER) {
|
|
4120
4198
|
patch.ports_exposes = PLATFORM_AI_PORT;
|
|
4199
|
+
Object.assign(patch, healthCheckSettings('/api/health', Number(PLATFORM_AI_PORT)));
|
|
4200
|
+
}
|
|
4121
4201
|
await coolify.updateApplication(app.appId, patch);
|
|
4122
4202
|
return app;
|
|
4123
4203
|
};
|
|
@@ -4127,7 +4207,7 @@ async function provisionPlatformAi(context, deps) {
|
|
|
4127
4207
|
// ---- env -----------------------------------------------------------------
|
|
4128
4208
|
// Both database URLs are live credentials: handed to `bulkUpsertEnvs` and
|
|
4129
4209
|
// nowhere else, never returned, recorded, logged or put in an error.
|
|
4130
|
-
const runtimeUrl = await ensureAiRuntimeUrl(coolify, [runner.appId, worker.appId, migrate.appId], ownerUrl);
|
|
4210
|
+
const runtimeUrl = await ensureAiRuntimeUrl(coolify, [runner.appId, worker.appId, migrate.appId], ownerUrl, 'platform:ai');
|
|
4131
4211
|
const migrateEnvs = [
|
|
4132
4212
|
{ key: 'DATABASE_URL_OWNER', value: ownerUrl },
|
|
4133
4213
|
{ key: 'DATABASE_URL', value: runtimeUrl },
|
|
@@ -4202,18 +4282,23 @@ async function provisionPlatformAi(context, deps) {
|
|
|
4202
4282
|
}
|
|
4203
4283
|
await coolify.raw('POST', `/applications/${migrate.appId}/stop`);
|
|
4204
4284
|
}
|
|
4205
|
-
// #252: confirmed before anything below -- the applied ref recorded
|
|
4206
|
-
// after, and the mint further down -- depends on the new version
|
|
4207
|
-
// being live. Never read the old container's healthy status as
|
|
4208
|
-
// deployment's (`CLAUDE.md`'s platform-step rule 5)
|
|
4209
|
-
//
|
|
4210
|
-
//
|
|
4211
|
-
//
|
|
4212
|
-
//
|
|
4213
|
-
//
|
|
4214
|
-
//
|
|
4285
|
+
// #252/#543: confirmed before anything below -- the applied ref recorded
|
|
4286
|
+
// just after, and the mint further down -- depends on the new version
|
|
4287
|
+
// actually being live. Never read the old container's healthy status as
|
|
4288
|
+
// the new deployment's (`CLAUDE.md`'s platform-step rule 5). Hoisted above
|
|
4289
|
+
// both waits (`record(PLATFORM_AI_INTERNAL_URL_SCOPE_KEY, ...)` below used
|
|
4290
|
+
// to be the first place this was built) so the runner's own wait can probe
|
|
4291
|
+
// it: the runner exposes `/api/health` on {@link PLATFORM_AI_PORT}
|
|
4292
|
+
// (`ports_exposes` is set on it, just above, for exactly this), so `'ready'`
|
|
4293
|
+
// closes the loophole `'deployment-only'` cannot (a deployment that
|
|
4294
|
+
// "finished" with a crash-looping container). The worker stays
|
|
4295
|
+
// `'deployment-only'`: it has no `ports_exposes` at all (see the comment on
|
|
4296
|
+
// that patch above), so there is no url this executor could ever reach to
|
|
4297
|
+
// probe it, the same reason the runner's own doc comment gives for why
|
|
4298
|
+
// nothing public is wired to either app.
|
|
4299
|
+
const internalUrl = `http://${PLATFORM_AI_RUNNER.internalName}:${PLATFORM_AI_PORT}`;
|
|
4215
4300
|
if (deployRunner) {
|
|
4216
|
-
await waitForNewDeployment(coolify, runner.appId, { kind: '
|
|
4301
|
+
await waitForNewDeployment(coolify, runner.appId, { kind: 'ready', url: `${internalUrl}/api/health` }, runtime, sleep, 'platform:ai', `the runner stays at ${recorded[PLATFORM_AI_APPLIED_GITREF_SCOPE_KEY] ?? 'no recorded ref'}`);
|
|
4217
4302
|
}
|
|
4218
4303
|
if (deployWorker) {
|
|
4219
4304
|
await waitForNewDeployment(coolify, worker.appId, { kind: 'deployment-only' }, runtime, sleep, 'platform:ai', `the worker stays at ${recorded[PLATFORM_AI_APPLIED_GITREF_SCOPE_KEY] ?? 'no recorded ref'}`);
|
|
@@ -4226,7 +4311,6 @@ async function provisionPlatformAi(context, deps) {
|
|
|
4226
4311
|
// the key is delivered.
|
|
4227
4312
|
await record(PLATFORM_AI_APPLIED_HOSTNAME_SCOPE_KEY, hostname);
|
|
4228
4313
|
await record(PLATFORM_AI_APPLIED_GITREF_SCOPE_KEY, gitRef);
|
|
4229
|
-
const internalUrl = `http://${PLATFORM_AI_RUNNER.internalName}:${PLATFORM_AI_PORT}`;
|
|
4230
4314
|
await record(PLATFORM_AI_INTERNAL_URL_SCOPE_KEY, internalUrl);
|
|
4231
4315
|
// ---- Archon's AI management key, once ------------------------------------
|
|
4232
4316
|
// #252: nothing left to wait on here -- the deployment above is already
|
|
@@ -4494,7 +4578,7 @@ async function waitForAiMigration(coolify, appId, runtime, sleep) {
|
|
|
4494
4578
|
* inside a try: Node's parse error carries the whole input, credential
|
|
4495
4579
|
* included, so it is replaced with a fixed message.
|
|
4496
4580
|
*/
|
|
4497
|
-
async function ensureAiRuntimeUrl(coolify, appIds, ownerUrl) {
|
|
4581
|
+
async function ensureAiRuntimeUrl(coolify, appIds, ownerUrl, step) {
|
|
4498
4582
|
for (const appId of appIds) {
|
|
4499
4583
|
const existing = (await coolify.listEnvs(appId)).find((e) => e.key === 'DATABASE_URL' && !e.is_preview && e.value);
|
|
4500
4584
|
if (existing)
|
|
@@ -4507,11 +4591,11 @@ async function ensureAiRuntimeUrl(coolify, appIds, ownerUrl) {
|
|
|
4507
4591
|
({ host, pathname, search } = new URL(ownerUrl));
|
|
4508
4592
|
}
|
|
4509
4593
|
catch {
|
|
4510
|
-
throw new Error(
|
|
4594
|
+
throw new Error(`${step}: the database's internal_db_url could not be parsed`);
|
|
4511
4595
|
}
|
|
4512
4596
|
const database = pathname.replace(/^\//, '');
|
|
4513
4597
|
if (!/^[A-Za-z0-9_]+$/.test(database)) {
|
|
4514
|
-
throw new Error(
|
|
4598
|
+
throw new Error(`${step}: the database's name cannot form a runtime role name (letters, digits and _ only)`);
|
|
4515
4599
|
}
|
|
4516
4600
|
const password = randomBytes(24).toString('base64url');
|
|
4517
4601
|
return `postgres://${database}_rt:${password}@${host}${pathname}${search}`;
|
|
@@ -4810,6 +4894,8 @@ const FILES_RUNNER_APP_NAME = 'files';
|
|
|
4810
4894
|
const FILES_RUNNER_INTERNAL_NAME = 'platform-files';
|
|
4811
4895
|
/** The runner's port (`Dockerfile`'s `runner` stage: `PORT=3000`, `EXPOSE 3000`). */
|
|
4812
4896
|
const FILES_PORT = '3000';
|
|
4897
|
+
/** files' own health route, the same one `healthUrls` (`executor.ts`) assumes every hosted app serves. */
|
|
4898
|
+
const FILES_HEALTH_CHECK_PATH = '/api/health';
|
|
4813
4899
|
/** The Postgres database name; files' runtime role is `<database>_rt`. */
|
|
4814
4900
|
const FILES_DB = 'files';
|
|
4815
4901
|
const FILES_RUNTIME_ROLE = `${FILES_DB}_rt`;
|
|
@@ -5002,6 +5088,12 @@ shareDomains) {
|
|
|
5002
5088
|
dockerfile_target_build: 'runner',
|
|
5003
5089
|
custom_internal_name: FILES_RUNNER_INTERNAL_NAME,
|
|
5004
5090
|
git_branch: gitRef,
|
|
5091
|
+
// #543: the health-check quartet, same reason Archon's doc comment
|
|
5092
|
+
// gives. The migrate app gets none -- it idles rather than serves
|
|
5093
|
+
// anything (`waitForMigration`/`waitForMigrateDeployment` below are its
|
|
5094
|
+
// own, different wait), so there is no health route to point Coolify's
|
|
5095
|
+
// probe at.
|
|
5096
|
+
...healthCheckSettings(FILES_HEALTH_CHECK_PATH, Number(FILES_PORT)),
|
|
5005
5097
|
// Coolify wants full URLs. Written only when the set changed, so a runner
|
|
5006
5098
|
// nobody gave a share host keeps no `domains`.
|
|
5007
5099
|
...filesShareDomainsPatch(recorded, shareDomains),
|
|
@@ -5146,15 +5238,18 @@ shareDomains) {
|
|
|
5146
5238
|
// mark a deployment that actually migrated as `failed`.
|
|
5147
5239
|
await waitForMigrateDeployment(deps.coolify, migrateApp.appId, queuedDeploymentUuid(migrateDeployed) ?? 'untracked', 'platform:files', 'files-migrate', runtime.now ?? Date.now, runtime.migrateTimeoutMs ?? 900_000, runtime.migrateIntervalMs ?? 10_000, sleep);
|
|
5148
5240
|
await deps.coolify.raw('POST', `/applications/${migrateApp.appId}/stop`);
|
|
5149
|
-
// #252: `
|
|
5150
|
-
//
|
|
5151
|
-
//
|
|
5152
|
-
//
|
|
5153
|
-
//
|
|
5154
|
-
//
|
|
5155
|
-
//
|
|
5156
|
-
|
|
5157
|
-
|
|
5241
|
+
// #252/#543: no public `domains`, tunnel or DNS record is wired to the
|
|
5242
|
+
// runner (its own doc comment: "Nothing public is wired"), but it does
|
|
5243
|
+
// answer {@link FILES_HEALTH_CHECK_PATH} on its own Coolify-network
|
|
5244
|
+
// address -- reachable from this executor the same way `provisionPlatformAi`'s
|
|
5245
|
+
// runner now is. `'ready'` closes the loophole `'deployment-only'` alone
|
|
5246
|
+
// cannot (a deployment Coolify calls "finished" with a crash-looping
|
|
5247
|
+
// container); `coolify.deployAndWait` reaching Coolify's own deployment
|
|
5248
|
+
// record first is still what rules out the old container staying healthy
|
|
5249
|
+
// through a failed deploy.
|
|
5250
|
+
const internalUrl = `http://${FILES_RUNNER_INTERNAL_NAME}:${FILES_PORT}`;
|
|
5251
|
+
await waitForNewDeployment(deps.coolify, runnerApp.appId, { kind: 'ready', url: `${internalUrl}${FILES_HEALTH_CHECK_PATH}` }, runtime, sleep, 'platform:files', `the runner stays at ${recorded[PLATFORM_FILES_APPLIED_GIT_REF_SCOPE_KEY] ?? 'no recorded ref'}`);
|
|
5252
|
+
await record(PLATFORM_FILES_INTERNAL_URL_SCOPE_KEY, internalUrl);
|
|
5158
5253
|
// Last: a run that failed anywhere above is planned again.
|
|
5159
5254
|
context.recordScopeId(PLATFORM_FILES_APPLIED_HOSTNAME_SCOPE_KEY, platform.files.hostname);
|
|
5160
5255
|
await record(PLATFORM_FILES_APPLIED_GIT_REF_SCOPE_KEY, gitRef);
|
|
@@ -5359,10 +5454,13 @@ async function provisionPlatformEdge(context, deps) {
|
|
|
5359
5454
|
// Set on every run, not only after a create: a throw right after the
|
|
5360
5455
|
// create must not leave the app on the wrong target for good, and a
|
|
5361
5456
|
// changed `gitRef` has to reach an app made on an earlier one.
|
|
5457
|
+
// #543: the health-check quartet too -- edge's own `/api/health`, on the
|
|
5458
|
+
// port the `waitForNewDeployment` call below already probes internally.
|
|
5362
5459
|
await deps.coolify.updateApplication(appId, {
|
|
5363
5460
|
dockerfile_target_build: 'runner',
|
|
5364
5461
|
custom_internal_name: EDGE_RUNNER_INTERNAL_NAME,
|
|
5365
5462
|
git_branch: gitRef,
|
|
5463
|
+
...healthCheckSettings('/api/health', Number(EDGE_PORT)),
|
|
5366
5464
|
});
|
|
5367
5465
|
const runtimeUrl = await ensureEdgeRuntimeUrl(deps.coolify, appId, db.internal_db_url);
|
|
5368
5466
|
// ---- the platform key, once ----------------------------------------------
|
|
@@ -5457,6 +5555,224 @@ async function provisionPlatformEdge(context, deps) {
|
|
|
5457
5555
|
await record(PLATFORM_EDGE_APPLIED_GIT_REF_SCOPE_KEY, gitRef);
|
|
5458
5556
|
return { resourceId: appId };
|
|
5459
5557
|
}
|
|
5558
|
+
/** `healthchecks/healthchecks` on Docker Hub -- #540's own "third-party, not this estate's own repo" image. */
|
|
5559
|
+
const HC_IMAGE = 'healthchecks/healthchecks';
|
|
5560
|
+
const PLATFORM_HC_PROJECT = 'platform';
|
|
5561
|
+
const PLATFORM_HC_DB_NAME = 'platform-hc-db';
|
|
5562
|
+
const PLATFORM_HC_APP_NAME = 'platform-hc';
|
|
5563
|
+
/** The app's fixed `custom_internal_name`, the `EDGE_RUNNER_INTERNAL_NAME` convention. */
|
|
5564
|
+
const HC_INTERNAL_NAME = 'platform-hc';
|
|
5565
|
+
/** Healthchecks' own default uWSGI/gunicorn port inside the container. */
|
|
5566
|
+
const HC_PORT = '8000';
|
|
5567
|
+
/**
|
|
5568
|
+
* Healthchecks ships no dedicated readiness route (confirmed against its
|
|
5569
|
+
* `hc/front/urls.py`: the closest is `checks/<uuid>/status/`, which needs a
|
|
5570
|
+
* check's own id) -- `/` (`views.index`) is the documented fallback, a
|
|
5571
|
+
* public page for an anonymous visitor.
|
|
5572
|
+
*/
|
|
5573
|
+
const HC_HEALTH_CHECK_PATH = '/';
|
|
5574
|
+
const PLATFORM_HC_DB = 'hc';
|
|
5575
|
+
const PLATFORM_HC_DB_USER = 'hc';
|
|
5576
|
+
const HC_TUNNEL_NAME = 'platform-hc-tunnel';
|
|
5577
|
+
/** #197: the one Coolify app that runs `cloudflared` for {@link HC_TUNNEL_NAME}. */
|
|
5578
|
+
const HC_CLOUDFLARED_APP_NAME = 'platform-hc-cloudflared';
|
|
5579
|
+
const PLATFORM_HC_PROJECT_SCOPE_KEY = 'coolify_project_platform_hc';
|
|
5580
|
+
const PLATFORM_HC_DB_SCOPE_KEY = 'coolify_platform_hc_database';
|
|
5581
|
+
const PLATFORM_HC_APP_SCOPE_KEY = 'coolify_platform_hc_app';
|
|
5582
|
+
const PLATFORM_HC_TUNNEL_SCOPE_KEY = 'cloudflare_platform_hc_tunnel';
|
|
5583
|
+
const PLATFORM_HC_CONNECTOR_APP_SCOPE_KEY = 'coolify_platform_hc_connector_app';
|
|
5584
|
+
const PLATFORM_HC_INTERNAL_URL_SCOPE_KEY = 'platform_hc_internal_url';
|
|
5585
|
+
/**
|
|
5586
|
+
* #540 round 2: a thin wrapper over the estate-wide {@link floatingTagBlocked},
|
|
5587
|
+
* the {@link mailImageTagBlocked} pattern for hc's own image tag -- #540
|
|
5588
|
+
* shipped with no guard at all here, and `DEFAULT_HC_IMAGE_TAG` floating at
|
|
5589
|
+
* `'latest'` by default was exactly what `floatingTagBlocked`'s own
|
|
5590
|
+
* `FLOATING_TAGS` set exists to catch.
|
|
5591
|
+
*/
|
|
5592
|
+
function hcImageTagBlocked(tag) {
|
|
5593
|
+
const reason = floatingTagBlocked('Healthchecks', tag, VERSIONS.hc);
|
|
5594
|
+
return reason ? `the heartbeat server is blocked, not planned: ${reason} as the hc gitRef` : null;
|
|
5595
|
+
}
|
|
5596
|
+
/**
|
|
5597
|
+
* #540: the platform's own heartbeat server (self-hosted Healthchecks) --
|
|
5598
|
+
* a Coolify Postgres plus one `healthchecks/healthchecks` Docker-image app,
|
|
5599
|
+
* reached through edge's tunnel and DNS at `platform.hc.hostname`. No
|
|
5600
|
+
* ZITADEL client, no sign-in: `hc` is an internal operational tool, read by
|
|
5601
|
+
* nobody outside foundry and whoever holds its checks' ping URLs.
|
|
5602
|
+
*
|
|
5603
|
+
* `REGISTRATION_OPEN=False` is set alongside the rest of the env below --
|
|
5604
|
+
* Healthchecks defaults that setting to `true` (confirmed against its own
|
|
5605
|
+
* `hc/settings.py`, 2026-10), and `hostname` is a public, Cloudflare-proxied
|
|
5606
|
+
* DNS record the moment this step's tunnel and DNS write land, so leaving
|
|
5607
|
+
* it unset would let anyone who finds the hostname sign themselves up.
|
|
5608
|
+
*
|
|
5609
|
+
* Idempotent the {@link provisionPlatformEdge} way: no reader reports this
|
|
5610
|
+
* Coolify app, so the planner's own recorded-id check
|
|
5611
|
+
* (`platform-diff.ts`'s `planHc`) is what stops this running on every
|
|
5612
|
+
* apply, not a live comparison in here.
|
|
5613
|
+
*/
|
|
5614
|
+
async function provisionPlatformHc(context, deps) {
|
|
5615
|
+
const { platform } = context;
|
|
5616
|
+
const hostname = platform.hc.hostname;
|
|
5617
|
+
const gitRef = platform.hc.gitRef;
|
|
5618
|
+
const tagBlocked = hcImageTagBlocked(gitRef);
|
|
5619
|
+
if (tagBlocked)
|
|
5620
|
+
throw new Error(tagBlocked);
|
|
5621
|
+
const recorded = context.recordedIds();
|
|
5622
|
+
const runtime = deps.runtime ?? {};
|
|
5623
|
+
const sleep = runtime.sleep ?? defaultSleep;
|
|
5624
|
+
const record = async (key, id) => {
|
|
5625
|
+
context.recordScopeId(key, id);
|
|
5626
|
+
await context.persist?.();
|
|
5627
|
+
};
|
|
5628
|
+
let projectUuid = recorded[PLATFORM_HC_PROJECT_SCOPE_KEY];
|
|
5629
|
+
if (!projectUuid) {
|
|
5630
|
+
const found = await deps.coolify.findProjectByName(PLATFORM_HC_PROJECT);
|
|
5631
|
+
projectUuid = found ? found.uuid : (await deps.coolify.createProject(PLATFORM_HC_PROJECT)).uuid;
|
|
5632
|
+
await record(PLATFORM_HC_PROJECT_SCOPE_KEY, projectUuid);
|
|
5633
|
+
}
|
|
5634
|
+
const { environment } = await deps.coolify.findOrCreateEnvironment(projectUuid, 'production');
|
|
5635
|
+
const server = await resolvePlatformServer(deps.coolify);
|
|
5636
|
+
// ---- database ------------------------------------------------------------
|
|
5637
|
+
let dbId = recorded[PLATFORM_HC_DB_SCOPE_KEY];
|
|
5638
|
+
if (!dbId) {
|
|
5639
|
+
const found = (await deps.coolify.listDatabases()).find((db) => db.name === PLATFORM_HC_DB_NAME && db.environment_id === environment.id);
|
|
5640
|
+
if (found) {
|
|
5641
|
+
dbId = found.uuid;
|
|
5642
|
+
}
|
|
5643
|
+
else {
|
|
5644
|
+
const created = await deps.coolify.createPostgresDatabase({
|
|
5645
|
+
project_uuid: projectUuid,
|
|
5646
|
+
server_uuid: server.uuid,
|
|
5647
|
+
environment_name: environment.name,
|
|
5648
|
+
environment_uuid: environment.uuid,
|
|
5649
|
+
name: PLATFORM_HC_DB_NAME,
|
|
5650
|
+
postgres_db: PLATFORM_HC_DB,
|
|
5651
|
+
postgres_user: PLATFORM_HC_DB_USER,
|
|
5652
|
+
});
|
|
5653
|
+
dbId = created.uuid;
|
|
5654
|
+
}
|
|
5655
|
+
await record(PLATFORM_HC_DB_SCOPE_KEY, dbId);
|
|
5656
|
+
}
|
|
5657
|
+
const db = await getDatabaseStarted(deps.coolify, dbId);
|
|
5658
|
+
if (!db.internal_db_url) {
|
|
5659
|
+
throw new Error('coolify:app for the platform (hc): the database has no internal_db_url yet -- retry this apply once Coolify reports one');
|
|
5660
|
+
}
|
|
5661
|
+
// ---- the app --------------------------------------------------------------
|
|
5662
|
+
const app = await findOrCreatePlatformApp(deps.coolify, PLATFORM_HC_APP_NAME, environment, async () => {
|
|
5663
|
+
return deps.coolify.createDockerImageApplication({
|
|
5664
|
+
project_uuid: projectUuid,
|
|
5665
|
+
server_uuid: server.uuid,
|
|
5666
|
+
environment_name: environment.name,
|
|
5667
|
+
environment_uuid: environment.uuid,
|
|
5668
|
+
docker_registry_image_name: HC_IMAGE,
|
|
5669
|
+
docker_registry_image_tag: gitRef,
|
|
5670
|
+
ports_exposes: HC_PORT,
|
|
5671
|
+
name: PLATFORM_HC_APP_NAME,
|
|
5672
|
+
description: `Healthchecks for the platform's own heartbeats (#540, ${HC_IMAGE}:${gitRef})`,
|
|
5673
|
+
instant_deploy: false,
|
|
5674
|
+
});
|
|
5675
|
+
});
|
|
5676
|
+
const appId = app.appId;
|
|
5677
|
+
await record(PLATFORM_HC_APP_SCOPE_KEY, appId);
|
|
5678
|
+
// Set on every run, not only after a create -- `provisionPlatformAuth`'s
|
|
5679
|
+
// own rule. The image tag too, so a declared release change reaches the app.
|
|
5680
|
+
await deps.coolify.updateApplication(appId, {
|
|
5681
|
+
custom_internal_name: HC_INTERNAL_NAME,
|
|
5682
|
+
docker_registry_image_tag: gitRef,
|
|
5683
|
+
...healthCheckSettings(HC_HEALTH_CHECK_PATH, Number(HC_PORT)),
|
|
5684
|
+
});
|
|
5685
|
+
// ---- env, parsed off the database's own owner credentials -----------------
|
|
5686
|
+
// Healthchecks takes discrete DB_* vars, not a DATABASE_URL (its own
|
|
5687
|
+
// `hc/settings.py`) -- no separate runtime role the way edge/files mint
|
|
5688
|
+
// one: Healthchecks runs migrations and serves from the same one process,
|
|
5689
|
+
// so there is no second, more-restricted role to split off here.
|
|
5690
|
+
let dbHost;
|
|
5691
|
+
let dbPort;
|
|
5692
|
+
let dbName;
|
|
5693
|
+
let dbUser;
|
|
5694
|
+
let dbPassword;
|
|
5695
|
+
try {
|
|
5696
|
+
const parsed = new URL(db.internal_db_url);
|
|
5697
|
+
dbHost = parsed.hostname;
|
|
5698
|
+
dbPort = parsed.port || '5432';
|
|
5699
|
+
dbName = parsed.pathname.replace(/^\//, '');
|
|
5700
|
+
dbUser = decodeURIComponent(parsed.username);
|
|
5701
|
+
dbPassword = decodeURIComponent(parsed.password);
|
|
5702
|
+
}
|
|
5703
|
+
catch {
|
|
5704
|
+
throw new Error("coolify:app for the platform (hc): the database's internal_db_url could not be parsed into DB_* env vars");
|
|
5705
|
+
}
|
|
5706
|
+
const secretKey = await ensureGeneratedEnv(deps.coolify, appId, 'SECRET_KEY', () => randomBytes(32).toString('base64url'));
|
|
5707
|
+
const envs = [
|
|
5708
|
+
{ key: 'DB', value: 'postgres' },
|
|
5709
|
+
{ key: 'DB_HOST', value: dbHost },
|
|
5710
|
+
{ key: 'DB_PORT', value: dbPort },
|
|
5711
|
+
{ key: 'DB_NAME', value: dbName },
|
|
5712
|
+
{ key: 'DB_USER', value: dbUser },
|
|
5713
|
+
{ key: 'DB_PASSWORD', value: dbPassword },
|
|
5714
|
+
{ key: 'SITE_ROOT', value: `https://${hostname}` },
|
|
5715
|
+
{ key: 'ALLOWED_HOSTS', value: hostname },
|
|
5716
|
+
// Healthchecks defaults this to `true` -- see this function's own doc
|
|
5717
|
+
// comment. `hostname` is public and Cloudflare-proxied, so sign-up stays
|
|
5718
|
+
// closed until a real invite mechanism exists.
|
|
5719
|
+
{ key: 'REGISTRATION_OPEN', value: 'False' },
|
|
5720
|
+
];
|
|
5721
|
+
if (secretKey)
|
|
5722
|
+
envs.push({ key: 'SECRET_KEY', value: secretKey });
|
|
5723
|
+
const envsChanged = await envsDiffer(deps.coolify, appId, envs);
|
|
5724
|
+
await deps.coolify.bulkUpsertEnvs(appId, envs);
|
|
5725
|
+
const declaredChanged = recorded[PLATFORM_HC_APPLIED_HOSTNAME_SCOPE_KEY] !== hostname ||
|
|
5726
|
+
recorded[PLATFORM_HC_APPLIED_GIT_REF_SCOPE_KEY] !== gitRef;
|
|
5727
|
+
const internalUrl = `http://${HC_INTERNAL_NAME}:${HC_PORT}`;
|
|
5728
|
+
if (app.justCreated || !app.running || envsChanged || declaredChanged) {
|
|
5729
|
+
await waitForNewDeployment(deps.coolify, appId, { kind: 'ready', url: `${internalUrl}${HC_HEALTH_CHECK_PATH}` }, runtime, sleep, 'platform:hc', `hc stays at ${recorded[PLATFORM_HC_APPLIED_GIT_REF_SCOPE_KEY] ?? 'no recorded ref'}`);
|
|
5730
|
+
}
|
|
5731
|
+
await record(PLATFORM_HC_INTERNAL_URL_SCOPE_KEY, internalUrl);
|
|
5732
|
+
// ---- tunnel and DNS ---------------------------------------------------------
|
|
5733
|
+
const tunnels = await platformTunnels(deps, context.recordedIds(), 'platform:hc');
|
|
5734
|
+
const existingTunnel = await tunnels.findTunnelByName(HC_TUNNEL_NAME);
|
|
5735
|
+
const tunnel = existingTunnel ?? (await tunnels.createTunnel(HC_TUNNEL_NAME));
|
|
5736
|
+
await record(PLATFORM_HC_TUNNEL_SCOPE_KEY, tunnel.id);
|
|
5737
|
+
await tunnels.putTunnelConfig(tunnel.id, {
|
|
5738
|
+
ingress: [{ hostname, service: internalUrl }, { service: 'http_status:404' }],
|
|
5739
|
+
});
|
|
5740
|
+
// #197: without a connector, `hostname` answers with a Cloudflare error.
|
|
5741
|
+
await ensureCloudflaredConnector({
|
|
5742
|
+
coolify: deps.coolify,
|
|
5743
|
+
tunnels,
|
|
5744
|
+
tunnelId: tunnel.id,
|
|
5745
|
+
appName: HC_CLOUDFLARED_APP_NAME,
|
|
5746
|
+
projectUuid,
|
|
5747
|
+
environment,
|
|
5748
|
+
resolveServer: async () => server,
|
|
5749
|
+
recordedConnectorAppId: recorded[PLATFORM_HC_CONNECTOR_APP_SCOPE_KEY] ?? null,
|
|
5750
|
+
recordAppId: (connectorAppId) => record(PLATFORM_HC_CONNECTOR_APP_SCOPE_KEY, connectorAppId),
|
|
5751
|
+
});
|
|
5752
|
+
const dns = await platformDns(deps, context.recordedIds(), 'platform:hc');
|
|
5753
|
+
const dnsTarget = `${tunnel.id}.cfargotunnel.com`;
|
|
5754
|
+
const [dnsRecord] = await dns.findDnsRecord(platform.apex, hostname, 'CNAME');
|
|
5755
|
+
if (!dnsRecord) {
|
|
5756
|
+
await dns.createDnsRecord(platform.apex, {
|
|
5757
|
+
type: 'CNAME',
|
|
5758
|
+
name: hostname,
|
|
5759
|
+
content: dnsTarget,
|
|
5760
|
+
proxied: true,
|
|
5761
|
+
});
|
|
5762
|
+
}
|
|
5763
|
+
else if (dnsRecord.content !== dnsTarget) {
|
|
5764
|
+
await dns.updateDnsRecord(platform.apex, dnsRecord.id, {
|
|
5765
|
+
type: 'CNAME',
|
|
5766
|
+
name: hostname,
|
|
5767
|
+
content: dnsTarget,
|
|
5768
|
+
proxied: true,
|
|
5769
|
+
});
|
|
5770
|
+
}
|
|
5771
|
+
// Last: a run that failed anywhere above is planned again.
|
|
5772
|
+
context.recordScopeId(PLATFORM_HC_APPLIED_HOSTNAME_SCOPE_KEY, hostname);
|
|
5773
|
+
await record(PLATFORM_HC_APPLIED_GIT_REF_SCOPE_KEY, gitRef);
|
|
5774
|
+
return { resourceId: appId };
|
|
5775
|
+
}
|
|
5460
5776
|
/**
|
|
5461
5777
|
* The runtime `DATABASE_URL` the names app gets: the value already set on
|
|
5462
5778
|
* it, else a new `names_rt` credential on the database's host, the
|
|
@@ -6229,6 +6545,41 @@ const PLATFORM_BACKUP_S3_STORAGE_FINGERPRINT_SCOPE_KEY = 'coolify_platform_backu
|
|
|
6229
6545
|
const PLATFORM_BACKUP_S3_STORAGE_PENDING_SCOPE_KEY = 'coolify_platform_backup_s3_storage_pending';
|
|
6230
6546
|
/** Coolify's own name for "once a day" -- `createDatabaseBackup`'s `frequency`. */
|
|
6231
6547
|
const PLATFORM_BACKUP_DEFAULT_FREQUENCY = 'daily';
|
|
6548
|
+
/**
|
|
6549
|
+
* The database a service's schedule must name. A schedule that names none
|
|
6550
|
+
* dumps the Coolify resource's first database, which for live Archon is the
|
|
6551
|
+
* abandoned `archon`, not the one it runs on (see {@link ARCHON_POSTGRES_DB}).
|
|
6552
|
+
* Every other service runs on its resource's first database and needs no entry.
|
|
6553
|
+
*/
|
|
6554
|
+
const PLATFORM_BACKUP_DATABASE_NAMES = {
|
|
6555
|
+
archon: ARCHON_POSTGRES_DB,
|
|
6556
|
+
};
|
|
6557
|
+
/**
|
|
6558
|
+
* The one schedule both the reader and {@link provisionPlatformBackup} treat
|
|
6559
|
+
* as a service's backup, so the two never disagree: an enabled, `save_s3`
|
|
6560
|
+
* schedule that dumps the right database, else the first enabled, `save_s3`
|
|
6561
|
+
* one with `named: false` (it needs the repair). A service with no entry in
|
|
6562
|
+
* {@link PLATFORM_BACKUP_DATABASE_NAMES} is always `named`. The list
|
|
6563
|
+
* response is not confirmed to carry `databases_to_backup`, so a schedule
|
|
6564
|
+
* without the field is read off its detail route, the same fallback
|
|
6565
|
+
* {@link repointPlatformBackupSchedules} uses for `s3_storage_uuid`.
|
|
6566
|
+
*/
|
|
6567
|
+
async function findPlatformBackupSchedule(coolify, service, databaseId) {
|
|
6568
|
+
const schedules = (await coolify.listDatabaseBackups(databaseId)).filter((backup) => backup.enabled && backup.save_s3);
|
|
6569
|
+
const database = PLATFORM_BACKUP_DATABASE_NAMES[service];
|
|
6570
|
+
if (!database)
|
|
6571
|
+
return { schedule: schedules[0], named: true };
|
|
6572
|
+
for (const schedule of schedules) {
|
|
6573
|
+
let databases = schedule.databases_to_backup;
|
|
6574
|
+
if (databases === undefined) {
|
|
6575
|
+
const detail = (await coolify.raw('GET', `/databases/${databaseId}/backups/${schedule.uuid}`));
|
|
6576
|
+
databases = detail?.databases_to_backup;
|
|
6577
|
+
}
|
|
6578
|
+
if (databases === database)
|
|
6579
|
+
return { schedule, named: true };
|
|
6580
|
+
}
|
|
6581
|
+
return { schedule: schedules[0], named: false };
|
|
6582
|
+
}
|
|
6232
6583
|
/**
|
|
6233
6584
|
* Trims a trailing slash and lowercases the host, so a Coolify-normalised
|
|
6234
6585
|
* endpoint (or one a person typed with a trailing slash) is never read as a
|
|
@@ -6590,17 +6941,109 @@ async function provisionPlatformBackup(action, context, deps) {
|
|
|
6590
6941
|
const databaseId = action.desired.databaseId;
|
|
6591
6942
|
const recorded = context.recordedIds();
|
|
6592
6943
|
const storageUuid = await ensurePlatformBackupStorage(coolify, r2, recorded, context);
|
|
6593
|
-
const
|
|
6594
|
-
|
|
6595
|
-
|
|
6944
|
+
const service = action.key;
|
|
6945
|
+
const { schedule, named } = await findPlatformBackupSchedule(coolify, service, databaseId);
|
|
6946
|
+
const scheduleUuid = schedule?.uuid ??
|
|
6947
|
+
(await coolify.createDatabaseBackup(databaseId, {
|
|
6596
6948
|
frequency: deps.frequency ?? PLATFORM_BACKUP_DEFAULT_FREQUENCY,
|
|
6597
6949
|
enabled: true,
|
|
6598
6950
|
save_s3: true,
|
|
6599
6951
|
s3_storage_uuid: storageUuid,
|
|
6952
|
+
})).uuid;
|
|
6953
|
+
// `CoolifyPort.createDatabaseBackup` takes no database name, so a new
|
|
6954
|
+
// schedule is named here too, by the same PATCH that repairs an existing
|
|
6955
|
+
// one dumping the wrong database.
|
|
6956
|
+
if (!named) {
|
|
6957
|
+
await coolify.raw('PATCH', `/databases/${databaseId}/backups/${scheduleUuid}`, {
|
|
6958
|
+
databases_to_backup: PLATFORM_BACKUP_DATABASE_NAMES[service],
|
|
6600
6959
|
});
|
|
6601
6960
|
}
|
|
6602
6961
|
return undefined;
|
|
6603
6962
|
}
|
|
6963
|
+
/**
|
|
6964
|
+
* #540: reconciles the live Healthchecks checks against the ones foundry
|
|
6965
|
+
* wants -- `desiredHeartbeatCheckNames` (`platform-diff.ts`): the uptime
|
|
6966
|
+
* Worker's own, plus one per platform database backup that already has a
|
|
6967
|
+
* recorded database id. Idempotent twice over: `planHeartbeats` only plans
|
|
6968
|
+
* this step when the declared desired set has changed since the last
|
|
6969
|
+
* successful reconcile (its own doc comment says why), and this function
|
|
6970
|
+
* itself lists live checks first, so a name already live is never created
|
|
6971
|
+
* twice -- a second run, even one somehow forced, creates none (it still
|
|
6972
|
+
* re-saves the address every time; see `PlatformHeartbeatsDeps.put`'s own
|
|
6973
|
+
* doc comment for why that is the round-2 fix, not a regression).
|
|
6974
|
+
*
|
|
6975
|
+
* Deletes only a live check whose name matches this step's own naming
|
|
6976
|
+
* scheme (`PLATFORM_UPTIME_HEARTBEAT_CHECK_NAME` or
|
|
6977
|
+
* `platformBackupHeartbeatCheckName`), is no longer desired, AND is held
|
|
6978
|
+
* under that exact name's own ref -- #540 round 2 added the third
|
|
6979
|
+
* condition (`PlatformHeartbeatsDeps.resolve`'s own doc comment says why):
|
|
6980
|
+
* a name match alone is never enough to delete, so this reconcile can
|
|
6981
|
+
* never destroy a check it did not itself create, whether that check is a
|
|
6982
|
+
* stranger's or simply one this step never got to create in the first
|
|
6983
|
+
* place (no `resolve` wired at all reads as "prove nothing", the safe
|
|
6984
|
+
* direction).
|
|
6985
|
+
*
|
|
6986
|
+
* `uptime-worker`'s own ping URL is recorded under
|
|
6987
|
+
* `PLATFORM_UPTIME_HEARTBEAT_URL_REF`, but nothing pings it yet: delivering
|
|
6988
|
+
* it to a running uptime Worker is foundry#542's own job, and that issue is
|
|
6989
|
+
* blocked today on a missing KV/cron/secret-binding capability in
|
|
6990
|
+
* `@wtfalch/edge` (wtfalch/edge#26) -- a known gap, not an oversight here.
|
|
6991
|
+
* Each backup check's ping URL is recorded the same way under
|
|
6992
|
+
* `platformBackupHeartbeatUrlRef(service)`, and nothing pings those either:
|
|
6993
|
+
* Coolify's own `createDatabaseBackup` API (`executor.ts`'s
|
|
6994
|
+
* `CoolifyPort.createDatabaseBackup`) has no notification/webhook field at
|
|
6995
|
+
* all, so there is no way for Coolify's backup scheduler to call this check
|
|
6996
|
+
* when a backup actually runs. Both gaps are named here, not worked around
|
|
6997
|
+
* -- no polling, no wrapping Coolify's scheduler. Every check this function
|
|
6998
|
+
* creates is paused on creation for exactly that reason (see
|
|
6999
|
+
* `HeartbeatsPort.pauseCheck`'s own doc comment): until one of those two
|
|
7000
|
+
* gaps closes, a newly created check would otherwise read "down" within its
|
|
7001
|
+
* own grace period, with nothing actually wrong.
|
|
7002
|
+
*/
|
|
7003
|
+
async function provisionPlatformHeartbeats(context, deps) {
|
|
7004
|
+
const recorded = context.recordedIds();
|
|
7005
|
+
const desiredNames = desiredHeartbeatCheckNames(recorded);
|
|
7006
|
+
const desiredSet = new Set(desiredNames);
|
|
7007
|
+
const live = await deps.heartbeats.listChecks();
|
|
7008
|
+
const liveByName = new Map(live.map((check) => [check.name, check]));
|
|
7009
|
+
for (const name of desiredNames) {
|
|
7010
|
+
const existing = liveByName.get(name);
|
|
7011
|
+
const put = deps.heldKeys?.put;
|
|
7012
|
+
if (!put) {
|
|
7013
|
+
throw new Error(`heartbeats:reconcile for the platform: ${name}'s ping address needs saving, but the host built the platform executor without a way to hold it (heldKeys.put, #540)`);
|
|
7014
|
+
}
|
|
7015
|
+
if (existing) {
|
|
7016
|
+
// Already live. Re-save its ping address on every run, whether or not
|
|
7017
|
+
// it looks already correct -- see `PlatformHeartbeatsDeps.put`'s own
|
|
7018
|
+
// doc comment for why this, not a one-time save at create, is the fix.
|
|
7019
|
+
await put(heartbeatCheckUrlRef(name), existing.pingUrl);
|
|
7020
|
+
continue;
|
|
7021
|
+
}
|
|
7022
|
+
const created = await deps.heartbeats.createCheck(name);
|
|
7023
|
+
await put(heartbeatCheckUrlRef(name), created.pingUrl);
|
|
7024
|
+
await deps.heartbeats.pauseCheck(created.id);
|
|
7025
|
+
}
|
|
7026
|
+
const ownedNames = new Set([
|
|
7027
|
+
PLATFORM_UPTIME_HEARTBEAT_CHECK_NAME,
|
|
7028
|
+
...PLATFORM_BACKUP_SERVICE_KEYS.map((service) => platformBackupHeartbeatCheckName(service)),
|
|
7029
|
+
]);
|
|
7030
|
+
const resolve = deps.heldKeys?.resolve;
|
|
7031
|
+
for (const check of live) {
|
|
7032
|
+
if (!ownedNames.has(check.name) || desiredSet.has(check.name))
|
|
7033
|
+
continue;
|
|
7034
|
+
// #540 round 2: a name match this reconcile holds no ref for is never
|
|
7035
|
+
// deleted -- see this function's own doc comment, third paragraph.
|
|
7036
|
+
const heldValue = resolve ? await resolve(heartbeatCheckUrlRef(check.name)) : null;
|
|
7037
|
+
if (!heldValue)
|
|
7038
|
+
continue;
|
|
7039
|
+
await deps.heartbeats.deleteCheck(check.id);
|
|
7040
|
+
}
|
|
7041
|
+
// Last: so a run that failed anywhere above (an unreconciled create or
|
|
7042
|
+
// delete) leaves this unset and `planHeartbeats` plans the step again --
|
|
7043
|
+
// the same "record markers last" rule `provisionPlatformHc` follows.
|
|
7044
|
+
context.recordScopeId(PLATFORM_HEARTBEATS_APPLIED_SCOPE_KEY, [...desiredNames].sort().join(','));
|
|
7045
|
+
return undefined;
|
|
7046
|
+
}
|
|
6604
7047
|
/**
|
|
6605
7048
|
* The real {@link PlatformCloudflareReader}: the platform's apex zone as
|
|
6606
7049
|
* `planApexZone` expects to see it, and nothing else. The zone's DNS records
|
|
@@ -6677,6 +7120,16 @@ export class ProvidersPlatformCoolifyReader {
|
|
|
6677
7120
|
databaseStatus = undefined;
|
|
6678
7121
|
}
|
|
6679
7122
|
}
|
|
7123
|
+
// An app with no `DATABASE_URL_OWNER` predates the owner/runtime
|
|
7124
|
+
// split, so `planJobs` plans the step again to set both URLs. Left
|
|
7125
|
+
// out when the envs cannot be read: unknown is not "missing".
|
|
7126
|
+
let ownerUrlMissing;
|
|
7127
|
+
try {
|
|
7128
|
+
ownerUrlMissing = !(await this.client.listEnvs(app.uuid)).some((e) => e.key === 'DATABASE_URL_OWNER' && !e.is_preview && e.value);
|
|
7129
|
+
}
|
|
7130
|
+
catch {
|
|
7131
|
+
ownerUrlMissing = undefined;
|
|
7132
|
+
}
|
|
6680
7133
|
resources.push({
|
|
6681
7134
|
system: 'coolify',
|
|
6682
7135
|
kind: 'jobs',
|
|
@@ -6685,6 +7138,7 @@ export class ProvidersPlatformCoolifyReader {
|
|
|
6685
7138
|
uuid: app.uuid,
|
|
6686
7139
|
status: app.status,
|
|
6687
7140
|
...(databaseStatus !== undefined && { databaseStatus }),
|
|
7141
|
+
...(ownerUrlMissing && { ownerUrlMissing }),
|
|
6688
7142
|
},
|
|
6689
7143
|
});
|
|
6690
7144
|
}
|
|
@@ -6702,8 +7156,10 @@ export class ProvidersPlatformCoolifyReader {
|
|
|
6702
7156
|
if (!databaseId)
|
|
6703
7157
|
continue;
|
|
6704
7158
|
try {
|
|
6705
|
-
|
|
6706
|
-
|
|
7159
|
+
// A schedule that dumps the wrong database is not live: the step is
|
|
7160
|
+
// planned again and `provisionPlatformBackup` repairs it.
|
|
7161
|
+
const { schedule, named } = await findPlatformBackupSchedule(this.client, service, databaseId);
|
|
7162
|
+
const scheduled = named ? schedule : undefined;
|
|
6707
7163
|
if (scheduled) {
|
|
6708
7164
|
resources.push({
|
|
6709
7165
|
system: 'coolify',
|