@wtfalch/foundry 0.19.1 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bootstrap/declare.d.ts +13 -0
- package/dist/bootstrap/declare.js +31 -7
- package/dist/bootstrap/versions.d.ts +1 -0
- package/dist/index.d.ts +7 -3
- package/dist/index.js +4 -2
- package/dist/migrations/0046_platform_hc.sql +19 -0
- package/dist/plan/cloudflared-connector.d.ts +3 -2
- package/dist/plan/cloudflared-connector.js +2 -2
- package/dist/plan/deploy-health.js +5 -1
- package/dist/plan/diff.js +27 -4
- package/dist/plan/edge-access.d.ts +68 -0
- package/dist/plan/edge-access.js +123 -0
- package/dist/plan/edge-dns.d.ts +89 -0
- package/dist/plan/edge-dns.js +227 -0
- package/dist/plan/edge-ratelimits.d.ts +54 -0
- package/dist/plan/edge-ratelimits.js +79 -0
- package/dist/plan/edge-tunnels.d.ts +44 -0
- package/dist/plan/edge-tunnels.js +108 -0
- package/dist/plan/edge-workers.d.ts +31 -0
- package/dist/plan/edge-workers.js +165 -0
- package/dist/plan/edge-zones.d.ts +37 -0
- package/dist/plan/edge-zones.js +64 -0
- package/dist/plan/entry-edge-key.d.ts +10 -0
- package/dist/plan/entry-edge-key.js +38 -8
- package/dist/plan/executor.d.ts +146 -143
- package/dist/plan/executor.js +370 -115
- package/dist/plan/health-check-settings.d.ts +47 -0
- package/dist/plan/health-check-settings.js +47 -0
- package/dist/plan/health-targets.d.ts +86 -0
- package/dist/plan/health-targets.js +344 -0
- package/dist/plan/index.d.ts +17 -3
- package/dist/plan/index.js +9 -2
- package/dist/plan/live.d.ts +12 -1
- package/dist/plan/plan-apply.d.ts +4 -0
- package/dist/plan/plan-apply.js +5 -1
- package/dist/plan/platform-diff.d.ts +127 -0
- package/dist/plan/platform-diff.js +304 -8
- package/dist/plan/platform-providers.d.ts +202 -7
- package/dist/plan/platform-providers.js +545 -79
- package/dist/plan/providers-readers.d.ts +23 -1
- package/dist/plan/providers-readers.js +61 -1
- package/dist/plan/rollout.d.ts +9 -8
- package/dist/platform.d.ts +14 -1
- package/dist/platform.js +1 -0
- package/dist/store/postgres.js +8 -0
- package/dist/store/schema.d.ts +51 -0
- package/dist/store/schema.js +4 -0
- package/package.json +2 -1
- package/versions.json +2 -1
package/dist/plan/executor.js
CHANGED
|
@@ -9,10 +9,17 @@ import { MAS_IMAGE, MAS_TAG, SYNAPSE_IMAGE, SYNAPSE_TAG, callsMasConfigYaml, cal
|
|
|
9
9
|
import { CHAT_REPO, CHAT_SERVICE_DOCKERFILE, CHAT_SERVICE_PORT, CHAT_SYNAPSE_DOCKERFILE, CHAT_SYNAPSE_PORT, chatInternalNames, chatMasConfigYaml, chatServiceEnvs, chatSynapseHomeserverYaml, chatUpstreams, } from './chat-matrix.js';
|
|
10
10
|
import { ensureCloudflaredConnector } from './cloudflared-connector.js';
|
|
11
11
|
import { CORE_FILES_COLLECTION_NAME, FILES_CORE_COLLECTION_SCOPE_KEY, ZITADEL_FILES_PROJECT_SCOPE_KEY, } from './core-files.js';
|
|
12
|
+
import { waitForNewDeployment, } from './deploy-health.js';
|
|
12
13
|
import { DEFAULT_AI_LIMIT_MICROS } from './diff.js';
|
|
14
|
+
import { createEdgeAccess, } from './edge-access.js';
|
|
15
|
+
import { EDGE_DNS_WRITE_CONCURRENCY, createEdgeDns, limiter, } from './edge-dns.js';
|
|
16
|
+
import { createEdgeRateLimits, } from './edge-ratelimits.js';
|
|
17
|
+
import { createEdgeTunnels } from './edge-tunnels.js';
|
|
18
|
+
import { createEdgeZones } from './edge-zones.js';
|
|
13
19
|
import { AI_ENTRY_RENEW_WINDOW_DAYS, entryAiAppliedScopeKey, entryAiClampedScopeKey, entryAiExpiresScopeKey, entryAiPendingExpiresScopeKey, entryAiRetiredScopeKey, entryAiUndeliveredScopeKey, entryAiUnrevokedScopeKey, entryAiWebhookPendingScopeKey, entryAiWrittenScopeKey, parseIdList, } from './entry-ai-key.js';
|
|
14
|
-
import { EDGE_ENTRY_KEY_DAYS, EDGE_RESERVED_LABELS_APPLIED_SCOPE_KEY, ENTRY_EDGE_KEY_DELIVERY_ATTEMPTS, entryEdgeMintedAtScopeKey, entryEdgeRetiredScopeKey, entryEdgeUndeliveredScopeKey, } from './entry-edge-key.js';
|
|
20
|
+
import { BOULE_EDGE_ENTRY_KEY, EDGE_ENTRY_KEY_DAYS, EDGE_RESERVED_LABELS_APPLIED_SCOPE_KEY, ENTRY_EDGE_KEY_DELIVERY_ATTEMPTS, edgeKeyAppRecordKey, entryEdgeMintedAtScopeKey, entryEdgeRetiredScopeKey, entryEdgeUndeliveredScopeKey, } from './entry-edge-key.js';
|
|
15
21
|
import { FILES_AGORA_KEY_SCOPE_KEY, FILES_AGORA_UNDELIVERED_SCOPE_KEY, FILES_KEY_DAYS, FILES_KEY_RENEW_WINDOW_DAYS, FILES_ORG_OWNER_SCOPE_KEY, FILES_ROOT_EXPIRES_SCOPE_KEY, FILES_ROOT_PREVIOUS_SCOPE_KEY, FILES_ROOT_UNDELIVERED_SCOPE_KEY, FILES_ROOT_UNREVOKED_SCOPE_KEY, entryFilesAppliedScopeKey, entryFilesExpiresScopeKey, entryFilesMinterScopeKey, entryFilesPendingExpiresScopeKey, entryFilesRetiredScopeKey, entryFilesUndeliveredScopeKey, entryFilesUnrevokedScopeKey, entryFilesWrittenScopeKey, filesOwnerUserId, parseIdList as parseFilesIdList, } from './entry-files-key.js';
|
|
22
|
+
import { healthCheckSettings } from './health-check-settings.js';
|
|
16
23
|
import { appHostname, coolifyDomains, entryHostnames, tunnelDnsHostnames, tunnelIngressHostnames, } from './hostnames.js';
|
|
17
24
|
import { LOGIN_BINDING_SCOPE_KEY, LOGIN_HOST_PENDING_REMOVAL_SCOPE_KEY, LOGIN_HOST_SHARED_SCOPE_KEY, loginBindingState, loginHostClientIds, pendingRemovalHosts, pendingRemovalState, } from './login-host.js';
|
|
18
25
|
import { MAIL_COPY_COMPLETED_SCOPE_KEY, MAIL_COPY_DONE_SCOPE_KEY, parseMailCopyCompleted, runMailCopyStep, } from './mail-copy.js';
|
|
@@ -79,7 +86,9 @@ function normalizedLoginBaseUri(uri) {
|
|
|
79
86
|
* is non-null by the same `validateSite` guarantee
|
|
80
87
|
* {@link tunnelIngressHostnames}'s own doc comment cites.
|
|
81
88
|
*/
|
|
82
|
-
|
|
89
|
+
// #541: exported so `health-targets.ts` reuses this hostname rule rather
|
|
90
|
+
// than restating it; nothing else about this function changes.
|
|
91
|
+
export function cmsHostnameFor(org) {
|
|
83
92
|
return `cms.${org.domain}`;
|
|
84
93
|
}
|
|
85
94
|
/**
|
|
@@ -92,7 +101,8 @@ function cmsHostnameFor(org) {
|
|
|
92
101
|
* ({@link tunnelIngressHostnames}), the same subdomain-of-the-org's-own-domain
|
|
93
102
|
* shape {@link cmsHostnameFor} uses for `cms.<domain>`.
|
|
94
103
|
*/
|
|
95
|
-
|
|
104
|
+
// #541: exported for the same reason `cmsHostnameFor` just above now is.
|
|
105
|
+
export function callsHostnameFor(org) {
|
|
96
106
|
return org.calls.usesDomain && org.domain
|
|
97
107
|
? `calls.${org.domain}`
|
|
98
108
|
: `${org.slug}-calls.oikos.host`;
|
|
@@ -228,6 +238,14 @@ const DEFAULT_APP_DOCKERFILE = '/apps/hestia/Dockerfile';
|
|
|
228
238
|
const CMS_REPO = 'wtfalch/cms';
|
|
229
239
|
const CMS_APP_DOCKERFILE = '/Dockerfile';
|
|
230
240
|
const CMS_SITE_APP_DOCKERFILE = '/apps/site/Dockerfile';
|
|
241
|
+
/**
|
|
242
|
+
* #543: both of `wtfalch/cms`'s apps are Next.js hosts on the same
|
|
243
|
+
* app-template convention as Archon/edge/ai's runner/Boule -- port 3000,
|
|
244
|
+
* `/api/health` (`healthUrls`'s own default for a non-static entry). Neither
|
|
245
|
+
* create call below passes `ports_exposes` (unlike Agora's #506 fix), so
|
|
246
|
+
* this is the port Coolify's own default build already listens its proxy on.
|
|
247
|
+
*/
|
|
248
|
+
const CMS_APP_PORT = 3000;
|
|
231
249
|
/**
|
|
232
250
|
* #164: the repo `coolifyAgora` builds the org's Agora from
|
|
233
251
|
* (https://github.com/wtfalch/agora) -- the issue's own "How" names this
|
|
@@ -261,6 +279,8 @@ const AGORA_APP_DOCKERFILE = '/apps/agora/Dockerfile';
|
|
|
261
279
|
*/
|
|
262
280
|
const BOULE_REPO = 'wtfalch/boule';
|
|
263
281
|
const BOULE_APP_DOCKERFILE = '/Dockerfile';
|
|
282
|
+
/** Boule's own `EXPOSE 3000` (its doc comment just above) -- the port its health check (#543) probes. */
|
|
283
|
+
const BOULE_PORT = 3000;
|
|
264
284
|
/**
|
|
265
285
|
* Every app foundry deploys keeps its own logs across a redeploy, so a
|
|
266
286
|
* database outage does not also erase the only other copy of the evidence
|
|
@@ -479,6 +499,16 @@ const MAIL_ROLE_MAILBOXES_SCOPE_KEY = 'mail_role_mailboxes';
|
|
|
479
499
|
const AI_KEY_UNDELIVERED_SCOPE_KEY = 'ai_key_undelivered';
|
|
480
500
|
/** How many times `aiKey` tries `deliverSecret` before giving up. */
|
|
481
501
|
const AI_KEY_DELIVERY_ATTEMPTS = 3;
|
|
502
|
+
/**
|
|
503
|
+
* #515: the limit `aiBudget` last applied, a flat scope id (no `:`) alongside
|
|
504
|
+
* `AI_KEY_UNDELIVERED_SCOPE_KEY`'s own. AI's `updateBudget` offers no read
|
|
505
|
+
* back, so nothing else can tell `diff.ts`'s `planAiBudget` the limit is
|
|
506
|
+
* already live -- without this, every plan for an applied org proposed
|
|
507
|
+
* setting it again. `providers-readers.ts`'s `ProvidersAiReader` keeps its
|
|
508
|
+
* own copy of this literal, the same "restated, not shared" choice this
|
|
509
|
+
* codebase already makes for `cloudflare_zone`/`zitadel_org`.
|
|
510
|
+
*/
|
|
511
|
+
const AI_BUDGET_LIMIT_SCOPE_KEY = 'ai_budget_limit_micros';
|
|
482
512
|
const DAY_MS = 24 * 60 * 60 * 1000;
|
|
483
513
|
/** What a failed files call says about itself: its HTTP status, never its message, which can echo a value. */
|
|
484
514
|
function filesErrorCause(error) {
|
|
@@ -582,6 +612,21 @@ const CALLS_SERVICE_DB_SCOPE_KEY = 'coolify_calls_service_database';
|
|
|
582
612
|
const CALLS_SERVICE_REPO = 'wtfalch/calls';
|
|
583
613
|
/** `packages/service/Dockerfile`, built from the repo root (that Dockerfile's own top comment). */
|
|
584
614
|
const CALLS_SERVICE_DOCKERFILE = '/packages/service/Dockerfile';
|
|
615
|
+
/**
|
|
616
|
+
* #543: the calls service's own health route. Restated, not imported, from
|
|
617
|
+
* `health-targets.ts`'s private `CALLS_HEALTH_PATH` of the same value --
|
|
618
|
+
* issue #543's own text keeps that issue's (#541) diff and this one
|
|
619
|
+
* independent of each other.
|
|
620
|
+
*/
|
|
621
|
+
const CALLS_HEALTH_CHECK_PATH = '/healthz';
|
|
622
|
+
/**
|
|
623
|
+
* #543: the chat service's own health route, reached through Agora's proxy
|
|
624
|
+
* (`chat-matrix.ts`'s "NO PUBLIC HOSTNAME OF ITS OWN": Synapse and MAS have
|
|
625
|
+
* none, only this). Restated from `health-targets.ts`'s private
|
|
626
|
+
* `CHAT_SERVICE_HEALTH_PATH` of the same value, for the same reason as
|
|
627
|
+
* {@link CALLS_HEALTH_CHECK_PATH} above.
|
|
628
|
+
*/
|
|
629
|
+
const CHAT_SERVICE_HEALTH_CHECK_PATH = '/_chat/health';
|
|
585
630
|
/**
|
|
586
631
|
* Whether an application's own Coolify `status` (`` `running:healthy`,
|
|
587
632
|
* `exited:unhealthy` `` -- `@wtfalch/providers`' own doc comment) means it is
|
|
@@ -627,6 +672,43 @@ export async function getDatabaseStarted(coolify, dbId) {
|
|
|
627
672
|
await sleep(graceMs);
|
|
628
673
|
return db;
|
|
629
674
|
}
|
|
675
|
+
/**
|
|
676
|
+
* #543: the plain `setTimeout`-based sleep every company-app deploy wait
|
|
677
|
+
* falls back to when `FoundryStepExecutorPorts.runtime.sleep` is not given --
|
|
678
|
+
* the same default `platform-providers.ts`'s own per-step `defaultSleep`
|
|
679
|
+
* constants use.
|
|
680
|
+
*/
|
|
681
|
+
const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
682
|
+
/**
|
|
683
|
+
* #543 review: a deploy waits with `'ready'` on a public url only when
|
|
684
|
+
* nothing still has to create or change the route to it. Otherwise it is
|
|
685
|
+
* `'deployment-only'`, so a fresh company is not failed for a tunnel, DNS
|
|
686
|
+
* record or Agora that a later step in the same plan creates. From
|
|
687
|
+
* `recorded`: a hostname under `org.domain` has a route once the tunnel step
|
|
688
|
+
* published its DNS record ({@link CLOUDFLARE_TUNNEL_DNS_HOSTNAMES_SCOPE_KEY});
|
|
689
|
+
* a hostname elsewhere is on the shared wildcard tunnel and already routed.
|
|
690
|
+
* `viaAgora` (the chat service, reached through Agora's proxy) also needs
|
|
691
|
+
* Agora deployed ({@link AGORA_DEPLOYED_SCOPE_KEY}). A hostname behind
|
|
692
|
+
* Cloudflare Access is never probed: with no credentials every answer is a
|
|
693
|
+
* redirect or 403.
|
|
694
|
+
*/
|
|
695
|
+
function publicHealthCheck(org, recorded, hostname, path, options = {}) {
|
|
696
|
+
const app = appEntry(org.hosting);
|
|
697
|
+
const behindAccess = (org.hosting.access ?? []).some((rule) => rule.hostname === hostname) ||
|
|
698
|
+
(app !== undefined &&
|
|
699
|
+
app.accessEmails.length > 0 &&
|
|
700
|
+
entryHostnames(org, app).includes(hostname));
|
|
701
|
+
if (behindAccess)
|
|
702
|
+
return { kind: 'deployment-only' };
|
|
703
|
+
const routed = (host) => !org.domain ||
|
|
704
|
+
!(host === org.domain || host.endsWith(`.${org.domain}`)) ||
|
|
705
|
+
(recorded[CLOUDFLARE_TUNNEL_DNS_HOSTNAMES_SCOPE_KEY] ?? '').split(',').includes(host);
|
|
706
|
+
const agoraReady = !options.viaAgora ||
|
|
707
|
+
(Boolean(recorded[AGORA_DEPLOYED_SCOPE_KEY]) && routed(agoraHostnameFor(org)));
|
|
708
|
+
if (!agoraReady || !routed(hostname))
|
|
709
|
+
return { kind: 'deployment-only' };
|
|
710
|
+
return { kind: 'ready', url: `https://${hostname}${path}` };
|
|
711
|
+
}
|
|
630
712
|
/** Timing for the wait in {@link getDatabaseStarted}; a test swaps `sleep` and `now` for a fake clock. */
|
|
631
713
|
export const databaseStartWait = {
|
|
632
714
|
sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
|
@@ -640,6 +722,12 @@ export const databaseStartWait = {
|
|
|
640
722
|
// ---------------------------------------------------------------------------
|
|
641
723
|
export class FoundryStepExecutor {
|
|
642
724
|
ports;
|
|
725
|
+
edgeDns;
|
|
726
|
+
edgeTunnels;
|
|
727
|
+
edgeAccess;
|
|
728
|
+
edgeRateLimits;
|
|
729
|
+
edgeZones;
|
|
730
|
+
edgeWriteSlot;
|
|
643
731
|
constructor(ports) {
|
|
644
732
|
this.ports = ports;
|
|
645
733
|
}
|
|
@@ -686,7 +774,7 @@ export class FoundryStepExecutor {
|
|
|
686
774
|
case 'ai:organisation':
|
|
687
775
|
return this.aiOrganisation(action, org);
|
|
688
776
|
case 'ai:budget':
|
|
689
|
-
return this.aiBudget(action, org);
|
|
777
|
+
return this.aiBudget(action, org, context);
|
|
690
778
|
case 'ai:key':
|
|
691
779
|
// The org's own key is keyed `<orgId>`, an entry's `<orgId>:<entry>` (F6a).
|
|
692
780
|
return action.key.includes(':')
|
|
@@ -774,7 +862,7 @@ export class FoundryStepExecutor {
|
|
|
774
862
|
// `everyone` is Cloudflare's own shape for "match every visitor".
|
|
775
863
|
return { decision: 'bypass', include: [{ everyone: {} }] };
|
|
776
864
|
case 'service-token': {
|
|
777
|
-
const token = await this.
|
|
865
|
+
const token = await this.accessPort().findServiceTokenByName(rule.serviceTokenName);
|
|
778
866
|
if (!token) {
|
|
779
867
|
throw new Error(`cloudflare:access for ${org.slug}: no Cloudflare Access service token named '${rule.serviceTokenName}'`);
|
|
780
868
|
}
|
|
@@ -788,7 +876,7 @@ export class FoundryStepExecutor {
|
|
|
788
876
|
*/
|
|
789
877
|
async cloudflareAccessApp(org, opts) {
|
|
790
878
|
const { appName, domain, policyName, policy } = opts;
|
|
791
|
-
const cf = this.
|
|
879
|
+
const cf = this.accessPort();
|
|
792
880
|
const onDomain = (await cf.listAccessApps()).filter((a) => a.domain === domain);
|
|
793
881
|
const foreign = onDomain.find((a) => a.name !== appName);
|
|
794
882
|
if (foreign) {
|
|
@@ -812,12 +900,10 @@ export class FoundryStepExecutor {
|
|
|
812
900
|
* gate), so `org.domain` is set and that zone is the org's alone, never
|
|
813
901
|
* shared with another org the way `oikos.host` is.
|
|
814
902
|
*
|
|
815
|
-
* Rate limiting rules
|
|
816
|
-
*
|
|
817
|
-
*
|
|
818
|
-
*
|
|
819
|
-
* `rules[]` and matching `RATE_LIMIT_RULE_DESCRIPTION`, the same shape
|
|
820
|
-
* `cloudflareAccess` above finds its own Access app by name. The Free
|
|
903
|
+
* Rate limiting rules go through edge (`RateLimitPort`) -- so "find by
|
|
904
|
+
* name" here means listing the zone's rules and matching
|
|
905
|
+
* `RATE_LIMIT_RULE_DESCRIPTION`, the same shape `cloudflareAccess` above
|
|
906
|
+
* finds its own Access app by name. The Free
|
|
821
907
|
* plan allows exactly one rule per zone, so a rule already there under a
|
|
822
908
|
* different description is refused, never adopted or replaced, mirroring
|
|
823
909
|
* `cloudflareAccess`'s own refusal when a foreign Access app already
|
|
@@ -827,40 +913,33 @@ export class FoundryStepExecutor {
|
|
|
827
913
|
if (!org.domain) {
|
|
828
914
|
throw new Error(`cloudflare:ratelimit for ${org.slug}: hosting.usesDomain is set but domain is null`);
|
|
829
915
|
}
|
|
830
|
-
const cf = this.
|
|
916
|
+
const cf = this.zonePort();
|
|
831
917
|
const zone = await cf.findZoneByApex(org.domain);
|
|
832
918
|
if (!zone) {
|
|
833
919
|
throw new Error(`cloudflare:ratelimit for ${org.slug}: no Cloudflare zone found for ${org.domain} -- cloudflare:zone must provision it first`);
|
|
834
920
|
}
|
|
835
|
-
const
|
|
836
|
-
const
|
|
837
|
-
const rules = existing?.rules ?? [];
|
|
921
|
+
const rateLimits = this.rateLimitPort();
|
|
922
|
+
const rules = await rateLimits.listRateLimitRules(zone.id);
|
|
838
923
|
const own = rules.find((r) => r.description === RATE_LIMIT_RULE_DESCRIPTION);
|
|
839
|
-
if (own)
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
}
|
|
845
|
-
const updated = await cf.request('PUT', path, {
|
|
846
|
-
rules: [
|
|
847
|
-
{
|
|
848
|
-
description: RATE_LIMIT_RULE_DESCRIPTION,
|
|
849
|
-
expression: SIGN_IN_PATH_EXPRESSION,
|
|
850
|
-
action: 'block',
|
|
851
|
-
ratelimit: {
|
|
852
|
-
characteristics: ['cf.colo.id', 'ip.src'],
|
|
853
|
-
period: RATE_LIMIT_FREE_PLAN_PERIOD_SECONDS,
|
|
854
|
-
requests_per_period: RATE_LIMIT_REQUESTS_PER_PERIOD,
|
|
855
|
-
mitigation_timeout: RATE_LIMIT_FREE_PLAN_PERIOD_SECONDS,
|
|
856
|
-
},
|
|
857
|
-
},
|
|
858
|
-
],
|
|
859
|
-
});
|
|
860
|
-
const created = updated.rules?.[0];
|
|
861
|
-
if (!created) {
|
|
862
|
-
throw new Error(`cloudflare:ratelimit for ${org.slug}: PUT ${path} did not return the created rule`);
|
|
924
|
+
if (!own) {
|
|
925
|
+
const foreign = rules[0];
|
|
926
|
+
if (foreign) {
|
|
927
|
+
throw new Error(`cloudflare:ratelimit for ${org.slug}: zone ${org.domain} already has a rate limiting rule ('${foreign.description ?? foreign.id}') foundry did not create; the Free plan allows only one rule per zone, so foundry does not adopt or replace it`);
|
|
928
|
+
}
|
|
863
929
|
}
|
|
930
|
+
// Edge adopts a live rule with no edge row (made before edge owned rate
|
|
931
|
+
// limits) and then makes the rule once: a repeat returns it unchanged.
|
|
932
|
+
const created = await rateLimits.createRateLimitRule(zone.id, {
|
|
933
|
+
description: RATE_LIMIT_RULE_DESCRIPTION,
|
|
934
|
+
expression: SIGN_IN_PATH_EXPRESSION,
|
|
935
|
+
action: 'block',
|
|
936
|
+
ratelimit: {
|
|
937
|
+
characteristics: ['cf.colo.id', 'ip.src'],
|
|
938
|
+
period: RATE_LIMIT_FREE_PLAN_PERIOD_SECONDS,
|
|
939
|
+
requests_per_period: RATE_LIMIT_REQUESTS_PER_PERIOD,
|
|
940
|
+
mitigation_timeout: RATE_LIMIT_FREE_PLAN_PERIOD_SECONDS,
|
|
941
|
+
},
|
|
942
|
+
});
|
|
864
943
|
return { resourceId: created.id };
|
|
865
944
|
}
|
|
866
945
|
/**
|
|
@@ -913,7 +992,7 @@ export class FoundryStepExecutor {
|
|
|
913
992
|
if (!org.domain) {
|
|
914
993
|
throw new Error(`${at}: worker needs org.domain, and validateHosting already refuses it without one`);
|
|
915
994
|
}
|
|
916
|
-
const zone = await this.
|
|
995
|
+
const zone = await this.zonePort().findZoneByApex(org.domain);
|
|
917
996
|
if (!zone) {
|
|
918
997
|
throw new Error(`${at}: no Cloudflare zone found for ${org.domain} -- cloudflare:zone must provision it first`);
|
|
919
998
|
}
|
|
@@ -1066,7 +1145,7 @@ export class FoundryStepExecutor {
|
|
|
1066
1145
|
throw new Error(`${at}: a route is recorded but its zone is not, so foundry cannot tell which zone holds it; refuses to forget it`);
|
|
1067
1146
|
}
|
|
1068
1147
|
const { id, apex } = JSON.parse(recordedZone);
|
|
1069
|
-
const zone = await this.
|
|
1148
|
+
const zone = await this.zonePort().findZoneByApex(apex);
|
|
1070
1149
|
return zone && zone.id === id ? id : undefined;
|
|
1071
1150
|
}
|
|
1072
1151
|
/** Deletes the route an interrupted apply was creating, if `findRoute` shows it exists, points at that apply's script, and is not `keepRouteId`. */
|
|
@@ -1670,6 +1749,20 @@ export class FoundryStepExecutor {
|
|
|
1670
1749
|
}
|
|
1671
1750
|
if (want.autoDeploy !== have.autoDeploy)
|
|
1672
1751
|
patch.is_auto_deploy_enabled = want.autoDeploy;
|
|
1752
|
+
// #543: the health-check quartet too, every run -- the same reason
|
|
1753
|
+
// Archon's own doc comment gives (without one, Coolify's rolling update
|
|
1754
|
+
// has nothing to wait on and stops the old container immediately). The
|
|
1755
|
+
// port Coolify's own health probe reaches is whatever `ports_exposes`
|
|
1756
|
+
// names: 80 for a static entry (Coolify's own web server), or the
|
|
1757
|
+
// entry's declared `port`. An entry with neither (an arbitrary buildpack,
|
|
1758
|
+
// no declared port) gets none here -- there is no port this executor
|
|
1759
|
+
// could honestly name for Coolify to probe, the same "nothing to check
|
|
1760
|
+
// instead" reasoning `platform:ai`'s worker and `platform:files`'
|
|
1761
|
+
// migrate job are left out of #543's own quartet for.
|
|
1762
|
+
const healthCheckPort = isStatic ? 80 : want.port;
|
|
1763
|
+
if (healthCheckPort !== null) {
|
|
1764
|
+
Object.assign(patch, healthCheckSettings(entry.healthPath ?? (isStatic ? '/' : '/api/health'), healthCheckPort));
|
|
1765
|
+
}
|
|
1673
1766
|
// Create names the app after `domains`; a changed name reaches a live app
|
|
1674
1767
|
// here (rule 3). The internal name is set after create, as the calls
|
|
1675
1768
|
// stack's apps do.
|
|
@@ -1747,10 +1840,23 @@ export class FoundryStepExecutor {
|
|
|
1747
1840
|
// declared appEnv changed.
|
|
1748
1841
|
// A new folder needs a build, which a restart does not do, so a settings
|
|
1749
1842
|
// change deploys instead (rule 3), and that carries the env too.
|
|
1750
|
-
if (settingsNeedDeploy)
|
|
1751
|
-
|
|
1752
|
-
|
|
1843
|
+
if (settingsNeedDeploy) {
|
|
1844
|
+
// #543: every deploy waits for health through the shared helper, never
|
|
1845
|
+
// a fire-and-forget `deploy` call -- `names[0]` is the same hostname
|
|
1846
|
+
// `healthUrls` probes for this entry; an internal entry (no hostname)
|
|
1847
|
+
// has nothing this executor could reach from outside the app's own
|
|
1848
|
+
// Docker network, so `'deployment-only'` is the honest fallback there
|
|
1849
|
+
// (Coolify's own deployment record is still the whole check).
|
|
1850
|
+
const [hostname] = names;
|
|
1851
|
+
const healthPath = entry.healthPath ?? (isStatic ? '/' : '/api/health');
|
|
1852
|
+
const runtime = this.ports.runtime ?? {};
|
|
1853
|
+
await waitForNewDeployment(this.ports.coolify, appId, hostname
|
|
1854
|
+
? publicHealthCheck(org, context.recordedIds(), hostname, healthPath)
|
|
1855
|
+
: { kind: 'deployment-only' }, runtime, runtime.sleep ?? defaultSleep, `coolify:app:${entry.key}`, `${org.slug}'s ${entry.key === APP_ENTRY_KEY ? 'app' : `${entry.key} app`} stays at its previous deploy`);
|
|
1856
|
+
}
|
|
1857
|
+
else if ((appEnv.changed || dbNeedsRestart) && existing) {
|
|
1753
1858
|
await this.ports.coolify.restart(appId);
|
|
1859
|
+
}
|
|
1754
1860
|
// Recorded only after the redeploy succeeded: a failed one leaves the old
|
|
1755
1861
|
// value, so the next plan replans (rule 3). The db marker is not a URL.
|
|
1756
1862
|
// A port taken back to null was not patched, so the marker keeps the old
|
|
@@ -2481,11 +2587,11 @@ export class FoundryStepExecutor {
|
|
|
2481
2587
|
scopedProjectIds.push(platformLettermintProjectId);
|
|
2482
2588
|
}
|
|
2483
2589
|
await this.ports.lettermint.restrictDomainToProjects(domain.id, scopedProjectIds);
|
|
2484
|
-
const zone = await this.
|
|
2590
|
+
const zone = await this.zonePort().findZoneByApex(mailDomain);
|
|
2485
2591
|
if (!zone) {
|
|
2486
2592
|
throw new Error(`Lettermint for ${org.slug}: no Cloudflare zone found for ${mailDomain}`);
|
|
2487
2593
|
}
|
|
2488
|
-
await this.publishLettermintDnsRecords(zone.
|
|
2594
|
+
await this.publishLettermintDnsRecords(zone.name, domain.dnsRecords);
|
|
2489
2595
|
await this.ports.lettermint.verifyDomain(domain.id);
|
|
2490
2596
|
}
|
|
2491
2597
|
/**
|
|
@@ -2590,6 +2696,21 @@ export class FoundryStepExecutor {
|
|
|
2590
2696
|
dockerfileLocation: CMS_APP_DOCKERFILE,
|
|
2591
2697
|
domain: cmsHostnameFor(org),
|
|
2592
2698
|
});
|
|
2699
|
+
// #543: the health-check quartet, every run, on both apps -- their own
|
|
2700
|
+
// `/api/health` (`healthUrls`'s own default for site/cms), on the port
|
|
2701
|
+
// {@link CMS_APP_PORT}'s own doc comment explains. Unlike every other
|
|
2702
|
+
// company app in this file, neither app here is ever deployed through an
|
|
2703
|
+
// explicit `coolify.deploy`/`waitForNewDeployment` call: `findOrCreateCmsApp`
|
|
2704
|
+
// (below) never sets `instant_deploy` either, the same gap #504 found and
|
|
2705
|
+
// fixed for Agora and Boule -- fixing that for cms/site is its own,
|
|
2706
|
+
// separate issue (out of #543's scope: this issue adds waits to existing
|
|
2707
|
+
// deploys, it does not add a deploy where none exists).
|
|
2708
|
+
await this.ports.coolify.updateApplication(site.appId, {
|
|
2709
|
+
...healthCheckSettings('/api/health', CMS_APP_PORT),
|
|
2710
|
+
});
|
|
2711
|
+
await this.ports.coolify.updateApplication(cms.appId, {
|
|
2712
|
+
...healthCheckSettings('/api/health', CMS_APP_PORT),
|
|
2713
|
+
});
|
|
2593
2714
|
// Every app foundry deploys keeps its own logs (#75) -- reused as-is for
|
|
2594
2715
|
// both of these two, the same unconditional call `coolifyApp` makes for
|
|
2595
2716
|
// its own one app.
|
|
@@ -2967,10 +3088,21 @@ export class FoundryStepExecutor {
|
|
|
2967
3088
|
// converges. A brand new app or a freshly minted secret set are the only
|
|
2968
3089
|
// two reasons the file just written could differ from what the container
|
|
2969
3090
|
// already has -- reusing existing secrets means byte-identical content.
|
|
2970
|
-
|
|
2971
|
-
|
|
2972
|
-
|
|
2973
|
-
|
|
3091
|
+
//
|
|
3092
|
+
// #543: both waits are `'deployment-only'` -- Synapse and MAS have their
|
|
3093
|
+
// own health check off (just above: "Neither Synapse's nor MAS's health
|
|
3094
|
+
// check can pass before the other is up"), so there is no url either
|
|
3095
|
+
// could honestly be probed at; `coolify.deployAndWait` reaching Coolify's
|
|
3096
|
+
// own deployment record is still the whole check, same as `platform:ai`'s
|
|
3097
|
+
// worker.
|
|
3098
|
+
const runtime = this.ports.runtime ?? {};
|
|
3099
|
+
const sleep = runtime.sleep ?? defaultSleep;
|
|
3100
|
+
if (!synapse.existed || secrets.changed) {
|
|
3101
|
+
await waitForNewDeployment(this.ports.coolify, synapse.appId, { kind: 'deployment-only' }, runtime, sleep, 'coolify:calls', `${org.slug}'s calls Synapse stays at its previous deploy`);
|
|
3102
|
+
}
|
|
3103
|
+
if (!mas.existed || secrets.changed) {
|
|
3104
|
+
await waitForNewDeployment(this.ports.coolify, mas.appId, { kind: 'deployment-only' }, runtime, sleep, 'coolify:calls', `${org.slug}'s calls MAS stays at its previous deploy`);
|
|
3105
|
+
}
|
|
2974
3106
|
return { resourceId: mas.appId };
|
|
2975
3107
|
}
|
|
2976
3108
|
/**
|
|
@@ -3186,6 +3318,15 @@ export class FoundryStepExecutor {
|
|
|
3186
3318
|
*/
|
|
3187
3319
|
async coolifyChat(action, org, context) {
|
|
3188
3320
|
const recorded = context.recordedIds();
|
|
3321
|
+
// #543: every deploy below waits for health. The chat service answers
|
|
3322
|
+
// `CHAT_SERVICE_HEALTH_CHECK_PATH` only through Agora's own proxy
|
|
3323
|
+
// (`chat-matrix.ts`'s "NO PUBLIC HOSTNAME OF ITS OWN" -- the service has
|
|
3324
|
+
// no domain of its own), so that is the url every `'ready'` check below
|
|
3325
|
+
// probes; Synapse and MAS stay `'deployment-only'`, the same reason
|
|
3326
|
+
// `coolifyCalls`'s own comment gives for its Synapse/MAS.
|
|
3327
|
+
const runtime = this.ports.runtime ?? {};
|
|
3328
|
+
const sleep = runtime.sleep ?? defaultSleep;
|
|
3329
|
+
const chatServiceHealth = () => publicHealthCheck(org, context.recordedIds(), agoraHostnameFor(org), CHAT_SERVICE_HEALTH_CHECK_PATH, { viaAgora: true });
|
|
3189
3330
|
if (recorded[`coolify:chat:${action.key}`]) {
|
|
3190
3331
|
// #456: `planChat` plans this distinct, non-re-key action for a live
|
|
3191
3332
|
// chat app that predates the owner/runtime database split (#239) --
|
|
@@ -3210,7 +3351,7 @@ export class FoundryStepExecutor {
|
|
|
3210
3351
|
throw new Error(`coolify:chat database split for ${org.slug}: the chat service's database (${serviceDbId}) has no internal_db_url -- it may have been deleted or is unreachable; this is not expected for an already-live chat`);
|
|
3211
3352
|
}
|
|
3212
3353
|
if (push.changed || !recorded[markerKey]) {
|
|
3213
|
-
await this.ports.coolify.deploy
|
|
3354
|
+
await waitForNewDeployment(this.ports.coolify, serviceAppId, chatServiceHealth(), runtime, sleep, 'coolify:chat', `${org.slug}'s chat service stays at its previous deploy`);
|
|
3214
3355
|
}
|
|
3215
3356
|
context.recordScopeId(markerKey, '1');
|
|
3216
3357
|
return { resourceId: serviceAppId };
|
|
@@ -3233,8 +3374,9 @@ export class FoundryStepExecutor {
|
|
|
3233
3374
|
]);
|
|
3234
3375
|
// Deploy first, marker last: a failed deploy leaves the marker unset,
|
|
3235
3376
|
// so the next plan still carries this step and the container gets the env.
|
|
3236
|
-
if (client.changed || !hadMarker)
|
|
3237
|
-
await this.ports.coolify.deploy
|
|
3377
|
+
if (client.changed || !hadMarker) {
|
|
3378
|
+
await waitForNewDeployment(this.ports.coolify, serviceAppId, chatServiceHealth(), runtime, sleep, 'coolify:chat', `${org.slug}'s chat service stays at its previous deploy`);
|
|
3379
|
+
}
|
|
3238
3380
|
context.recordScopeId(CHAT_INTROSPECTION_SCOPE_KEY, client.appId);
|
|
3239
3381
|
return { resourceId: serviceAppId };
|
|
3240
3382
|
}
|
|
@@ -3336,6 +3478,8 @@ export class FoundryStepExecutor {
|
|
|
3336
3478
|
});
|
|
3337
3479
|
await this.ports.coolify.updateApplication(service.appId, {
|
|
3338
3480
|
custom_internal_name: names.service,
|
|
3481
|
+
// No Coolify health check on the chat service (#543 review): its image
|
|
3482
|
+
// is node:22-alpine with no curl, which Coolify's container check runs.
|
|
3339
3483
|
});
|
|
3340
3484
|
context.recordScopeId(CHAT_SYNAPSE_SCOPE_KEY, synapse.appId);
|
|
3341
3485
|
context.recordScopeId(CHAT_SERVICE_SCOPE_KEY, service.appId);
|
|
@@ -3402,7 +3546,11 @@ export class FoundryStepExecutor {
|
|
|
3402
3546
|
secrets.changed ||
|
|
3403
3547
|
(app === service && (serviceDbPush.changed || introspection.changed)) ||
|
|
3404
3548
|
!isRunningOrStarting((await this.ports.coolify.getApplication(app.appId)).status)) {
|
|
3405
|
-
|
|
3549
|
+
// #543: `'ready'` for the service (it has a real route, probed
|
|
3550
|
+
// through Agora); `'deployment-only'` for Synapse/MAS, same as this
|
|
3551
|
+
// function's two re-key paths above and `coolifyCalls`'s own
|
|
3552
|
+
// Synapse/MAS.
|
|
3553
|
+
await waitForNewDeployment(this.ports.coolify, app.appId, app === service ? chatServiceHealth() : { kind: 'deployment-only' }, runtime, sleep, 'coolify:chat', `${org.slug}'s chat ${app === service ? 'service' : app === synapse ? 'Synapse' : 'MAS'} stays at its previous deploy`);
|
|
3406
3554
|
}
|
|
3407
3555
|
}
|
|
3408
3556
|
// After the write and the deploys, so a failure before them leaves the
|
|
@@ -3490,6 +3638,8 @@ export class FoundryStepExecutor {
|
|
|
3490
3638
|
// step again for a live app whose marker is missing or stale, so an app
|
|
3491
3639
|
// made before this existed gets its name here. The marker, recorded last,
|
|
3492
3640
|
// tells `desiredAgoraEnv` that the name and the env are both in place.
|
|
3641
|
+
// No Coolify health check on the calls service (#543 review): its image
|
|
3642
|
+
// is node:24-alpine with no curl, which Coolify's container check runs.
|
|
3493
3643
|
await this.ports.coolify.updateApplication(app.appId, { custom_internal_name: name });
|
|
3494
3644
|
// Every app foundry deploys keeps its own logs (#75), reused as-is here.
|
|
3495
3645
|
await this.ensureLogVolume(app.appId, org, app.existed);
|
|
@@ -3557,7 +3707,11 @@ export class FoundryStepExecutor {
|
|
|
3557
3707
|
// existed, still pointed at a role nothing has created.
|
|
3558
3708
|
const givesTokenEnv = token !== null && context.recordedIds()[CALLS_SERVICE_TOKEN_ENV_SCOPE_KEY] !== tokenMarker;
|
|
3559
3709
|
if (app.existed && dbSplitNeedsRefresh) {
|
|
3560
|
-
|
|
3710
|
+
// #543: every deploy waits for health -- the calls service's own
|
|
3711
|
+
// public hostname and `/healthz`, the same path the quartet above
|
|
3712
|
+
// just set on the app.
|
|
3713
|
+
const runtime = this.ports.runtime ?? {};
|
|
3714
|
+
await waitForNewDeployment(this.ports.coolify, app.appId, publicHealthCheck(org, context.recordedIds(), callsHostnameFor(org), CALLS_HEALTH_CHECK_PATH), runtime, runtime.sleep ?? defaultSleep, 'coolify:calls-service', `${org.slug}'s calls service stays at its previous deploy`);
|
|
3561
3715
|
}
|
|
3562
3716
|
else if (app.existed && (!callsServiceHasName(org, context.recordedIds()) || givesTokenEnv)) {
|
|
3563
3717
|
await this.ports.coolify.restart(app.appId);
|
|
@@ -3771,11 +3925,14 @@ export class FoundryStepExecutor {
|
|
|
3771
3925
|
const portDiffers = app.existed &&
|
|
3772
3926
|
(await this.ports.coolify.getApplication(app.appId)).ports_exposes !==
|
|
3773
3927
|
AGORA_APP_PORTS_EXPOSES;
|
|
3774
|
-
|
|
3775
|
-
|
|
3776
|
-
|
|
3777
|
-
|
|
3778
|
-
|
|
3928
|
+
// #543: the health-check quartet, every run, like Archon's own patch --
|
|
3929
|
+
// Agora is static (Caddy on {@link AGORA_APP_PORTS_EXPOSES}), so the only
|
|
3930
|
+
// health route available is a plain page load at `/` (the same depth
|
|
3931
|
+
// `health-targets.ts` gives it: `'page'`, not an API).
|
|
3932
|
+
await this.ports.coolify.updateApplication(app.appId, {
|
|
3933
|
+
...(portDiffers ? { ports_exposes: AGORA_APP_PORTS_EXPOSES } : {}),
|
|
3934
|
+
...healthCheckSettings('/', Number(AGORA_APP_PORTS_EXPOSES)),
|
|
3935
|
+
});
|
|
3779
3936
|
// #442: read before overwriting -- `bulkUpsertEnvs` below always leaves
|
|
3780
3937
|
// Coolify's own env store matching `env`, which is why that store alone
|
|
3781
3938
|
// can no longer tell a changed value from one already live.
|
|
@@ -3794,16 +3951,16 @@ export class FoundryStepExecutor {
|
|
|
3794
3951
|
// confirmed (the marker never changes value once set), never again -- a
|
|
3795
3952
|
// live app is only ever restarted from here on, unless #506's port fix
|
|
3796
3953
|
// below needs its own deploy.
|
|
3797
|
-
if (!context.recordedIds()[AGORA_DEPLOYED_SCOPE_KEY]) {
|
|
3798
|
-
|
|
3954
|
+
if (!context.recordedIds()[AGORA_DEPLOYED_SCOPE_KEY] || portDiffers) {
|
|
3955
|
+
// #543/#506: every deploy waits for health -- Agora's own page at `/`.
|
|
3956
|
+
// A first deploy and a port fix share this one wait (a port fix needs
|
|
3957
|
+
// its own full deploy; env drift alone only needs the cheaper restart
|
|
3958
|
+
// in the branch below, and this deploy already carries this run's env,
|
|
3959
|
+
// pushed above, so the two never both run).
|
|
3960
|
+
const runtime = this.ports.runtime ?? {};
|
|
3961
|
+
await waitForNewDeployment(this.ports.coolify, app.appId, publicHealthCheck(org, context.recordedIds(), agoraHostnameFor(org), '/'), runtime, runtime.sleep ?? defaultSleep, 'coolify:agora', `${org.slug}'s Agora stays at its previous deploy`);
|
|
3799
3962
|
context.recordScopeId(AGORA_DEPLOYED_SCOPE_KEY, '1');
|
|
3800
3963
|
}
|
|
3801
|
-
else if (portDiffers) {
|
|
3802
|
-
// #506: a port fix needs its own full deploy -- env drift alone only
|
|
3803
|
-
// needs the cheaper restart, and this deploy already carries this
|
|
3804
|
-
// run's env (pushed above), so the two never both run.
|
|
3805
|
-
await this.ports.coolify.deploy(app.appId);
|
|
3806
|
-
}
|
|
3807
3964
|
else if (app.existed && (envChanged || !alreadyActivated)) {
|
|
3808
3965
|
await this.ports.coolify.restart(app.appId);
|
|
3809
3966
|
}
|
|
@@ -3870,6 +4027,12 @@ export class FoundryStepExecutor {
|
|
|
3870
4027
|
}));
|
|
3871
4028
|
// Every app foundry deploys keeps its own logs (#75), reused as-is here.
|
|
3872
4029
|
await this.ensureLogVolume(app.appId, org, app.existed);
|
|
4030
|
+
// #543: the health-check quartet, every run, like Archon's own patch --
|
|
4031
|
+
// Boule's own `/api/health` (`healthUrls`'s default) on its Dockerfile's
|
|
4032
|
+
// `EXPOSE 3000` ({@link BOULE_PORT}'s own doc comment).
|
|
4033
|
+
await this.ports.coolify.updateApplication(app.appId, {
|
|
4034
|
+
...healthCheckSettings('/api/health', BOULE_PORT),
|
|
4035
|
+
});
|
|
3873
4036
|
const liveEnv = new Map((await this.ports.coolify.listEnvs(app.appId))
|
|
3874
4037
|
.filter((e) => !e.is_preview)
|
|
3875
4038
|
.map((e) => [e.key, e.value]));
|
|
@@ -3894,7 +4057,9 @@ export class FoundryStepExecutor {
|
|
|
3894
4057
|
// but never started. Deployed once this executor has confirmed, never
|
|
3895
4058
|
// again -- a live app is only ever restarted from here on.
|
|
3896
4059
|
if (!context.recordedIds()[BOULE_DEPLOYED_SCOPE_KEY]) {
|
|
3897
|
-
|
|
4060
|
+
// #543: every deploy waits for health -- Boule's own public hostname.
|
|
4061
|
+
const runtime = this.ports.runtime ?? {};
|
|
4062
|
+
await waitForNewDeployment(this.ports.coolify, app.appId, publicHealthCheck(org, context.recordedIds(), bouleHostnameFor(org), '/api/health'), runtime, runtime.sleep ?? defaultSleep, 'coolify:boule', `${org.slug}'s Boule stays at its previous deploy`);
|
|
3898
4063
|
context.recordScopeId(BOULE_DEPLOYED_SCOPE_KEY, '1');
|
|
3899
4064
|
}
|
|
3900
4065
|
else if (app.existed &&
|
|
@@ -3934,6 +4099,87 @@ export class FoundryStepExecutor {
|
|
|
3934
4099
|
throw new Error('ai: this executor was built without an ai port (#177)');
|
|
3935
4100
|
return port;
|
|
3936
4101
|
}
|
|
4102
|
+
/**
|
|
4103
|
+
* Every DNS record call runs through edge (`edge.dns`, the platform
|
|
4104
|
+
* credential), adopting live records before it first changes them. `dns` on
|
|
4105
|
+
* the ports is a seam for a host or test that supplies its own {@link DnsPort}.
|
|
4106
|
+
*/
|
|
4107
|
+
dnsPort() {
|
|
4108
|
+
if (this.ports.dns)
|
|
4109
|
+
return this.ports.dns;
|
|
4110
|
+
if (!this.ports.edge) {
|
|
4111
|
+
throw new Error('dns: this executor was built without an EdgePort (F4.dns)');
|
|
4112
|
+
}
|
|
4113
|
+
this.edgeWriteSlot ??= limiter(EDGE_DNS_WRITE_CONCURRENCY);
|
|
4114
|
+
this.edgeDns ??= createEdgeDns(this.ports.edge.dns, { writeSlot: this.edgeWriteSlot });
|
|
4115
|
+
return this.edgeDns;
|
|
4116
|
+
}
|
|
4117
|
+
/**
|
|
4118
|
+
* Every tunnel call runs through edge (`edge.tunnels`, the platform
|
|
4119
|
+
* credential), adopting a live tunnel before it first configures, reads the
|
|
4120
|
+
* token of or deletes it. Shares edge's write slot with {@link dnsPort}.
|
|
4121
|
+
*/
|
|
4122
|
+
tunnelPort() {
|
|
4123
|
+
if (this.ports.tunnels)
|
|
4124
|
+
return this.ports.tunnels;
|
|
4125
|
+
if (!this.ports.edge) {
|
|
4126
|
+
throw new Error('tunnels: this executor was built without an EdgePort (F4.tunnels)');
|
|
4127
|
+
}
|
|
4128
|
+
this.edgeWriteSlot ??= limiter(EDGE_DNS_WRITE_CONCURRENCY);
|
|
4129
|
+
this.edgeTunnels ??= createEdgeTunnels(this.ports.edge.tunnels, {
|
|
4130
|
+
writeSlot: this.edgeWriteSlot,
|
|
4131
|
+
});
|
|
4132
|
+
return this.edgeTunnels;
|
|
4133
|
+
}
|
|
4134
|
+
/**
|
|
4135
|
+
* Every Access call runs through edge (`edge.access`, the platform
|
|
4136
|
+
* credential), adopting a live app before it first adds a policy to or
|
|
4137
|
+
* deletes it. Shares edge's write slot with {@link dnsPort}.
|
|
4138
|
+
*/
|
|
4139
|
+
accessPort() {
|
|
4140
|
+
if (this.ports.access)
|
|
4141
|
+
return this.ports.access;
|
|
4142
|
+
if (!this.ports.edge) {
|
|
4143
|
+
throw new Error('access: this executor was built without an EdgePort (F4.access)');
|
|
4144
|
+
}
|
|
4145
|
+
this.edgeWriteSlot ??= limiter(EDGE_DNS_WRITE_CONCURRENCY);
|
|
4146
|
+
this.edgeAccess ??= createEdgeAccess(this.ports.edge.access, {
|
|
4147
|
+
writeSlot: this.edgeWriteSlot,
|
|
4148
|
+
});
|
|
4149
|
+
return this.edgeAccess;
|
|
4150
|
+
}
|
|
4151
|
+
/**
|
|
4152
|
+
* Every rate limiting rule call runs through edge (`edge.ratelimits`, the
|
|
4153
|
+
* platform credential), adopting the live rule before it first creates it.
|
|
4154
|
+
* Shares edge's write slot with {@link dnsPort}.
|
|
4155
|
+
*/
|
|
4156
|
+
rateLimitPort() {
|
|
4157
|
+
if (this.ports.ratelimits)
|
|
4158
|
+
return this.ports.ratelimits;
|
|
4159
|
+
if (!this.ports.edge) {
|
|
4160
|
+
throw new Error('ratelimits: this executor was built without an EdgePort (F4.ratelimits)');
|
|
4161
|
+
}
|
|
4162
|
+
this.edgeWriteSlot ??= limiter(EDGE_DNS_WRITE_CONCURRENCY);
|
|
4163
|
+
this.edgeRateLimits ??= createEdgeRateLimits(this.ports.edge.ratelimits, {
|
|
4164
|
+
writeSlot: this.edgeWriteSlot,
|
|
4165
|
+
});
|
|
4166
|
+
return this.edgeRateLimits;
|
|
4167
|
+
}
|
|
4168
|
+
/**
|
|
4169
|
+
* Every zone lookup and create runs through edge (`edge.zones`, the
|
|
4170
|
+
* platform credential), adopting a live zone before it first creates it.
|
|
4171
|
+
* Shares edge's write slot with {@link dnsPort}.
|
|
4172
|
+
*/
|
|
4173
|
+
zonePort() {
|
|
4174
|
+
if (this.ports.zones)
|
|
4175
|
+
return this.ports.zones;
|
|
4176
|
+
if (!this.ports.edge) {
|
|
4177
|
+
throw new Error('zones: this executor was built without an EdgePort (F4.zones)');
|
|
4178
|
+
}
|
|
4179
|
+
this.edgeWriteSlot ??= limiter(EDGE_DNS_WRITE_CONCURRENCY);
|
|
4180
|
+
this.edgeZones ??= createEdgeZones(this.ports.edge.zones, { writeSlot: this.edgeWriteSlot });
|
|
4181
|
+
return this.edgeZones;
|
|
4182
|
+
}
|
|
3937
4183
|
async jobsPort(org) {
|
|
3938
4184
|
const port = this.ports.jobsScheduler ?? (await this.ports.platformOrg?.jobs());
|
|
3939
4185
|
if (!port) {
|
|
@@ -4002,7 +4248,7 @@ export class FoundryStepExecutor {
|
|
|
4002
4248
|
}
|
|
4003
4249
|
// Cleared, not deleted (#7 corrections 1): an ingress with only the
|
|
4004
4250
|
// mandatory catch-all routes nothing.
|
|
4005
|
-
await this.
|
|
4251
|
+
await this.tunnelPort().putTunnelConfig(tunnelId, {
|
|
4006
4252
|
ingress: [{ service: 'http_status:404' }],
|
|
4007
4253
|
});
|
|
4008
4254
|
// #235: park stops the connector too -- an ingress with nothing
|
|
@@ -4028,8 +4274,9 @@ export class FoundryStepExecutor {
|
|
|
4028
4274
|
// domain declared still leaves `destroy` a way to recognise the old
|
|
4029
4275
|
// id's CNAMEs as this tunnel's own rather than a foreign record.
|
|
4030
4276
|
const recordedTunnelId = context.recordedIds()[`cloudflare:tunnel:${action.key}`];
|
|
4031
|
-
const
|
|
4032
|
-
const
|
|
4277
|
+
const tunnels = this.tunnelPort();
|
|
4278
|
+
const existing = await tunnels.findTunnelByName(tunnelName);
|
|
4279
|
+
const tunnel = existing ?? (await tunnels.createTunnel(tunnelName));
|
|
4033
4280
|
const previousIds = parseTunnelPreviousIds(context.recordedIds());
|
|
4034
4281
|
if (recordedTunnelId && recordedTunnelId !== tunnel.id) {
|
|
4035
4282
|
previousIds.add(recordedTunnelId);
|
|
@@ -4056,7 +4303,7 @@ export class FoundryStepExecutor {
|
|
|
4056
4303
|
hostname,
|
|
4057
4304
|
service: COOLIFY_PROXY_ORIGIN,
|
|
4058
4305
|
}));
|
|
4059
|
-
await this.
|
|
4306
|
+
await this.tunnelPort().putTunnelConfig(tunnel.id, {
|
|
4060
4307
|
ingress: [...ingress, { service: 'http_status:404' }],
|
|
4061
4308
|
});
|
|
4062
4309
|
// #235: a connector for this org's own tunnel, the same call every
|
|
@@ -4070,7 +4317,7 @@ export class FoundryStepExecutor {
|
|
|
4070
4317
|
const { environment } = await this.ports.coolify.findOrCreateEnvironment(projectUuid, 'production');
|
|
4071
4318
|
await ensureCloudflaredConnector({
|
|
4072
4319
|
coolify: this.ports.coolify,
|
|
4073
|
-
|
|
4320
|
+
tunnels: this.tunnelPort(),
|
|
4074
4321
|
tunnelId: tunnel.id,
|
|
4075
4322
|
appName: `${org.slug}-cloudflared`,
|
|
4076
4323
|
projectUuid,
|
|
@@ -4201,7 +4448,7 @@ export class FoundryStepExecutor {
|
|
|
4201
4448
|
* nothing of this domain's left in that set either way.
|
|
4202
4449
|
*/
|
|
4203
4450
|
async deleteStaleTunnelDnsRecords(tunnelId, previousIds, previousDomain, previousHostnames, context) {
|
|
4204
|
-
const zone = await this.
|
|
4451
|
+
const zone = await this.zonePort().findZoneByApex(previousDomain);
|
|
4205
4452
|
if (!zone)
|
|
4206
4453
|
return;
|
|
4207
4454
|
const dnsTargets = new Set([`${tunnelId}.cfargotunnel.com`]);
|
|
@@ -4209,13 +4456,13 @@ export class FoundryStepExecutor {
|
|
|
4209
4456
|
if (id !== tunnelId)
|
|
4210
4457
|
dnsTargets.add(`${id}.cfargotunnel.com`);
|
|
4211
4458
|
}
|
|
4212
|
-
const
|
|
4459
|
+
const dns = this.dnsPort();
|
|
4213
4460
|
const records = previousHostnames.length > 0
|
|
4214
|
-
? (await Promise.all(previousHostnames.map((hostname) =>
|
|
4215
|
-
: await
|
|
4461
|
+
? (await Promise.all(previousHostnames.map((hostname) => dns.findDnsRecord(zone.name, hostname, 'CNAME')))).flat()
|
|
4462
|
+
: await dns.listDnsRecords(zone.name, { type: 'CNAME' });
|
|
4216
4463
|
for (const record of records) {
|
|
4217
4464
|
if (dnsTargets.has(record.content)) {
|
|
4218
|
-
await
|
|
4465
|
+
await dns.deleteDnsRecord(zone.name, record.id);
|
|
4219
4466
|
}
|
|
4220
4467
|
}
|
|
4221
4468
|
if (previousHostnames.length > 0) {
|
|
@@ -4283,8 +4530,8 @@ export class FoundryStepExecutor {
|
|
|
4283
4530
|
* both read this set.
|
|
4284
4531
|
*/
|
|
4285
4532
|
async publishTunnelDnsRecords(org, domain, tunnelId, previousIds, previousHostnames, context) {
|
|
4286
|
-
const
|
|
4287
|
-
const zone = await
|
|
4533
|
+
const dns = this.dnsPort();
|
|
4534
|
+
const zone = await this.zonePort().findZoneByApex(domain);
|
|
4288
4535
|
if (!zone) {
|
|
4289
4536
|
throw new Error(`cloudflare:tunnel for ${org.slug}: no Cloudflare zone found for ${domain} to point its hostnames at -- cloudflare:zone must provision it first`);
|
|
4290
4537
|
}
|
|
@@ -4296,14 +4543,14 @@ export class FoundryStepExecutor {
|
|
|
4296
4543
|
const toCreate = [];
|
|
4297
4544
|
const toRepoint = [];
|
|
4298
4545
|
for (const hostname of hostnames) {
|
|
4299
|
-
const [a] = await
|
|
4300
|
-
const [aaaa] = await
|
|
4546
|
+
const [a] = await dns.findDnsRecord(zone.name, hostname, 'A');
|
|
4547
|
+
const [aaaa] = await dns.findDnsRecord(zone.name, hostname, 'AAAA');
|
|
4301
4548
|
const foreign = a ?? aaaa;
|
|
4302
4549
|
if (foreign) {
|
|
4303
4550
|
conflicts.push(`${hostname} already has a ${foreign.type} record`);
|
|
4304
4551
|
continue;
|
|
4305
4552
|
}
|
|
4306
|
-
const [cname] = await
|
|
4553
|
+
const [cname] = await dns.findDnsRecord(zone.name, hostname, 'CNAME');
|
|
4307
4554
|
if (cname && cname.content !== dnsTarget) {
|
|
4308
4555
|
if (previousTargets.has(cname.content)) {
|
|
4309
4556
|
toRepoint.push({ hostname, recordId: cname.id });
|
|
@@ -4335,7 +4582,7 @@ export class FoundryStepExecutor {
|
|
|
4335
4582
|
for (const hostname of settled)
|
|
4336
4583
|
recordTracked(hostname);
|
|
4337
4584
|
for (const hostname of toCreate) {
|
|
4338
|
-
await
|
|
4585
|
+
await dns.createDnsRecord(zone.name, {
|
|
4339
4586
|
type: 'CNAME',
|
|
4340
4587
|
name: hostname,
|
|
4341
4588
|
content: dnsTarget,
|
|
@@ -4344,7 +4591,7 @@ export class FoundryStepExecutor {
|
|
|
4344
4591
|
recordTracked(hostname);
|
|
4345
4592
|
}
|
|
4346
4593
|
for (const { hostname, recordId } of toRepoint) {
|
|
4347
|
-
await
|
|
4594
|
+
await dns.updateDnsRecord(zone.name, recordId, {
|
|
4348
4595
|
type: 'CNAME',
|
|
4349
4596
|
name: hostname,
|
|
4350
4597
|
content: dnsTarget,
|
|
@@ -4363,9 +4610,9 @@ export class FoundryStepExecutor {
|
|
|
4363
4610
|
// tracking under `domain`, so it is dropped from the shared set either
|
|
4364
4611
|
// way.
|
|
4365
4612
|
for (const hostname of droppedHostnames) {
|
|
4366
|
-
const [cname] = await
|
|
4613
|
+
const [cname] = await dns.findDnsRecord(zone.name, hostname, 'CNAME');
|
|
4367
4614
|
if (cname && (cname.content === dnsTarget || previousTargets.has(cname.content))) {
|
|
4368
|
-
await
|
|
4615
|
+
await dns.deleteDnsRecord(zone.name, cname.id);
|
|
4369
4616
|
}
|
|
4370
4617
|
tracked.delete(trackedTunnelDnsEntry(domain, hostname));
|
|
4371
4618
|
context.recordScopeId(CLOUDFLARE_TUNNEL_DNS_TRACKED_SCOPE_KEY, trackedTunnelDnsState(tracked));
|
|
@@ -4399,7 +4646,7 @@ export class FoundryStepExecutor {
|
|
|
4399
4646
|
*/
|
|
4400
4647
|
async cloudflareZone(action, org, context) {
|
|
4401
4648
|
const domain = action.key;
|
|
4402
|
-
const cf = this.
|
|
4649
|
+
const cf = this.zonePort();
|
|
4403
4650
|
const existing = await cf.findZoneByApex(domain);
|
|
4404
4651
|
if (existing) {
|
|
4405
4652
|
const owner = await this.ports.declaredOrgStore.findOrgIdForZone(existing.id);
|
|
@@ -4600,12 +4847,12 @@ export class FoundryStepExecutor {
|
|
|
4600
4847
|
* publishes it.
|
|
4601
4848
|
*/
|
|
4602
4849
|
async publishMailDnsRecords(org, result) {
|
|
4603
|
-
const zone = await this.
|
|
4850
|
+
const zone = await this.zonePort().findZoneByApex(result.domain);
|
|
4604
4851
|
if (!zone) {
|
|
4605
4852
|
throw new Error(`coolify:mail for ${org.slug}: no Cloudflare zone found for ${result.domain}`);
|
|
4606
4853
|
}
|
|
4607
4854
|
for (const record of result.records) {
|
|
4608
|
-
await this.upsertMailDnsRecord(zone.
|
|
4855
|
+
await this.upsertMailDnsRecord(zone.name, record);
|
|
4609
4856
|
}
|
|
4610
4857
|
}
|
|
4611
4858
|
/**
|
|
@@ -4619,7 +4866,7 @@ export class FoundryStepExecutor {
|
|
|
4619
4866
|
* a record someone proxied by hand is put back. An `MX` record carries the
|
|
4620
4867
|
* priority it was given, which Cloudflare requires.
|
|
4621
4868
|
*
|
|
4622
|
-
* Found by an exact name+type match (`
|
|
4869
|
+
* Found by an exact name+type match (`DnsPort.findDnsRecord`'s own
|
|
4623
4870
|
* doc comment) so a stale record of a different type is never taken for a
|
|
4624
4871
|
* match. A name can hold several `TXT` records (an apex often holds a site
|
|
4625
4872
|
* verification beside SPF), so a `TXT` record only replaces one that
|
|
@@ -4628,8 +4875,8 @@ export class FoundryStepExecutor {
|
|
|
4628
4875
|
* whatever it was.
|
|
4629
4876
|
*/
|
|
4630
4877
|
async upsertMailDnsRecord(zoneId, record) {
|
|
4631
|
-
const
|
|
4632
|
-
const found = await
|
|
4878
|
+
const dns = this.dnsPort();
|
|
4879
|
+
const found = await dns.findDnsRecord(zoneId, record.name, record.type);
|
|
4633
4880
|
const existing = record.type === 'TXT'
|
|
4634
4881
|
? found.find((r) => txtRecordTag(r.content) === txtRecordTag(record.value))
|
|
4635
4882
|
: found[0];
|
|
@@ -4641,10 +4888,10 @@ export class FoundryStepExecutor {
|
|
|
4641
4888
|
...(record.priority === undefined ? {} : { priority: record.priority }),
|
|
4642
4889
|
};
|
|
4643
4890
|
if (existing) {
|
|
4644
|
-
await
|
|
4891
|
+
await dns.updateDnsRecord(zoneId, existing.id, rec);
|
|
4645
4892
|
}
|
|
4646
4893
|
else {
|
|
4647
|
-
await
|
|
4894
|
+
await dns.createDnsRecord(zoneId, rec);
|
|
4648
4895
|
}
|
|
4649
4896
|
}
|
|
4650
4897
|
/**
|
|
@@ -4700,13 +4947,13 @@ export class FoundryStepExecutor {
|
|
|
4700
4947
|
* `dkim_secondary`, none of them SPF -- so that merge has no case left to
|
|
4701
4948
|
* handle here and is dropped rather than kept as dead code.
|
|
4702
4949
|
*/
|
|
4703
|
-
async publishLettermintDnsRecords(
|
|
4950
|
+
async publishLettermintDnsRecords(zone, records) {
|
|
4704
4951
|
for (const record of records) {
|
|
4705
4952
|
if (record.type === 'TXT' && record.value.toLowerCase().startsWith('v=dmarc1')) {
|
|
4706
|
-
await this.ensureLettermintDmarcRecord(this.
|
|
4953
|
+
await this.ensureLettermintDmarcRecord(this.dnsPort(), zone, record);
|
|
4707
4954
|
continue;
|
|
4708
4955
|
}
|
|
4709
|
-
await this.upsertMailDnsRecord(
|
|
4956
|
+
await this.upsertMailDnsRecord(zone, record);
|
|
4710
4957
|
}
|
|
4711
4958
|
}
|
|
4712
4959
|
/**
|
|
@@ -4714,11 +4961,11 @@ export class FoundryStepExecutor {
|
|
|
4714
4961
|
* already set (`p=quarantine`/`p=reject`) should not be relaxed back to
|
|
4715
4962
|
* Lettermint's own `p=none` default.
|
|
4716
4963
|
*/
|
|
4717
|
-
async ensureLettermintDmarcRecord(
|
|
4718
|
-
const hasDmarc = (await
|
|
4964
|
+
async ensureLettermintDmarcRecord(dns, zoneId, record) {
|
|
4965
|
+
const hasDmarc = (await dns.findDnsRecord(zoneId, record.name, 'TXT')).some((r) => txtRecordTag(r.content) === 'v=dmarc1');
|
|
4719
4966
|
if (hasDmarc)
|
|
4720
4967
|
return;
|
|
4721
|
-
await
|
|
4968
|
+
await dns.createDnsRecord(zoneId, {
|
|
4722
4969
|
type: 'TXT',
|
|
4723
4970
|
name: record.name,
|
|
4724
4971
|
content: record.value,
|
|
@@ -4845,7 +5092,7 @@ export class FoundryStepExecutor {
|
|
|
4845
5092
|
await ai.setOrganisationOffers(org.id, action.desired.offers ?? []);
|
|
4846
5093
|
return { resourceId: provisioned.organisationId };
|
|
4847
5094
|
}
|
|
4848
|
-
async aiBudget(action, org) {
|
|
5095
|
+
async aiBudget(action, org, context) {
|
|
4849
5096
|
// #484: ai/packages/sdk/src/types.ts:134-137's UpdateBudget.limitMicros
|
|
4850
5097
|
// is a required string -- there is no "uncapped" this API can express
|
|
4851
5098
|
// yet (Finding, this campaign's report) -- so a null `desired.limitMicros`
|
|
@@ -4856,6 +5103,10 @@ export class FoundryStepExecutor {
|
|
|
4856
5103
|
const limitMicros = action.desired.limitMicros ?? DEFAULT_AI_LIMIT_MICROS;
|
|
4857
5104
|
const ai = await this.aiPort();
|
|
4858
5105
|
await ai.updateBudget(org.id, { keyId: null, limitMicros });
|
|
5106
|
+
// #515: recorded only once `updateBudget` has actually succeeded, so a
|
|
5107
|
+
// crash or a thrown call before this line leaves nothing recorded and
|
|
5108
|
+
// the step replans, rather than recording a limit that was never applied.
|
|
5109
|
+
context.recordScopeId(AI_BUDGET_LIMIT_SCOPE_KEY, limitMicros);
|
|
4859
5110
|
return { resourceId: org.id };
|
|
4860
5111
|
}
|
|
4861
5112
|
/**
|
|
@@ -5115,7 +5366,7 @@ export class FoundryStepExecutor {
|
|
|
5115
5366
|
throw error;
|
|
5116
5367
|
}
|
|
5117
5368
|
}
|
|
5118
|
-
const appId = recorded[
|
|
5369
|
+
const appId = recorded[edgeKeyAppRecordKey(org, entryKey)];
|
|
5119
5370
|
if (appId) {
|
|
5120
5371
|
// A revoked key's env values are dead, not merely stale -- remove
|
|
5121
5372
|
// them so nothing reads a credential edge has already revoked. A
|
|
@@ -5149,10 +5400,14 @@ export class FoundryStepExecutor {
|
|
|
5149
5400
|
context.recordScopeId(entryEdgeRetiredScopeKey(entryKey), keyId);
|
|
5150
5401
|
return {};
|
|
5151
5402
|
}
|
|
5152
|
-
|
|
5153
|
-
|
|
5403
|
+
if (entryKey === BOULE_EDGE_ENTRY_KEY) {
|
|
5404
|
+
if (!org.domain)
|
|
5405
|
+
throw new Error(`${at}: Boule's edge key needs org.domain`);
|
|
5406
|
+
}
|
|
5407
|
+
else if (!org.hosting.entries.some((e) => e.key === entryKey && e.uses?.edge)) {
|
|
5154
5408
|
throw new Error(`${at}: the entry no longer declares uses.edge`);
|
|
5155
|
-
|
|
5409
|
+
}
|
|
5410
|
+
const appId = context.recordedIds()[edgeKeyAppRecordKey(org, entryKey)];
|
|
5156
5411
|
if (!appId)
|
|
5157
5412
|
throw new Error(`${at}: the entry's app is not recorded yet`);
|
|
5158
5413
|
if (!org.domain) {
|
|
@@ -5590,15 +5845,15 @@ export class FoundryStepExecutor {
|
|
|
5590
5845
|
}
|
|
5591
5846
|
for (const [dnsDomain, dnsHostnames] of hostnamesByDomain) {
|
|
5592
5847
|
try {
|
|
5593
|
-
const zone = await this.
|
|
5848
|
+
const zone = await this.zonePort().findZoneByApex(dnsDomain);
|
|
5594
5849
|
if (zone) {
|
|
5595
|
-
const
|
|
5850
|
+
const dns = this.dnsPort();
|
|
5596
5851
|
const records = dnsHostnames.length > 0
|
|
5597
|
-
? (await Promise.all(dnsHostnames.map((hostname) =>
|
|
5598
|
-
: await
|
|
5852
|
+
? (await Promise.all(dnsHostnames.map((hostname) => dns.findDnsRecord(zone.name, hostname, 'CNAME')))).flat()
|
|
5853
|
+
: await dns.listDnsRecords(zone.name, { type: 'CNAME' });
|
|
5599
5854
|
for (const record of records) {
|
|
5600
5855
|
if (dnsTargets.has(record.content)) {
|
|
5601
|
-
await
|
|
5856
|
+
await dns.deleteDnsRecord(zone.name, record.id);
|
|
5602
5857
|
}
|
|
5603
5858
|
}
|
|
5604
5859
|
}
|
|
@@ -5608,11 +5863,11 @@ export class FoundryStepExecutor {
|
|
|
5608
5863
|
throw error;
|
|
5609
5864
|
}
|
|
5610
5865
|
}
|
|
5611
|
-
await this.
|
|
5866
|
+
await this.tunnelPort().deleteTunnel(target.id);
|
|
5612
5867
|
return;
|
|
5613
5868
|
}
|
|
5614
5869
|
case 'cloudflare:access':
|
|
5615
|
-
await this.
|
|
5870
|
+
await this.accessPort().deleteAccessApp(target.id);
|
|
5616
5871
|
return;
|
|
5617
5872
|
case 'ai:key': {
|
|
5618
5873
|
// F6a: an entry's key (`<orgId>:<entry>`). REVOKE SEAM (ai#157): a
|