@wtfalch/foundry 0.7.1 → 0.8.0

package/dist/plan/executor.js

@@ -2,11 +2,32 @@ import { randomBytes } from 'node:crypto';
 import { URL } from 'node:url';
 import { CALLS_SFU_PRIVATE_URL, CALLS_SFU_PUBLIC_URL } from '../calls-sfu.js';
 import { AGORA_CLIENT_ID_SCOPE_KEY, APP_CLIENT_ID_SCOPE_KEY, CMS_CLIENT_ID_SCOPE_KEY, agoraHostnameFor, desiredAgoraEnv, } from './agora.js';
-import { MAS_IMAGE, SYNAPSE_IMAGE, SYNAPSE_TAG, callsMasConfigYaml, callsServerName, callsSynapseHomeserverYaml, mintCallsMatrixSecrets, parseCallsDbUrl, } from './calls-matrix.js';
+import { MAS_IMAGE, MAS_TAG, SYNAPSE_IMAGE, SYNAPSE_TAG, callsMasConfigYaml, callsServerName, callsSynapseHomeserverYaml, mintCallsMatrixSecrets, parseCallsDbUrl, } from './calls-matrix.js';
 import { ensureCloudflaredConnector } from './cloudflared-connector.js';
-import { LOGIN_BINDING_SCOPE_KEY, loginBindingState, loginHostClientIds, } from './login-host.js';
+import { LOGIN_BINDING_SCOPE_KEY, LOGIN_HOST_PENDING_REMOVAL_SCOPE_KEY, loginBindingState, loginHostClientIds, pendingRemovalHosts, pendingRemovalState, } from './login-host.js';
 import { MAIL_COPY_COMPLETED_SCOPE_KEY, MAIL_COPY_DONE_SCOPE_KEY, parseMailCopyCompleted, runMailCopyStep, } from './mail-copy.js';
-import { APP_ENV_APPLIED_SCOPE_KEY, DB_ENV_APPLIED_SCOPE_KEY, ORG_MAIL_APPLIED_APEX_SCOPE_KEY, ORG_MAIL_APPLIED_HOST_SCOPE_KEY, resolveAppEnv, } from './platform-diff.js';
+import { APP_ENV_APPLIED_SCOPE_KEY, CALLS_MAS_APPLIED_IMAGE_TAG_SCOPE_KEY, CALLS_SYNAPSE_APPLIED_IMAGE_TAG_SCOPE_KEY, DB_ENV_APPLIED_SCOPE_KEY, ORG_MAIL_APPLIED_APEX_SCOPE_KEY, ORG_MAIL_APPLIED_HOST_SCOPE_KEY, resolveAppEnv, } from './platform-diff.js';
+/**
+ * #274 fix round 4: bridges `@wtfalch/providers`' real `ZitadelClient.
+ * listApps` -- a plain array, with no `loginBaseUri` or completeness signal
+ * -- into {@link ZitadelPort.listApps}'s own shape, so a host wiring that
+ * real client in satisfies this port without reaching into `executor.ts`
+ * itself to build the wrapper by hand. `complete: false` on every call:
+ * the real client's own `_search` request has no pagination on it yet
+ * (https://github.com/wtfalch/providers/issues/29), so this adapter cannot
+ * promise it read more than one page. `loginBaseUri` comes back `undefined`
+ * for every app for the identical reason -- the real client does not read
+ * it back at all yet, not "read it back as absent" -- which is exactly the
+ * state `zitadelLoginHost`'s fail-closed rule exists for: every pending
+ * host stays pending until the tracked change above lands and this adapter
+ * (or its replacement) can start reporting `string | null` for real.
+ */
+export function adaptZitadelListApps(client) {
+    return async (orgId, projectId) => ({
+        apps: await client.listApps(orgId, projectId),
+        complete: false,
+    });
+}
 // ---------------------------------------------------------------------------
 // Errors
 // ---------------------------------------------------------------------------
@@ -37,6 +58,17 @@ export class NotBuiltError extends Error {
 function hostnameFor(org) {
     return org.hosting.usesDomain && org.domain ? org.domain : `${org.slug}.oikos.host`;
 }
+/**
+ * #274 fix round 3 point 4: `zitadelLoginHost`'s own comparison between a
+ * live app's ZITADEL-reported login base URI and the one this step itself
+ * constructs (`https://<host>/`) -- lowercased and with its trailing `/`
+ * (there is always exactly one, a URI's path component, never more) trimmed,
+ * so a host is never read as "not in use" purely because ZITADEL's own
+ * casing or slashing differs from this step's.
+ */
+function normalizedLoginBaseUri(uri) {
+    return uri.toLowerCase().replace(/\/$/, '');
+}
 /**
  * `cms.<domain>` -- the cms studio's own hostname (`cmsSite` below), apart
  * from the public site itself at `org.domain` (used directly there). Called
@@ -278,6 +310,64 @@ const MAIL_DOMAIN_SCOPE_KEY = 'mail_domain';
  * (`platform-providers.ts`), never a shared tunnel with appended ingress.
  */
 const CLOUDFLARE_TUNNEL_CONNECTOR_APP_SCOPE_KEY = 'cloudflare_tunnel_connector_app';
+/**
+ * #236: the domain `publishTunnelDnsRecords` last published this org's
+ * tunnel's hostnames under -- compared against the org's current `domain`
+ * so `plan/diff.ts`'s `planTunnel` can tell a stale publish (never done at
+ * all, or the org's domain has since changed) from a steady state with
+ * nothing left to do, and so `destroy` and `deleteStaleTunnelDnsRecords`
+ * know which zone to clean up even once the org's declared domain has moved
+ * on. `diff.ts` keeps its own copy of this exact string, the same
+ * duplication {@link CLOUDFLARE_TUNNEL_CONNECTOR_APP_SCOPE_KEY} above
+ * already follows.
+ */
+const CLOUDFLARE_TUNNEL_DNS_DOMAIN_SCOPE_KEY = 'cloudflare_tunnel_dns_domain';
+/**
+ * #236 round 3: every hostname {@link FoundryStepExecutor.publishTunnelDnsRecords}
+ * last wrote a CNAME for, joined by `,` (hostnames carry no `,`, the same
+ * join {@link MAIL_ROLE_MAILBOXES_SCOPE_KEY} already relies on) -- so
+ * cleanup (`destroy`'s `cloudflare:tunnel` case and
+ * {@link FoundryStepExecutor.deleteStaleTunnelDnsRecords}) can look each one
+ * up by name instead of listing the zone. The real adapter's
+ * `listDnsRecords` (`@wtfalch/providers`' `cloudflare.ts`) asks for one page
+ * of 100 and never walks `result_info.total_pages`, so a zone with more
+ * records than that silently lost everything past the first page. A record
+ * from before this key existed has no entry here yet: cleanup falls back to
+ * the old `listDnsRecords` zone-wide pass for that one run, and the next
+ * successful publish leaves this key set so later cleanups never need to.
+ */
+const CLOUDFLARE_TUNNEL_DNS_HOSTNAMES_SCOPE_KEY = 'cloudflare_tunnel_dns_hostnames';
+/**
+ * #236 round 5 finding 1: the domain, and the hostnames under it,
+ * {@link FoundryStepExecutor.publishTunnelDnsRecords} is in the middle of
+ * writing CNAMEs for -- recorded after *each* individual CNAME write
+ * succeeds, well before the step as a whole returns and
+ * {@link CLOUDFLARE_TUNNEL_DNS_DOMAIN_SCOPE_KEY}/
+ * {@link CLOUDFLARE_TUNNEL_DNS_HOSTNAMES_SCOPE_KEY} themselves are updated
+ * (which, on a domain change, waits for the *old* domain's cleanup to
+ * succeed too -- round 4 finding 3). Without this, a `createDnsRecord`
+ * failure partway through a hostname list, or an old-domain cleanup failure
+ * right after a successful publish, left an already-created CNAME out of
+ * every recorded list, so `destroy` had no way to find and remove it.
+ * `destroy` checks this pair in addition to the authoritative one above.
+ * Never read by `cloudflareTunnel` itself to decide anything -- it is
+ * destroy's (and a retried apply's own idempotent re-check's) safety net,
+ * not a second source of truth to branch planning on.
+ */
+const CLOUDFLARE_TUNNEL_DNS_PENDING_DOMAIN_SCOPE_KEY = 'cloudflare_tunnel_dns_pending_domain';
+const CLOUDFLARE_TUNNEL_DNS_PENDING_HOSTNAMES_SCOPE_KEY = 'cloudflare_tunnel_dns_pending_hostnames';
+/**
+ * #236 round 5 finding 2: the tunnel id this step most recently replaced
+ * with a new one -- a recreate, `findTunnelByName` returning a different id
+ * for the same name -- recorded unconditionally, even on an apply where
+ * `org.domain` is unset and the DNS-publish block below never runs. A CNAME
+ * published while a domain *was* declared can still be live under that old
+ * id after the domain is later cleared, so `destroy` needs this even then.
+ * `publishTunnelDnsRecords`'s own repoint logic already accepts either the
+ * current tunnel id or this one when matching a CNAME as this org's own
+ * rather than a foreign conflict (round 3); `destroy` now does the same.
+ */
+const CLOUDFLARE_TUNNEL_PREVIOUS_ID_SCOPE_KEY = 'cloudflare_tunnel_previous_id';
 /**
  * #162: the owner's own mailbox, created once inside `coolify:mail` after
  * `applyOrgMail` succeeds, alongside {@link MAIL_DOMAIN_SCOPE_KEY} -- another
@@ -599,9 +689,70 @@ export class FoundryStepExecutor {
      * domains, writes zitadel-login's binding, then repoints the apps whose
      * client id that binding holds. Apps created later are not in it yet:
      * the next run replans (`diff.ts`'s `planLoginHost`) and moves them.
+     *
+     * #274: when the org's host has moved (`recorded[key]`, the *previous*
+     * run's resourceId, differs from this run's `host`), also removes every
+     * host that is no longer the org's current one from both places --
+     * `ensureTrustedDomain` and `upsertBinding` above only ever add, so
+     * without this an old host stays trusted and bound forever, and a later
+     * org that reuses the old slug inherits it unasked.
+     *
+     * #274 fix round 2: this always returns `host` -- the *current* desired
+     * host -- as its `resourceId`, never an old one, however many of its
+     * removals are still outstanding. Fix round 1 returned `previousHost`
+     * while a removal was skipped, banking on `planLoginHost` replanning the
+     * whole step off that mismatch alone; that loses track of the *new* host
+     * the moment a second host change moves `previousHost` on again (A -> B
+     * skipped, then B -> C: A's own removal has nothing left recording it was
+     * ever owed). `LOGIN_HOST_PENDING_REMOVAL_SCOPE_KEY`
+     * (`pendingRemovalHosts`/`pendingRemovalState`, `login-host.ts`) is the
+     * durable record instead -- every host still awaiting removal, independent
+     * of which one is current -- and `planLoginHost` replans the step whenever
+     * it is non-empty, not only when the current host itself differs.
+     *
+     * Whether a host is still "in use" is decided from each live app's actual
+     * ZITADEL Login V2 base URI (`ZitadelPort.listApps`' `loginBaseUri`), not
+     * from `boundAppIds` membership: fix round 1 skipped removal only while an
+     * app outside `boundAppIds` existed, but `boundAppIds` is deliberately
+     * narrower than "every live app" (it excludes `native`/`api` apps and any
+     * `web` app without a recorded client id yet) and a host can safely be
+     * removed even while such an app exists, so long as nothing is actually
+     * pointed at it. Removing by the wrong signal either breaks sign-in (an
+     * app repointed is not the same fact as an app with no client id yet) or
+     * strands a host forever (an org with a `native`/`api` app would never
+     * satisfy "every live app bound").
+     *
+     * #274 fix round 3, four more defects fix round 2 left:
+     *
+     * 1. The *current* host is excluded from the candidates for removal.
+     *    Without this, A -> B (A pending, not yet safe) -> A deletes the host
+     *    this very call just (re-)trusted and bound: a host already in
+     *    `LOGIN_HOST_PENDING_REMOVAL_SCOPE_KEY` from an earlier run can be
+     *    exactly the one the org's host is moving back to.
+     * 2. Fail closed on missing information, rather than ever reading
+     *    "nothing uses it" off of one: an app whose `loginBaseUri` cannot be
+     *    read at all (`undefined` -- which, until a real adapter is wired per
+     *    `ZitadelPort.listApps`'s own doc comment, is every app, every call)
+     *    means every pending host stays pending this run, not just the ones a
+     *    narrower check happened to miss.
+     * 3. Same fail-closed rule for `listApps`' own `complete` flag: an
+     *    incomplete read (today, always, for the identical reason) means the
+     *    set of apps this call saw could be missing one still pointed at a
+     *    pending host, so nothing is removed off of it either.
+     * 4. A base URI compares after lowercasing and trimming its trailing
+     *    `/` (`normalizedLoginBaseUri`) -- ZITADEL's own value and the one
+     *    this step constructs are not guaranteed to agree on case or a
+     *    trailing slash, and a comparison that is case- or slash-sensitive
+     *    both ways can report "in use" as "not" by accident.
+     * 5. Each live app's base URI is read as of *after* this call's own
+     *    repoint loop above, not the stale value `listApps` returned before
+     *    it: a bound app this very call just repointed away from a pending
+     *    host must not hold that host pending for one more run only because
+     *    the snapshot taken before the repoint still named it.
      */
     async zitadelLoginHost(action, org, context) {
         const recorded = context.recordedIds();
+        const previousHost = recorded[`zitadel:login-host:${action.key}`] ?? null;
         const zitadelOrgId = recorded.zitadel_org;
         if (!zitadelOrgId) {
             throw new Error(`zitadel:login-host for ${org.slug}: no ZITADEL org recorded yet -- the issuer must be provisioned first`);
@@ -628,17 +779,93 @@ export class FoundryStepExecutor {
             ...(recorded[AGORA_CLIENT_ID_SCOPE_KEY] ? ['agora'] : []),
         ].map((name) => recorded[`zitadel:app:${name}`]));
         const projectId = recorded.zitadel_project;
+        // No project yet means no apps yet: vacuously safe to decide from (fix
+        // round 3 point 2/3 apply only once there is something to fail closed
+        // on), and nothing to repoint either.
+        let liveApps = [];
+        let listingComplete = true;
         if (projectId) {
-            const apps = await this.ports.zitadel.listApps(zitadelOrgId, projectId);
-            for (const app of apps) {
-                if (app.state === 'APP_STATE_REMOVED' || !boundAppIds.has(app.id))
+            const listed = await this.ports.zitadel.listApps(zitadelOrgId, projectId);
+            listingComplete = listed.complete;
+            liveApps = listed.apps.filter((app) => app.state !== 'APP_STATE_REMOVED');
+            for (const app of liveApps) {
+                if (!boundAppIds.has(app.id))
                     continue;
                 await this.ports.zitadel.updateOidcAppLoginBaseUri(zitadelOrgId, projectId, app.id, `https://${host}/`);
             }
         }
-        // Last, so a crash above reruns the whole step. A client id recorded
-        // after this point differs from it and replans the step next run.
+        // Fix round 3 point 5: a bound app's current base URI is `host` for
+        // certain, just set above, whatever `listApps` read before that --
+        // never its stale pre-repoint snapshot. Every other live app's is
+        // exactly what `listApps` reported, unmodified by this call. `null`
+        // (fix round 4 point 1: read, and confirmed this app has never had one)
+        // is deliberately not added -- it names no host, so there is nothing to
+        // compare against, same as an app this loop never visits at all.
+        const currentLoginBaseUris = new Set();
+        for (const app of liveApps) {
+            const uri = boundAppIds.has(app.id) ? `https://${host}/` : app.loginBaseUri;
+            if (typeof uri === 'string')
+                currentLoginBaseUris.add(normalizedLoginBaseUri(uri));
+        }
+        // Fix round 3 points 2/3, fix round 4 point 1: an unbound app's base
+        // URI this call could not read *at all* (`undefined`, never `null` --
+        // {@link ZitadelPort.listApps}'s own doc comment has the three-state
+        // distinction), or a listing this call cannot be sure is complete,
+        // means there is no way to tell whether a pending host is still in use
+        // -- fail closed for every pending host this run rather than trust a
+        // partial picture. A bound app always has a known-good URI as of just
+        // above, so it is never the reason to fail closed; nor is an unbound
+        // app `listApps` confirmed has no base URI at all (`null`) -- fail
+        // round 3 collapsed that into "unreadable" too, which would have kept
+        // a host pending forever against an org with such an app, even once
+        // the real client can tell the two apart.
+        const canDecideSafely = listingComplete &&
+            liveApps.every((app) => boundAppIds.has(app.id) || app.loginBaseUri !== undefined);
+        // Every host up for removal this run: whatever was already pending,
+        // plus the host this run is moving away from (if any) -- deduplicated,
+        // since a host can arrive here twice across two runs that both saw it
+        // pending -- and, fix round 3 point 1, never the host this very call
+        // just (re-)trusted and bound, however it got into either set.
+        const candidates = new Set(pendingRemovalHosts(recorded));
+        candidates.delete(host);
+        if (previousHost && previousHost !== host)
+            candidates.add(previousHost);
+        // #274: a removal, once the new host is fully live (trusted, bound, and
+        // every repointable app repointed) -- never the other way round, so an
+        // org is never left with no trusted host at all if this throws partway
+        // through. Each removal catches its own not-found separately (`destroy`'s
+        // own `cloudflare:tunnel` case below takes the identical precaution, for
+        // its connector app and its tunnel): a domain already gone must not skip
+        // the binding delete that follows it.
+        const stillPending = [];
+        for (const candidate of candidates) {
+            if (!canDecideSafely ||
+                currentLoginBaseUris.has(normalizedLoginBaseUri(`https://${candidate}/`))) {
+                stillPending.push(candidate);
+                continue;
+            }
+            try {
+                await this.ports.zitadel.deleteTrustedDomain(candidate);
+            }
+            catch (error) {
+                if (!isNotFoundError(error))
+                    throw error;
+            }
+            try {
+                await bindings.deleteBinding(candidate);
+            }
+            catch (error) {
+                if (!isNotFoundError(error))
+                    throw error;
+            }
+        }
+        // Both last, so a crash above reruns the whole step rather than losing
+        // track of either. A client id recorded after this point differs from
+        // the binding and replans the step next run; a nonempty pending list
+        // does too (`planLoginHost`), independent of the host comparison that
+        // normally alone decides it.
         context.recordScopeId(LOGIN_BINDING_SCOPE_KEY, loginBindingState(host, clientIds));
+        context.recordScopeId(LOGIN_HOST_PENDING_REMOVAL_SCOPE_KEY, pendingRemovalState(stillPending));
         return { resourceId: host };
     }
     /**
@@ -776,7 +1003,7 @@ export class FoundryStepExecutor {
      * that is not the survivor this is looking for.
      */
     async findRecreatedApp(zitadelOrgId, projectId, name, appType) {
-        const apps = await this.ports.zitadel.listApps(zitadelOrgId, projectId);
+        const { apps } = await this.ports.zitadel.listApps(zitadelOrgId, projectId);
         return apps.find((app) => app.name === name && app.appType === appType && app.state !== 'APP_STATE_REMOVED');
     }
     async ensureZitadelProject(org, zitadelOrgId, context) {
@@ -1742,6 +1969,14 @@ export class FoundryStepExecutor {
             });
         }
         context.recordScopeId(CALLS_SYNAPSE_SCOPE_KEY, synapse.appId);
+        // #251: only on create -- nothing here patches `docker_registry_image_tag`
+        // on an app `findOrCreateCallsApp` found already running, so that is the
+        // only moment this tag is actually live. Recording it every run, the way
+        // the app id above is, would make `versionNotAppliedLine` compare a
+        // bumped `VERSIONS.synapse` against a value never actually applied.
+        if (!synapse.existed) {
+            context.recordScopeId(CALLS_SYNAPSE_APPLIED_IMAGE_TAG_SCOPE_KEY, SYNAPSE_TAG);
+        }
         const mas = await this.findOrCreateCallsApp(names.mas, environment.id, () => this.ports.coolify.createDockerfileApplication({
             project_uuid: projectUuid,
             server_uuid: server.uuid,
@@ -1759,6 +1994,10 @@ export class FoundryStepExecutor {
             });
         }
         context.recordScopeId(CALLS_MAS_SCOPE_KEY, mas.appId);
+        // #251: only on create -- see the matching comment on synapse above.
+        if (!mas.existed) {
+            context.recordScopeId(CALLS_MAS_APPLIED_IMAGE_TAG_SCOPE_KEY, MAS_TAG);
+        }
         // Synapse's own signing key, media store and pid file (#75's own
         // durable-volume convention, reused here for a different reason: this
         // volume is Synapse's own state, not a log).
@@ -2208,8 +2447,17 @@ export class FoundryStepExecutor {
             return { resourceId: tunnelId };
         }
         const tunnelName = `${org.slug}-tunnel`;
+        // #236 round 5 finding 2: read, and (if it is about to change) recorded,
+        // before `createTunnel` can replace it -- unconditionally, not only
+        // inside the `if (org.domain)` block below, so a recreate on an apply
+        // with no domain declared still leaves `destroy` a way to recognise the
+        // old id's CNAMEs as this tunnel's own rather than a foreign record.
+        const recordedTunnelId = context.recordedIds()[`cloudflare:tunnel:${action.key}`];
         const existing = await this.ports.cloudflare.findTunnelByName(tunnelName);
         const tunnel = existing ?? (await this.ports.cloudflare.createTunnel(tunnelName));
+        if (recordedTunnelId && recordedTunnelId !== tunnel.id) {
+            context.recordScopeId(CLOUDFLARE_TUNNEL_PREVIOUS_ID_SCOPE_KEY, recordedTunnelId);
+        }
         // Routes each of the org's hostnames (`tunnelIngressHostnames`) to
         // Coolify's own Traefik reverse proxy, not a per-app container address --
         // `COOLIFY_PROXY_ORIGIN`, the same constant name and value
@@ -2257,8 +2505,252 @@ export class FoundryStepExecutor {
                 await context.persistScopeIds();
             },
         });
+        // #236: an org with no domain (a level-1 org whose app still serves off
+        // the shared `*.oikos.host` wildcard tunnel, `scripts/shared-ingress.mjs`)
+        // has no zone of its own and nothing here to publish -- `org.domain` is
+        // set whenever `plan/diff.ts`'s `planTunnel` actually plans this step,
+        // but this executor is also exercised directly against an action it
+        // would never plan, so the check stays explicit rather than assumed.
+        //
+        // Round 3 finding: publish the new domain *before* cleaning up the old
+        // one. `previousDomain`/`previousHostnames`/`previousTunnelId` are
+        // captured here, before either scope id is recorded below, because
+        // `context.recordedIds()` is a live view of the same map this method
+        // writes to (`StepExecutionContext.recordedIds`'s own doc comment) --
+        // reading any of them any later would already see the new values. The
+        // old order (delete, then publish) could take a domain change's
+        // hostnames dark: if the new zone was missing or foreign-conflicted,
+        // `publishTunnelDnsRecords` threw *after* the old CNAMEs were already
+        // gone and nothing had replaced them. Publish throwing now leaves the
+        // old domain's records and its recorded scope ids untouched, so a
+        // conflict there is a no-op here rather than an outage.
+        //
+        // Round 4 finding 3: the new domain/hostnames are recorded here, by this
+        // method, only *after* `deleteStaleTunnelDnsRecords` has succeeded --
+        // never inside `publishTunnelDnsRecords` itself. The old order recorded
+        // the new domain as soon as publish returned; `runSteps`'s own catch
+        // block persists whatever was recorded so far even when the step then
+        // throws, so a cleanup failure right after a successful publish left the
+        // new domain durably recorded. A retry then read `previousDomain` as
+        // already equal to `org.domain` and skipped cleanup forever, orphaning
+        // the old domain's records. Recording after cleanup means a cleanup
+        // failure leaves the *old* domain recorded, so a retry sees the same
+        // domain change again and tries cleanup again.
+        if (org.domain) {
+            const recordedBefore = context.recordedIds();
+            const previousDomain = recordedBefore[CLOUDFLARE_TUNNEL_DNS_DOMAIN_SCOPE_KEY];
+            const previousHostnamesRaw = recordedBefore[CLOUDFLARE_TUNNEL_DNS_HOSTNAMES_SCOPE_KEY];
+            const previousHostnames = previousHostnamesRaw ? previousHostnamesRaw.split(',') : [];
+            // Same value `recordedTunnelId` captured above, before `createTunnel`
+            // could replace it -- not re-read here, since this block's own
+            // `context.recordScopeId` calls never touch that key.
+            const previousTunnelId = recordedTunnelId;
+            const domainChanged = previousDomain !== undefined && previousDomain !== org.domain;
+            const publishedHostnames = await this.publishTunnelDnsRecords(org, org.domain, tunnel.id, previousTunnelId, 
+            // Round 4 finding 2: a hostname this domain itself dropped (no domain
+            // change) is cleaned up inline by `publishTunnelDnsRecords`, which
+            // needs the domain's own previous hostname list for that. A domain
+            // change instead cleans the *whole* old domain below, so it gets an
+            // empty list here rather than a second, redundant pass over it.
+            domainChanged ? [] : previousHostnames, context);
+            if (domainChanged) {
+                await this.deleteStaleTunnelDnsRecords(tunnel.id, previousTunnelId, previousDomain, previousHostnames);
+            }
+            context.recordScopeId(CLOUDFLARE_TUNNEL_DNS_DOMAIN_SCOPE_KEY, org.domain);
+            context.recordScopeId(CLOUDFLARE_TUNNEL_DNS_HOSTNAMES_SCOPE_KEY, publishedHostnames.join(','));
+            // #236 round 5 finding 1: the authoritative pair above now carries
+            // everything the pending pair was tracking mid-write -- clear it so a
+            // later destroy (or crash-recovery apply) never reads a stale pending
+            // domain that no longer matches reality. A no-op the first time this
+            // step ever runs (both keys already absent).
+            context.recordScopeId(CLOUDFLARE_TUNNEL_DNS_PENDING_DOMAIN_SCOPE_KEY, '');
+            context.recordScopeId(CLOUDFLARE_TUNNEL_DNS_PENDING_HOSTNAMES_SCOPE_KEY, '');
+        }
         return { resourceId: tunnel.id };
     }
+    /**
+     * #236: when this tunnel's DNS was last published under a domain that
+     * differs from the one `org` now declares, deletes every CNAME in the
+     * *old* domain's zone that points at this tunnel, so a domain switch
+     * never leaves the old domain still answering for it. Takes the previous
+     * domain and hostnames as plain arguments rather than reading them off
+     * `context` itself (round 3): `cloudflareTunnel` reads the old values
+     * before calling either this or `publishTunnelDnsRecords`, and (round 4
+     * finding 3) does not record the new domain/hostnames until *after* this
+     * method has returned, so a crash here still sees the old domain on retry.
+     * A no-op when the old zone itself is already gone.
+     *
+     * `previousHostnames` (round 3) is {@link CLOUDFLARE_TUNNEL_DNS_HOSTNAMES_SCOPE_KEY}'s
+     * recorded list, looked up one by one instead of a zone-wide
+     * `listDnsRecords` -- the real adapter's version of that call asks for one
+     * page and never walks the rest. A domain change recorded before that key
+     * existed falls back to the old zone-wide pass, the one case where no
+     * hostname list was ever written to look up instead.
+     *
+     * `previousTunnelId` (round 4 finding 1): the old domain's CNAMEs were
+     * published under whatever tunnel id was live the *last* time this step
+     * recorded that domain, which is `previousTunnelId`, not necessarily
+     * `tunnelId` -- a tunnel recreated under the same name
+     * (`findTunnelByName` returning a new id) in the same apply that also
+     * changes the domain leaves the old domain's records pointed at the old
+     * id. Matching only `tunnelId`'s target left them live forever. Both
+     * targets are checked so this stays a no-op once a retry has already
+     * moved the old domain onto the new id.
+     */
+    async deleteStaleTunnelDnsRecords(tunnelId, previousTunnelId, previousDomain, previousHostnames) {
+        const zone = await this.ports.cloudflare.findZoneByApex(previousDomain);
+        if (!zone)
+            return;
+        const dnsTargets = new Set([`${tunnelId}.cfargotunnel.com`]);
+        if (previousTunnelId && previousTunnelId !== tunnelId) {
+            dnsTargets.add(`${previousTunnelId}.cfargotunnel.com`);
+        }
+        const cf = this.ports.cloudflare;
+        const records = previousHostnames.length > 0
+            ? (await Promise.all(previousHostnames.map((hostname) => cf.findDnsRecord(zone.id, hostname, 'CNAME')))).flat()
+            : await cf.listDnsRecords(zone.id, { type: 'CNAME' });
+        for (const record of records) {
+            if (dnsTargets.has(record.content)) {
+                await cf.deleteDnsRecord(zone.id, record.id);
+            }
+        }
+    }
+    /**
+     * #236: upserts a proxied CNAME to this tunnel for each of
+     * `tunnelIngressHostnames(org)` that actually belongs to `domain`'s own
+     * zone -- a hosting app still served off the shared `*.oikos.host`
+     * wildcard tunnel (`scripts/shared-ingress.mjs`) can appear in that list
+     * alongside `agora.<domain>` (#164's unconditional Agora hostname) when
+     * `hosting.usesDomain` is off, and `<slug>.oikos.host` is not this org's
+     * zone to touch.
+     *
+     * Checks every hostname for a conflicting record *before* writing any
+     * (round 2 finding: a conflict found only partway through a hostname-by-
+     * hostname pass left earlier hostnames written but the domain never
+     * recorded -- an orphaned partial write). An existing A or AAAA record,
+     * or a CNAME already pointed anywhere but this tunnel or (round 3) its own
+     * immediately prior tunnel id, blocks the whole step with a named
+     * conflict rather than being adopted or replaced, the same do-not-adopt
+     * policy {@link cloudflareZone} itself follows for the zone. A CNAME
+     * already pointed at this tunnel is left alone, so a second apply changes
+     * nothing. The caller ({@link cloudflareTunnel}) records the domain and the
+     * returned hostname list as scope ids only once this method -- and, on a
+     * domain change, the old domain's own cleanup -- has returned successfully
+     * (round 4 finding 3), so a crash mid-write is recovered by re-running
+     * this same idempotent step, never silently skipped.
+     *
+     * Round 3: `previousTunnelId` is the tunnel id this same step recorded
+     * last time it ran (`cloudflareTunnel`'s own `cloudflare:tunnel:<key>`
+     * resource id, read before `tunnel` above is computed for this run) -- a
+     * tunnel Cloudflare shows as recreated under the same name
+     * (`findTunnelByName` returning a new id for `<slug>-tunnel`) leaves every
+     * CNAME still pointed at the *old* id. That is this org's own stale
+     * record, not a foreign one, so it is repointed rather than refused.
+     *
+     * Round 4 finding 2: `previousHostnames` is this domain's own previously
+     * recorded hostname list (empty when the domain just changed --
+     * {@link cloudflareTunnel} hands the *whole* old domain to
+     * {@link deleteStaleTunnelDnsRecords} for that case instead). A name in it
+     * that the org's current config no longer produces is dropped from
+     * `hostnames` below but was never cleaned up before this round: its CNAME
+     * stayed live and its name stayed in the recorded list forever, because
+     * only a domain *change* ever ran cleanup. Each dropped name's CNAME is
+     * deleted here when it still points at this tunnel or its immediately
+     * prior id; a name already gone, or now pointed at something foreign, is
+     * simply left out of the returned list -- nothing here is this method's to
+     * reclaim or report as a conflict.
+     *
+     * Round 5 finding 1: `context` is used only to record
+     * {@link CLOUDFLARE_TUNNEL_DNS_PENDING_DOMAIN_SCOPE_KEY}/
+     * {@link CLOUDFLARE_TUNNEL_DNS_PENDING_HOSTNAMES_SCOPE_KEY} after each
+     * individual CNAME write below succeeds, so a crash or a thrown conflict
+     * partway through never leaves an already-created CNAME out of every
+     * recorded list -- `destroy` checks this pending pair too.
+     */
+    async publishTunnelDnsRecords(org, domain, tunnelId, previousTunnelId, previousHostnames, context) {
+        const cf = this.ports.cloudflare;
+        const zone = await cf.findZoneByApex(domain);
+        if (!zone) {
+            throw new Error(`cloudflare:tunnel for ${org.slug}: no Cloudflare zone found for ${domain} to point its hostnames at -- cloudflare:zone must provision it first`);
+        }
+        const dnsTarget = `${tunnelId}.cfargotunnel.com`;
+        const previousDnsTarget = previousTunnelId && previousTunnelId !== tunnelId
+            ? `${previousTunnelId}.cfargotunnel.com`
+            : undefined;
+        const hostnames = tunnelIngressHostnames(org).filter((hostname) => hostname === domain || hostname.endsWith(`.${domain}`));
+        const droppedHostnames = previousHostnames.filter((hostname) => !hostnames.includes(hostname));
+        const conflicts = [];
+        const toCreate = [];
+        const toRepoint = [];
+        for (const hostname of hostnames) {
+            const [a] = await cf.findDnsRecord(zone.id, hostname, 'A');
+            const [aaaa] = await cf.findDnsRecord(zone.id, hostname, 'AAAA');
+            const foreign = a ?? aaaa;
+            if (foreign) {
+                conflicts.push(`${hostname} already has a ${foreign.type} record`);
+                continue;
+            }
+            const [cname] = await cf.findDnsRecord(zone.id, hostname, 'CNAME');
+            if (cname && cname.content !== dnsTarget) {
+                if (previousDnsTarget && cname.content === previousDnsTarget) {
+                    toRepoint.push({ hostname, recordId: cname.id });
+                    continue;
+                }
+                conflicts.push(`${hostname} is already a CNAME to ${cname.content}`);
+                continue;
+            }
+            if (!cname)
+                toCreate.push(hostname);
+        }
+        if (conflicts.length > 0) {
+            throw new Error(`cloudflare:tunnel for ${org.slug}: will not point its tunnel at every hostname -- ${conflicts.join('; ')}; remove the conflicting record first`);
+        }
+        // Round 5 finding 1: `settled` tracks every hostname under `domain`
+        // confirmed to have the right CNAME so far -- starting with the ones
+        // that needed no write at all, then one more per successful create or
+        // repoint below -- and is persisted to the pending scope ids after each
+        // addition, well before this method returns. A failure on any later
+        // hostname still leaves everything settled up to that point recorded,
+        // rather than only in this function's own local state.
+        const settled = hostnames.filter((hostname) => !toCreate.includes(hostname) && !toRepoint.some((r) => r.hostname === hostname));
+        const recordPending = () => {
+            context.recordScopeId(CLOUDFLARE_TUNNEL_DNS_PENDING_DOMAIN_SCOPE_KEY, domain);
+            context.recordScopeId(CLOUDFLARE_TUNNEL_DNS_PENDING_HOSTNAMES_SCOPE_KEY, settled.join(','));
+        };
+        if (settled.length > 0)
+            recordPending();
+        for (const hostname of toCreate) {
+            await cf.createDnsRecord(zone.id, {
+                type: 'CNAME',
+                name: hostname,
+                content: dnsTarget,
+                proxied: true,
+            });
+            settled.push(hostname);
+            recordPending();
+        }
+        for (const { hostname, recordId } of toRepoint) {
+            await cf.updateDnsRecord(zone.id, recordId, {
+                type: 'CNAME',
+                name: hostname,
+                content: dnsTarget,
+                proxied: true,
+            });
+            settled.push(hostname);
+            recordPending();
+        }
+        // Round 4 finding 2: clean up a hostname this domain itself dropped.
+        // Not a conflict check -- there is nothing to create or repoint here,
+        // only a possible stale CNAME of this tunnel's own to remove. Never in
+        // `hostnames`/`settled` to begin with, so nothing to update there.
+        for (const hostname of droppedHostnames) {
+            const [cname] = await cf.findDnsRecord(zone.id, hostname, 'CNAME');
+            if (cname && (cname.content === dnsTarget || cname.content === previousDnsTarget)) {
+                await cf.deleteDnsRecord(zone.id, cname.id);
+            }
+        }
+        return hostnames;
+    }
     /**
      * `plan/diff.ts`'s `planZone`: finds or creates the Cloudflare zone for the
      * org's own domain. Refuses to adopt a zone the store already records
@@ -2654,6 +3146,42 @@ export class FoundryStepExecutor {
         const { org, recordedIds } = context;
         try {
             switch (`${target.system}:${target.kind}`) {
+                case 'zitadel:login-host': {
+                    // #274: `target.id` is the host itself (`zitadelLoginHost`'s own
+                    // `resourceId: host`), the same shape `plan-apply.ts`'s
+                    // `destroyServiceIdKey` and `ORDERED_PREFIXES` rely on. Each
+                    // removal catches its own not-found separately -- the domain
+                    // already gone must not skip the binding delete that follows it,
+                    // the same reason `cloudflare:tunnel` below handles its connector
+                    // app's 404 locally rather than leaning on this method's own
+                    // outer catch.
+                    const bindings = this.ports.loginHostBindings;
+                    // #274 fix round 2: any host `zitadelLoginHost` had already moved
+                    // away from but could not yet remove (some live app's login base
+                    // URI still pointed at it) is named only in
+                    // `LOGIN_HOST_PENDING_REMOVAL_SCOPE_KEY` -- nothing else ever runs
+                    // for this org again to finish removing it once destroy deletes
+                    // the ZITADEL project and org below, so it has to go here too.
+                    for (const host of [target.id, ...pendingRemovalHosts(recordedIds)]) {
+                        try {
+                            await this.ports.zitadel.deleteTrustedDomain(host);
+                        }
+                        catch (error) {
+                            if (!isNotFoundError(error))
+                                throw error;
+                        }
+                        if (bindings) {
+                            try {
+                                await bindings.deleteBinding(host);
+                            }
+                            catch (error) {
+                                if (!isNotFoundError(error))
+                                    throw error;
+                            }
+                        }
+                    }
+                    return;
+                }
                 case 'cloudflare:tunnel': {
                     // #235: the connector app first, its own 404 handled locally --
                     // a retry where this already succeeded must still reach
@@ -2669,6 +3197,75 @@ export class FoundryStepExecutor {
                                 throw error;
                         }
                     }
+                    // #236: every CNAME `publishTunnelDnsRecords` created for this
+                    // org's tunnel, in the zone it was *last* published under -- the
+                    // recorded domain, not whatever `org.domain` says now, since a
+                    // domain change (or the org being deleted entirely) can leave
+                    // them disagreeing. Round 3: looks each recorded hostname up by
+                    // name (`CLOUDFLARE_TUNNEL_DNS_HOSTNAMES_SCOPE_KEY`) rather than
+                    // listing every CNAME in the zone -- the real adapter's
+                    // `listDnsRecords` asks for one page of 100 and never walks the
+                    // rest, so a zone with more records than that silently lost
+                    // everything past the first page. A hostname a later config
+                    // change dropped is still in that recorded list, so it still gets
+                    // cleaned up. A record from before that key existed falls back to
+                    // the old zone-wide list, the one case with nothing recorded to
+                    // look up instead. 404 handled locally, same reason as the
+                    // connector above.
+                    //
+                    // Round 5 finding 2: `dnsTargets` accepts this tunnel's current id
+                    // *or* {@link CLOUDFLARE_TUNNEL_PREVIOUS_ID_SCOPE_KEY} -- the id a
+                    // recreate most recently replaced -- the same two-target match
+                    // `publishTunnelDnsRecords`'s own repoint logic already uses, so a
+                    // CNAME a domain-less apply's recreate left pointed at the old id
+                    // (never repointed, since that apply never ran the DNS-publish
+                    // block at all) still gets removed here.
+                    //
+                    // Round 5 finding 1: `dnsSources` also checks the *pending*
+                    // domain/hostnames pair alongside the authoritative one, so a
+                    // CNAME `publishTunnelDnsRecords` created but never got to record
+                    // in the authoritative pair (a later hostname's write failing, or
+                    // an old-domain cleanup failure right after a successful publish)
+                    // is still found and removed.
+                    const previousTunnelId = recordedIds[CLOUDFLARE_TUNNEL_PREVIOUS_ID_SCOPE_KEY];
+                    const dnsTargets = new Set([`${target.id}.cfargotunnel.com`]);
+                    if (previousTunnelId && previousTunnelId !== target.id) {
+                        dnsTargets.add(`${previousTunnelId}.cfargotunnel.com`);
+                    }
+                    const dnsSources = [
+                        {
+                            domain: recordedIds[CLOUDFLARE_TUNNEL_DNS_DOMAIN_SCOPE_KEY],
+                            hostnamesRaw: recordedIds[CLOUDFLARE_TUNNEL_DNS_HOSTNAMES_SCOPE_KEY],
+                        },
+                        {
+                            domain: recordedIds[CLOUDFLARE_TUNNEL_DNS_PENDING_DOMAIN_SCOPE_KEY],
+                            hostnamesRaw: recordedIds[CLOUDFLARE_TUNNEL_DNS_PENDING_HOSTNAMES_SCOPE_KEY],
+                        },
+                    ];
+                    for (const { domain: dnsDomain, hostnamesRaw: dnsHostnamesRaw } of dnsSources) {
+                        if (!dnsDomain)
+                            continue;
+                        try {
+                            const zone = await this.ports.cloudflare.findZoneByApex(dnsDomain);
+                            if (zone) {
+                                const cf = this.ports.cloudflare;
+                                const records = dnsHostnamesRaw
+                                    ? (await Promise.all(dnsHostnamesRaw
+                                        .split(',')
+                                        .map((hostname) => cf.findDnsRecord(zone.id, hostname, 'CNAME')))).flat()
+                                    : await cf.listDnsRecords(zone.id, { type: 'CNAME' });
+                                for (const record of records) {
+                                    if (dnsTargets.has(record.content)) {
+                                        await cf.deleteDnsRecord(zone.id, record.id);
+                                    }
+                                }
+                            }
+                        }
+                        catch (error) {
+                            if (!isNotFoundError(error))
+                                throw error;
+                        }
+                    }
                     await this.ports.cloudflare.deleteTunnel(target.id);
                     return;
                 }