@wtfalch/foundry 0.8.0 → 0.9.0

package/dist/plan/executor.js

@@ -1,12 +1,13 @@
 import { randomBytes } from 'node:crypto';
 import { URL } from 'node:url';
 import { CALLS_SFU_PRIVATE_URL, CALLS_SFU_PUBLIC_URL } from '../calls-sfu.js';
+import { APP_ENTRY_KEY, appEntry } from '../org.js';
 import { AGORA_CLIENT_ID_SCOPE_KEY, APP_CLIENT_ID_SCOPE_KEY, CMS_CLIENT_ID_SCOPE_KEY, agoraHostnameFor, desiredAgoraEnv, } from './agora.js';
 import { MAS_IMAGE, MAS_TAG, SYNAPSE_IMAGE, SYNAPSE_TAG, callsMasConfigYaml, callsServerName, callsSynapseHomeserverYaml, mintCallsMatrixSecrets, parseCallsDbUrl, } from './calls-matrix.js';
 import { ensureCloudflaredConnector } from './cloudflared-connector.js';
 import { LOGIN_BINDING_SCOPE_KEY, LOGIN_HOST_PENDING_REMOVAL_SCOPE_KEY, loginBindingState, loginHostClientIds, pendingRemovalHosts, pendingRemovalState, } from './login-host.js';
 import { MAIL_COPY_COMPLETED_SCOPE_KEY, MAIL_COPY_DONE_SCOPE_KEY, parseMailCopyCompleted, runMailCopyStep, } from './mail-copy.js';
-import { APP_ENV_APPLIED_SCOPE_KEY, CALLS_MAS_APPLIED_IMAGE_TAG_SCOPE_KEY, CALLS_SYNAPSE_APPLIED_IMAGE_TAG_SCOPE_KEY, DB_ENV_APPLIED_SCOPE_KEY, ORG_MAIL_APPLIED_APEX_SCOPE_KEY, ORG_MAIL_APPLIED_HOST_SCOPE_KEY, resolveAppEnv, } from './platform-diff.js';
+import { APP_ENV_APPLIED_SCOPE_KEY, CALLS_MAS_APPLIED_IMAGE_TAG_SCOPE_KEY, CALLS_SYNAPSE_APPLIED_IMAGE_TAG_SCOPE_KEY, DB_ENV_APPLIED_SCOPE_KEY, ORG_MAIL_APPLIED_APEX_SCOPE_KEY, ORG_MAIL_APPLIED_HOST_SCOPE_KEY, appStepKey, resolveAppEnv, } from './platform-diff.js';
 /**
  * #274 fix round 4: bridges `@wtfalch/providers`' real `ZitadelClient.
  * listApps` -- a plain array, with no `loginBaseUri` or completeness signal
@@ -47,6 +48,14 @@ export class NotBuiltError extends Error {
 // ---------------------------------------------------------------------------
 // Hostnames
 // ---------------------------------------------------------------------------
+/**
+ * The Coolify name of a hosting entry's app: `<slug>` for the `app` entry, so
+ * the move to a list finds the app that runs today, `<slug>-<key>` for any
+ * other.
+ */
+function appNameFor(org, entry) {
+    return entry.key === APP_ENTRY_KEY ? org.slug : `${org.slug}-${entry.key}`;
+}
 /**
  * `<slug>.oikos.host` (`../slug.ts`), or `DeclaredOrg.domain` once
  * `hosting.usesDomain` is true — the same rule `diff.ts`'s `planApp` and
@@ -338,36 +347,94 @@ const CLOUDFLARE_TUNNEL_DNS_DOMAIN_SCOPE_KEY = 'cloudflare_tunnel_dns_domain';
  */
 const CLOUDFLARE_TUNNEL_DNS_HOSTNAMES_SCOPE_KEY = 'cloudflare_tunnel_dns_hostnames';
 /**
- * #236 round 5 finding 1: the domain, and the hostnames under it,
- * {@link FoundryStepExecutor.publishTunnelDnsRecords} is in the middle of
- * writing CNAMEs for -- recorded after *each* individual CNAME write
- * succeeds, well before the step as a whole returns and
- * {@link CLOUDFLARE_TUNNEL_DNS_DOMAIN_SCOPE_KEY}/
- * {@link CLOUDFLARE_TUNNEL_DNS_HOSTNAMES_SCOPE_KEY} themselves are updated
- * (which, on a domain change, waits for the *old* domain's cleanup to
- * succeed too -- round 4 finding 3). Without this, a `createDnsRecord`
- * failure partway through a hostname list, or an old-domain cleanup failure
- * right after a successful publish, left an already-created CNAME out of
- * every recorded list, so `destroy` had no way to find and remove it.
- * `destroy` checks this pair in addition to the authoritative one above.
- * Never read by `cloudflareTunnel` itself to decide anything -- it is
- * destroy's (and a retried apply's own idempotent re-check's) safety net,
- * not a second source of truth to branch planning on.
+ * #236 round 6 finding 1: every `${domain}|${hostname}` pair
+ * {@link FoundryStepExecutor.publishTunnelDnsRecords} or
+ * {@link FoundryStepExecutor.deleteStaleTunnelDnsRecords} has written (or
+ * is in the middle of writing) a CNAME for -- across every apply, and
+ * across however many domains an org has moved through without a clean
+ * resolution, not only the most recent one. Replaces round 5's single
+ * pending domain/hostnames pair (`cloudflare_tunnel_dns_pending_domain`/
+ * `_hostnames`), which tracked one domain at a time and was overwritten
+ * whole on every call: apply 1 moves domain A to B, creates `h1.B`, and
+ * old-domain cleanup throws, leaving pending at `B/[h1]`; the org then
+ * declares C, and apply 2's own pending write overwrites it with
+ * `C/[...]` -- `h1.B` is now in no recorded list at all, so `destroy`
+ * leaves it live forever. This key instead accumulates entries (adds,
+ * never replaces the set) via {@link parseTrackedTunnelDns}/
+ * {@link trackedTunnelDnsState}, and an entry is dropped only once its own
+ * CNAME is deleted, confirmed absent, or confirmed not this tunnel's.
+ * `destroy` and {@link FoundryStepExecutor.deleteStaleTunnelDnsRecords}
+ * both walk every domain this set still names, not only the org's current
+ * or last-recorded one. Neither a domain nor a DNS hostname can contain
+ * `,` or `|`, so the join is unambiguous; sorted (like
+ * {@link LOGIN_HOST_PENDING_REMOVAL_SCOPE_KEY}, `login-host.ts`) so the
+ * recorded value is stable regardless of write order.
  */
-const CLOUDFLARE_TUNNEL_DNS_PENDING_DOMAIN_SCOPE_KEY = 'cloudflare_tunnel_dns_pending_domain';
-const CLOUDFLARE_TUNNEL_DNS_PENDING_HOSTNAMES_SCOPE_KEY = 'cloudflare_tunnel_dns_pending_hostnames';
+const CLOUDFLARE_TUNNEL_DNS_TRACKED_SCOPE_KEY = 'cloudflare_tunnel_dns_tracked';
+/** {@link CLOUDFLARE_TUNNEL_DNS_TRACKED_SCOPE_KEY}'s entries, parsed back out. */
+function parseTrackedTunnelDns(recorded) {
+    const raw = recorded[CLOUDFLARE_TUNNEL_DNS_TRACKED_SCOPE_KEY];
+    return new Set(raw ? raw.split(',').filter((entry) => entry.length > 0) : []);
+}
+/** One {@link CLOUDFLARE_TUNNEL_DNS_TRACKED_SCOPE_KEY} entry for `domain`'s `hostname`. */
+function trackedTunnelDnsEntry(domain, hostname) {
+    return `${domain}|${hostname}`;
+}
+/** {@link CLOUDFLARE_TUNNEL_DNS_TRACKED_SCOPE_KEY}'s own serialisation, sorted so the recorded value is stable regardless of write order. */
+function trackedTunnelDnsState(entries) {
+    return [...entries].sort().join(',');
+}
+/** {@link parseTrackedTunnelDns}'s entries, grouped by domain so cleanup can walk one zone at a time. */
+function trackedTunnelDnsByDomain(entries) {
+    const byDomain = new Map();
+    for (const entry of entries) {
+        const separator = entry.indexOf('|');
+        if (separator === -1)
+            continue;
+        const domain = entry.slice(0, separator);
+        const hostname = entry.slice(separator + 1);
+        const hostnames = byDomain.get(domain) ?? [];
+        hostnames.push(hostname);
+        byDomain.set(domain, hostnames);
+    }
+    return byDomain;
+}
 /**
- * #236 round 5 finding 2: the tunnel id this step most recently replaced
- * with a new one -- a recreate, `findTunnelByName` returning a different id
- * for the same name -- recorded unconditionally, even on an apply where
- * `org.domain` is unset and the DNS-publish block below never runs. A CNAME
- * published while a domain *was* declared can still be live under that old
- * id after the domain is later cleared, so `destroy` needs this even then.
- * `publishTunnelDnsRecords`'s own repoint logic already accepts either the
- * current tunnel id or this one when matching a CNAME as this org's own
- * rather than a foreign conflict (round 3); `destroy` now does the same.
+ * #236 round 6 finding 2: every tunnel id this step has ever replaced with
+ * a new one, not only the immediately prior one. Round 5's singular
+ * `cloudflare_tunnel_previous_id` was overwritten on each recreate, so two
+ * recreates with no successful DNS resolution in between (an apply with
+ * no domain declared, or one that keeps failing) left the CNAMEs from the
+ * *first* recreate matched by nothing -- `destroy` and this step's own
+ * repoint logic only ever compared against the latest id. Recorded
+ * unconditionally, even on an apply where `org.domain` is unset and the
+ * DNS-publish block in {@link FoundryStepExecutor.cloudflareTunnel} never
+ * runs: a CNAME published while a domain *was* declared can still be live
+ * under an old id after the domain is later cleared. Comma-joined, sorted,
+ * the same shape {@link CLOUDFLARE_TUNNEL_DNS_TRACKED_SCOPE_KEY} above
+ * uses (tunnel ids carry no `,`). Cleared once that DNS-publish block
+ * completes *and* (round 7) the tracked set names no domain but the one
+ * just published -- only then has every tracked CNAME that could have
+ * pointed at any of them been repointed to the current id or cleaned up.
+ * A tracked entry left under some other domain by an earlier apply whose
+ * own cleanup never ran, or itself threw, can still point at an old id, so
+ * clearing unconditionally (round 6) left `destroy` unable to recognise
+ * that CNAME as this tunnel's own once a later apply resolved a different
+ * domain change and moved on. An apply with no domain declared never
+ * reaches that point either, so the set is left to accumulate across such
+ * applies, and across an unresolved domain change, until one apply
+ * publishes with nothing tracked outside its own domain.
  */
-const CLOUDFLARE_TUNNEL_PREVIOUS_ID_SCOPE_KEY = 'cloudflare_tunnel_previous_id';
+const CLOUDFLARE_TUNNEL_PREVIOUS_IDS_SCOPE_KEY = 'cloudflare_tunnel_previous_ids';
+/** {@link CLOUDFLARE_TUNNEL_PREVIOUS_IDS_SCOPE_KEY}'s entries, parsed back out. */
+function parseTunnelPreviousIds(recorded) {
+    const raw = recorded[CLOUDFLARE_TUNNEL_PREVIOUS_IDS_SCOPE_KEY];
+    return new Set(raw ? raw.split(',').filter((id) => id.length > 0) : []);
+}
+/** {@link CLOUDFLARE_TUNNEL_PREVIOUS_IDS_SCOPE_KEY}'s own serialisation, sorted so the recorded value is stable regardless of write order. */
+function tunnelPreviousIdsState(ids) {
+    return [...ids].sort().join(',');
+}
 /**
  * #162: the owner's own mailbox, created once inside `coolify:mail` after
  * `applyOrgMail` succeeds, alongside {@link MAIL_DOMAIN_SCOPE_KEY} -- another
@@ -562,7 +629,7 @@ export class FoundryStepExecutor {
     }
     /**
      * `plan/diff.ts`'s `planAccess` (#84): puts the org's app hostname behind a
-     * Cloudflare Access app that lets only `hosting.accessEmails` through.
+     * Cloudflare Access app that lets only the `app` entry's `accessEmails` through.
      * foundry names its Access app `foundry:<orgId>` and reuses only an app with
      * that name, so a re-run after a crash does not duplicate it. It refuses,
      * rather than adopt, when another Access app already holds the hostname:
@@ -588,7 +655,7 @@ export class FoundryStepExecutor {
             await cf.createAccessPolicy(app.id, {
                 name: ACCESS_POLICY_NAME,
                 decision: 'allow',
-                include: org.hosting.accessEmails.map((email) => ({ email: { email } })),
+                include: (appEntry(org.hosting)?.accessEmails ?? []).map((email) => ({ email: { email } })),
             });
         }
         return { resourceId: app.id };
@@ -1036,23 +1103,42 @@ export class FoundryStepExecutor {
         return server;
     }
     async coolifyApp(action, org, context) {
+        const recordedAppId = context.recordedIds()[`coolify:app:${action.key}`];
         if (action.desired.enabled === false) {
-            const appId = context.recordedIds()[`coolify:app:${action.key}`];
-            if (!appId) {
+            if (!recordedAppId) {
                 throw new Error(`coolify:app for ${org.slug}: no live app recorded, nothing to park`);
             }
-            await this.ports.coolify.raw('POST', `/applications/${appId}/stop`);
-            return { resourceId: appId };
+            await this.ports.coolify.raw('POST', `/applications/${recordedAppId}/stop`);
+            return { resourceId: recordedAppId };
         }
+        const entry = appEntry(org.hosting);
+        if (!entry)
+            throw new Error(`coolify:app for ${org.slug}: no '${APP_ENTRY_KEY}' hosting entry`);
         // A level-1 org with no bespoke app of its own gets the shared standard
-        // app instead (gap P1, `.plans/2026-09-26-level-1-org-map.md`), in the
-        // same repo shape `bespokeAppRepo` itself uses.
-        const appRepo = org.bespokeAppRepo ?? DEFAULT_APP_REPO;
+        // app instead (gap P1, `.plans/2026-09-26-level-1-org-map.md`): an entry
+        // with no `repo`.
+        const appRepo = entry.repo ?? DEFAULT_APP_REPO;
         const projectUuid = await this.ensureCoolifyProject(org, context);
         const { environment } = await this.ports.coolify.findOrCreateEnvironment(projectUuid, 'production');
-        const existing = await this.ports.coolify.findApplicationByName(org.slug, {
-            environmentIds: [environment.id],
-        });
+        // The recorded id first; the name when nothing is recorded, or when the
+        // recorded app is gone.
+        let existing = null;
+        let live;
+        if (recordedAppId) {
+            try {
+                live = await this.ports.coolify.getApplication(recordedAppId);
+                existing = { uuid: recordedAppId };
+            }
+            catch (error) {
+                if (!isNotFoundError(error))
+                    throw error;
+            }
+        }
+        if (!existing) {
+            existing = await this.ports.coolify.findApplicationByName(appNameFor(org, entry), {
+                environmentIds: [environment.id],
+            });
+        }
         let appId;
         if (existing) {
             appId = existing.uuid;
@@ -1062,7 +1148,7 @@ export class FoundryStepExecutor {
             // stops the app without deleting it, so re-enabling found it here,
             // still stopped, and it stayed stopped forever despite `apply`
             // reporting success.
-            const live = await this.ports.coolify.getApplication(existing.uuid);
+            live ??= await this.ports.coolify.getApplication(existing.uuid);
             if (!isRunningOrStarting(live.status)) {
                 await this.ports.coolify.raw('POST', `/applications/${existing.uuid}/start`);
             }
@@ -1077,13 +1163,13 @@ export class FoundryStepExecutor {
                 environment_uuid: environment.uuid,
                 github_app_uuid: githubApp.uuid,
                 git_repository: appRepo,
-                git_branch: org.hosting.gitBranch,
-                build_pack: org.hosting.buildPack,
-                name: org.slug,
+                git_branch: entry.branch,
+                build_pack: entry.buildPack,
+                name: appNameFor(org, entry),
                 domains: hostnameFor(org),
                 // Only the shared standard app builds from this repo; a bespoke
                 // app's Dockerfile is at its own repo's root.
-                dockerfile_location: org.bespokeAppRepo ? undefined : DEFAULT_APP_DOCKERFILE,
+                dockerfile_location: entry.repo ? undefined : DEFAULT_APP_DOCKERFILE,
             });
             appId = created.uuid;
         }
@@ -1093,8 +1179,8 @@ export class FoundryStepExecutor {
         // covers the database being turned on after the app already exists (the
         // app step still ran this apply, found the app by name, and lands here).
         let dbPush = { pushed: false, changed: false };
-        if (org.hosting.database) {
-            const dbId = context.recordedIds()[`coolify:database:${action.key}`];
+        if (entry.database) {
+            const dbId = context.recordedIds()[`coolify:database:${org.id}`];
             if (dbId) {
                 dbPush = await this.pushDatabaseUrl(appId, dbId, org.slug.replace(/-/g, '_'));
             }
@@ -1114,7 +1200,7 @@ export class FoundryStepExecutor {
         // The app receives whatever it needs to sign in (gap P2). Unconditional,
         // unlike the database push above: every app has a hostname, and an
         // upsert of the same value on a later apply is a no-op, not a change.
-        const appEnv = await this.pushAppEnv(appId, org, context);
+        const appEnv = await this.pushAppEnv(appId, org, entry, context);
         // Rule 4 (#239, #243): a container reads its env only at start, so an app
         // that was already there is restarted, once, when its database URLs or its
         // declared appEnv changed.
@@ -1207,7 +1293,7 @@ export class FoundryStepExecutor {
         // Handles the database being provisioned after the app already exists
         // (`coolifyApp`'s own push handles the more common order, database
         // before app on the same apply -- `diff.ts`'s `computePlanDetail`).
-        const appId = context.recordedIds()[`coolify:app:${action.key}`];
+        const appId = context.recordedIds()[`coolify:app:${appStepKey(org)}`];
         if (appId) {
             const push = await this.pushDatabaseUrl(appId, dbId, org.slug.replace(/-/g, '_'));
             if (push.pushed) {
@@ -1288,7 +1374,7 @@ export class FoundryStepExecutor {
      * P2, `.plans/2026-09-26-level-1-org-map.md`) -- a second `bulkUpsertEnvs`
      * call rather than folding into `pushDatabaseUrl`'s, since this one runs
      * unconditionally (`coolifyApp`) where that one runs only once
-     * `hosting.database` is declared and provisioned.
+     * the `app` entry's `database` is declared and provisioned.
      *
      * `APP_URL` and `LOG_DIR` (#75) are pushed unconditionally; every other
      * value is pushed only once its own source step has actually recorded it
@@ -1301,10 +1387,10 @@ export class FoundryStepExecutor {
      * is its own design question, left out of #204's own "How" rather than
      * guessed) is out of scope here (task brief). `DATABASE_URL_OWNER`/
      * `DATABASE_URL` are pushed by `pushDatabaseUrl` instead, not here --
-     * that call runs only once `hosting.database` is declared and
+     * that call runs only once the `app` entry's `database` is declared and
      * provisioned, where this one is unconditional (#239).
      */
-    async pushAppEnv(appId, org, context) {
+    async pushAppEnv(appId, org, entry, context) {
         const recorded = context.recordedIds();
         const envs = [
             { key: 'APP_URL', value: `https://${hostnameFor(org)}` },
@@ -1339,7 +1425,7 @@ export class FoundryStepExecutor {
         // minted here; a source with no value yet (its own step has not run,
         // or has not run for this apply) is simply omitted, the same way every
         // other optional entry above is.
-        const appEnv = resolveAppEnv(org, recorded).values;
+        const appEnv = resolveAppEnv(org, entry, recorded).values;
         for (const [key, value] of Object.entries(appEnv))
             envs.push({ key, value });
         await this.ports.coolify.bulkUpsertEnvs(appId, envs);
@@ -2447,16 +2533,19 @@ export class FoundryStepExecutor {
             return { resourceId: tunnelId };
         }
         const tunnelName = `${org.slug}-tunnel`;
-        // #236 round 5 finding 2: read, and (if it is about to change) recorded,
-        // before `createTunnel` can replace it -- unconditionally, not only
-        // inside the `if (org.domain)` block below, so a recreate on an apply
-        // with no domain declared still leaves `destroy` a way to recognise the
-        // old id's CNAMEs as this tunnel's own rather than a foreign record.
+        // #236 round 5 finding 2 / round 6 finding 2: read, and (if it is about
+        // to change) added to the accumulated replaced-id set, before
+        // `createTunnel` can replace it -- unconditionally, not only inside the
+        // `if (org.domain)` block below, so a recreate on an apply with no
+        // domain declared still leaves `destroy` a way to recognise the old
+        // id's CNAMEs as this tunnel's own rather than a foreign record.
         const recordedTunnelId = context.recordedIds()[`cloudflare:tunnel:${action.key}`];
         const existing = await this.ports.cloudflare.findTunnelByName(tunnelName);
         const tunnel = existing ?? (await this.ports.cloudflare.createTunnel(tunnelName));
+        const previousIds = parseTunnelPreviousIds(context.recordedIds());
         if (recordedTunnelId && recordedTunnelId !== tunnel.id) {
-            context.recordScopeId(CLOUDFLARE_TUNNEL_PREVIOUS_ID_SCOPE_KEY, recordedTunnelId);
+            previousIds.add(recordedTunnelId);
+            context.recordScopeId(CLOUDFLARE_TUNNEL_PREVIOUS_IDS_SCOPE_KEY, tunnelPreviousIdsState(previousIds));
         }
         // Routes each of the org's hostnames (`tunnelIngressHostnames`) to
         // Coolify's own Traefik reverse proxy, not a per-app container address --
@@ -2541,12 +2630,12 @@ export class FoundryStepExecutor {
             const previousDomain = recordedBefore[CLOUDFLARE_TUNNEL_DNS_DOMAIN_SCOPE_KEY];
             const previousHostnamesRaw = recordedBefore[CLOUDFLARE_TUNNEL_DNS_HOSTNAMES_SCOPE_KEY];
             const previousHostnames = previousHostnamesRaw ? previousHostnamesRaw.split(',') : [];
-            // Same value `recordedTunnelId` captured above, before `createTunnel`
-            // could replace it -- not re-read here, since this block's own
-            // `context.recordScopeId` calls never touch that key.
-            const previousTunnelId = recordedTunnelId;
             const domainChanged = previousDomain !== undefined && previousDomain !== org.domain;
-            const publishedHostnames = await this.publishTunnelDnsRecords(org, org.domain, tunnel.id, previousTunnelId, 
+            const publishedHostnames = await this.publishTunnelDnsRecords(org, org.domain, tunnel.id, 
+            // Round 6 finding 2: every id this step has ever replaced, not only
+            // the one `recordedTunnelId` captured above -- see the scope key's
+            // own doc comment.
+            previousIds, 
             // Round 4 finding 2: a hostname this domain itself dropped (no domain
             // change) is cleaned up inline by `publishTunnelDnsRecords`, which
             // needs the domain's own previous hostname list for that. A domain
@@ -2554,17 +2643,31 @@ export class FoundryStepExecutor {
             // empty list here rather than a second, redundant pass over it.
             domainChanged ? [] : previousHostnames, context);
             if (domainChanged) {
-                await this.deleteStaleTunnelDnsRecords(tunnel.id, previousTunnelId, previousDomain, previousHostnames);
+                await this.deleteStaleTunnelDnsRecords(tunnel.id, previousIds, previousDomain, previousHostnames, context);
             }
             context.recordScopeId(CLOUDFLARE_TUNNEL_DNS_DOMAIN_SCOPE_KEY, org.domain);
             context.recordScopeId(CLOUDFLARE_TUNNEL_DNS_HOSTNAMES_SCOPE_KEY, publishedHostnames.join(','));
-            // #236 round 5 finding 1: the authoritative pair above now carries
-            // everything the pending pair was tracking mid-write -- clear it so a
-            // later destroy (or crash-recovery apply) never reads a stale pending
-            // domain that no longer matches reality. A no-op the first time this
-            // step ever runs (both keys already absent).
-            context.recordScopeId(CLOUDFLARE_TUNNEL_DNS_PENDING_DOMAIN_SCOPE_KEY, '');
-            context.recordScopeId(CLOUDFLARE_TUNNEL_DNS_PENDING_HOSTNAMES_SCOPE_KEY, '');
+            // #236 round 7: every previous id's CNAMEs *under `org.domain`* have
+            // now been repointed to `tunnel.id` or cleaned up by publish/cleanup
+            // above (current-domain hostnames by the write loop inside
+            // `publishTunnelDnsRecords`, the old domain's by
+            // `deleteStaleTunnelDnsRecords` above) -- but a tracked entry left
+            // under some *other* domain by an earlier apply whose own cleanup
+            // never ran, or itself threw, can still point at one of `previousIds`.
+            // Round 6 finding 2's unconditional clear missed exactly that: apply 1
+            // moves domain A to B, creates `h1.B` on tunnel T1, and A's cleanup
+            // throws; the tunnel is then recreated as T2 out of band and the org
+            // declares domain C; apply 2 publishes C and cleans up only A, leaving
+            // `h1.B` tracked and still pointed at T1 -- clearing the id set here
+            // would make `destroy` and the next `deleteStaleTunnelDnsRecords`
+            // match only T2 and leave `h1.B` live forever. So the set is cleared
+            // only once the tracked set names no domain but the one just
+            // published: the one case where nothing tracked can still reference
+            // an old id. A no-op the first time this step ever runs.
+            const trackedDomains = trackedTunnelDnsByDomain(parseTrackedTunnelDns(context.recordedIds())).keys();
+            if ([...trackedDomains].every((domain) => domain === org.domain)) {
+                context.recordScopeId(CLOUDFLARE_TUNNEL_PREVIOUS_IDS_SCOPE_KEY, '');
+            }
         }
         return { resourceId: tunnel.id };
     }
@@ -2587,23 +2690,36 @@ export class FoundryStepExecutor {
      * existed falls back to the old zone-wide pass, the one case where no
      * hostname list was ever written to look up instead.
      *
-     * `previousTunnelId` (round 4 finding 1): the old domain's CNAMEs were
-     * published under whatever tunnel id was live the *last* time this step
-     * recorded that domain, which is `previousTunnelId`, not necessarily
-     * `tunnelId` -- a tunnel recreated under the same name
-     * (`findTunnelByName` returning a new id) in the same apply that also
-     * changes the domain leaves the old domain's records pointed at the old
-     * id. Matching only `tunnelId`'s target left them live forever. Both
-     * targets are checked so this stays a no-op once a retry has already
-     * moved the old domain onto the new id.
+     * `previousIds` (round 4 finding 1, round 6 finding 2): the old domain's
+     * CNAMEs were published under whatever tunnel id was live the *last*
+     * time this step recorded that domain, which may be any id this step has
+     * ever replaced, not necessarily `tunnelId` -- a tunnel recreated under
+     * the same name (`findTunnelByName` returning a new id), possibly more
+     * than once with no successful publish in between, in the same apply
+     * that also changes the domain leaves the old domain's records pointed
+     * at an old id. Matching only `tunnelId`'s target left them live
+     * forever; matching only the single most recent previous id (round 4)
+     * still missed an older one two unresolved recreates back. Every target
+     * is checked so this stays a no-op once a retry has already moved the
+     * old domain onto the current id.
+     *
+     * Round 6 finding 1: once `previousHostnames` is non-empty, every name in
+     * it has now been looked up -- deleted (its CNAME matched one of
+     * `dnsTargets`), left alone (found but foreign), or never found at all
+     * (already gone) -- so none of them needs tracking under `previousDomain`
+     * in {@link CLOUDFLARE_TUNNEL_DNS_TRACKED_SCOPE_KEY} any longer; dropped
+     * from that set accordingly. The zone-wide fallback path (no recorded
+     * hostnames) never added entries of its own to begin with, so there is
+     * nothing of this domain's left in that set either way.
      */
-    async deleteStaleTunnelDnsRecords(tunnelId, previousTunnelId, previousDomain, previousHostnames) {
+    async deleteStaleTunnelDnsRecords(tunnelId, previousIds, previousDomain, previousHostnames, context) {
         const zone = await this.ports.cloudflare.findZoneByApex(previousDomain);
         if (!zone)
             return;
         const dnsTargets = new Set([`${tunnelId}.cfargotunnel.com`]);
-        if (previousTunnelId && previousTunnelId !== tunnelId) {
-            dnsTargets.add(`${previousTunnelId}.cfargotunnel.com`);
+        for (const id of previousIds) {
+            if (id !== tunnelId)
+                dnsTargets.add(`${id}.cfargotunnel.com`);
         }
         const cf = this.ports.cloudflare;
         const records = previousHostnames.length > 0
@@ -2614,6 +2730,13 @@ export class FoundryStepExecutor {
                 await cf.deleteDnsRecord(zone.id, record.id);
             }
         }
+        if (previousHostnames.length > 0) {
+            const tracked = parseTrackedTunnelDns(context.recordedIds());
+            for (const hostname of previousHostnames) {
+                tracked.delete(trackedTunnelDnsEntry(previousDomain, hostname));
+            }
+            context.recordScopeId(CLOUDFLARE_TUNNEL_DNS_TRACKED_SCOPE_KEY, trackedTunnelDnsState(tracked));
+        }
     }
     /**
      * #236: upserts a proxied CNAME to this tunnel for each of
@@ -2639,13 +2762,15 @@ export class FoundryStepExecutor {
      * (round 4 finding 3), so a crash mid-write is recovered by re-running
      * this same idempotent step, never silently skipped.
      *
-     * Round 3: `previousTunnelId` is the tunnel id this same step recorded
-     * last time it ran (`cloudflareTunnel`'s own `cloudflare:tunnel:<key>`
-     * resource id, read before `tunnel` above is computed for this run) -- a
-     * tunnel Cloudflare shows as recreated under the same name
-     * (`findTunnelByName` returning a new id for `<slug>-tunnel`) leaves every
-     * CNAME still pointed at the *old* id. That is this org's own stale
-     * record, not a foreign one, so it is repointed rather than refused.
+     * Round 3, round 6 finding 2: `previousIds` is every tunnel id this same
+     * step has ever replaced (`cloudflareTunnel`'s own accumulated set,
+     * captured before `tunnel` above is computed for this run) -- a tunnel
+     * Cloudflare shows as recreated under the same name (`findTunnelByName`
+     * returning a new id for `<slug>-tunnel`), possibly more than once with
+     * no successful publish in between, leaves every CNAME still pointed at
+     * one of those old ids. That is this org's own stale record, not a
+     * foreign one, so it is repointed rather than refused, whichever of
+     * `previousIds` it still names.
      *
      * Round 4 finding 2: `previousHostnames` is this domain's own previously
      * recorded hostname list (empty when the domain just changed --
@@ -2655,28 +2780,28 @@ export class FoundryStepExecutor {
      * `hostnames` below but was never cleaned up before this round: its CNAME
      * stayed live and its name stayed in the recorded list forever, because
      * only a domain *change* ever ran cleanup. Each dropped name's CNAME is
-     * deleted here when it still points at this tunnel or its immediately
-     * prior id; a name already gone, or now pointed at something foreign, is
-     * simply left out of the returned list -- nothing here is this method's to
-     * reclaim or report as a conflict.
+     * deleted here when it still points at this tunnel or any of
+     * `previousIds`; a name already gone, or now pointed at something
+     * foreign, is simply left out of the returned list -- nothing here is
+     * this method's to reclaim or report as a conflict.
      *
-     * Round 5 finding 1: `context` is used only to record
-     * {@link CLOUDFLARE_TUNNEL_DNS_PENDING_DOMAIN_SCOPE_KEY}/
-     * {@link CLOUDFLARE_TUNNEL_DNS_PENDING_HOSTNAMES_SCOPE_KEY} after each
-     * individual CNAME write below succeeds, so a crash or a thrown conflict
-     * partway through never leaves an already-created CNAME out of every
-     * recorded list -- `destroy` checks this pending pair too.
+     * Round 5 finding 1, round 6 finding 1: `context` is used to add every
+     * hostname this call confirms correct (settled, created or repointed) to
+     * {@link CLOUDFLARE_TUNNEL_DNS_TRACKED_SCOPE_KEY} as soon as it is
+     * confirmed, and to drop a dropped hostname from it once that hostname's
+     * own CNAME is resolved one way or another -- additively, alongside
+     * whatever other domains that shared set already names, never
+     * overwriting them. `destroy` and {@link deleteStaleTunnelDnsRecords}
+     * both read this set.
      */
-    async publishTunnelDnsRecords(org, domain, tunnelId, previousTunnelId, previousHostnames, context) {
+    async publishTunnelDnsRecords(org, domain, tunnelId, previousIds, previousHostnames, context) {
         const cf = this.ports.cloudflare;
         const zone = await cf.findZoneByApex(domain);
         if (!zone) {
             throw new Error(`cloudflare:tunnel for ${org.slug}: no Cloudflare zone found for ${domain} to point its hostnames at -- cloudflare:zone must provision it first`);
         }
         const dnsTarget = `${tunnelId}.cfargotunnel.com`;
-        const previousDnsTarget = previousTunnelId && previousTunnelId !== tunnelId
-            ? `${previousTunnelId}.cfargotunnel.com`
-            : undefined;
+        const previousTargets = new Set([...previousIds].filter((id) => id !== tunnelId).map((id) => `${id}.cfargotunnel.com`));
         const hostnames = tunnelIngressHostnames(org).filter((hostname) => hostname === domain || hostname.endsWith(`.${domain}`));
         const droppedHostnames = previousHostnames.filter((hostname) => !hostnames.includes(hostname));
         const conflicts = [];
@@ -2692,7 +2817,7 @@ export class FoundryStepExecutor {
             }
             const [cname] = await cf.findDnsRecord(zone.id, hostname, 'CNAME');
             if (cname && cname.content !== dnsTarget) {
-                if (previousDnsTarget && cname.content === previousDnsTarget) {
+                if (previousTargets.has(cname.content)) {
                     toRepoint.push({ hostname, recordId: cname.id });
                     continue;
                 }
@@ -2705,20 +2830,22 @@ export class FoundryStepExecutor {
         if (conflicts.length > 0) {
             throw new Error(`cloudflare:tunnel for ${org.slug}: will not point its tunnel at every hostname -- ${conflicts.join('; ')}; remove the conflicting record first`);
         }
-        // Round 5 finding 1: `settled` tracks every hostname under `domain`
-        // confirmed to have the right CNAME so far -- starting with the ones
-        // that needed no write at all, then one more per successful create or
-        // repoint below -- and is persisted to the pending scope ids after each
-        // addition, well before this method returns. A failure on any later
-        // hostname still leaves everything settled up to that point recorded,
-        // rather than only in this function's own local state.
+        // Round 5 finding 1, round 6 finding 1: `settled` is every hostname
+        // under `domain` confirmed to have the right CNAME so far -- starting
+        // with the ones that needed no write at all, then one more per
+        // successful create or repoint below -- and each is added to the
+        // shared tracked set as soon as it is confirmed, well before this
+        // method returns. A failure on any later hostname still leaves
+        // everything settled up to that point recorded, rather than only in
+        // this function's own local state.
         const settled = hostnames.filter((hostname) => !toCreate.includes(hostname) && !toRepoint.some((r) => r.hostname === hostname));
-        const recordPending = () => {
-            context.recordScopeId(CLOUDFLARE_TUNNEL_DNS_PENDING_DOMAIN_SCOPE_KEY, domain);
-            context.recordScopeId(CLOUDFLARE_TUNNEL_DNS_PENDING_HOSTNAMES_SCOPE_KEY, settled.join(','));
+        const tracked = parseTrackedTunnelDns(context.recordedIds());
+        const recordTracked = (hostname) => {
+            tracked.add(trackedTunnelDnsEntry(domain, hostname));
+            context.recordScopeId(CLOUDFLARE_TUNNEL_DNS_TRACKED_SCOPE_KEY, trackedTunnelDnsState(tracked));
         };
-        if (settled.length > 0)
-            recordPending();
+        for (const hostname of settled)
+            recordTracked(hostname);
         for (const hostname of toCreate) {
             await cf.createDnsRecord(zone.id, {
                 type: 'CNAME',
@@ -2726,8 +2853,7 @@ export class FoundryStepExecutor {
                 content: dnsTarget,
                 proxied: true,
             });
-            settled.push(hostname);
-            recordPending();
+            recordTracked(hostname);
         }
         for (const { hostname, recordId } of toRepoint) {
             await cf.updateDnsRecord(zone.id, recordId, {
@@ -2736,18 +2862,25 @@ export class FoundryStepExecutor {
                 content: dnsTarget,
                 proxied: true,
             });
-            settled.push(hostname);
-            recordPending();
+            recordTracked(hostname);
         }
         // Round 4 finding 2: clean up a hostname this domain itself dropped.
         // Not a conflict check -- there is nothing to create or repoint here,
         // only a possible stale CNAME of this tunnel's own to remove. Never in
-        // `hostnames`/`settled` to begin with, so nothing to update there.
+        // `hostnames`/`settled` to begin with, so nothing to track-add there.
+        //
+        // Round 6 finding 1: whatever happens -- deleted (its CNAME matched),
+        // left alone (found but foreign), or never found at all (already gone)
+        // -- this hostname's own entry (if it had one) no longer needs
+        // tracking under `domain`, so it is dropped from the shared set either
+        // way.
         for (const hostname of droppedHostnames) {
             const [cname] = await cf.findDnsRecord(zone.id, hostname, 'CNAME');
-            if (cname && (cname.content === dnsTarget || cname.content === previousDnsTarget)) {
+            if (cname && (cname.content === dnsTarget || previousTargets.has(cname.content))) {
                 await cf.deleteDnsRecord(zone.id, cname.id);
             }
+            tracked.delete(trackedTunnelDnsEntry(domain, hostname));
+            context.recordScopeId(CLOUDFLARE_TUNNEL_DNS_TRACKED_SCOPE_KEY, trackedTunnelDnsState(tracked));
         }
         return hostnames;
     }
@@ -3213,46 +3346,46 @@ export class FoundryStepExecutor {
                     // look up instead. 404 handled locally, same reason as the
                     // connector above.
                     //
-                    // Round 5 finding 2: `dnsTargets` accepts this tunnel's current id
-                    // *or* {@link CLOUDFLARE_TUNNEL_PREVIOUS_ID_SCOPE_KEY} -- the id a
-                    // recreate most recently replaced -- the same two-target match
-                    // `publishTunnelDnsRecords`'s own repoint logic already uses, so a
-                    // CNAME a domain-less apply's recreate left pointed at the old id
-                    // (never repointed, since that apply never ran the DNS-publish
-                    // block at all) still gets removed here.
-                    //
-                    // Round 5 finding 1: `dnsSources` also checks the *pending*
-                    // domain/hostnames pair alongside the authoritative one, so a
-                    // CNAME `publishTunnelDnsRecords` created but never got to record
-                    // in the authoritative pair (a later hostname's write failing, or
-                    // an old-domain cleanup failure right after a successful publish)
-                    // is still found and removed.
-                    const previousTunnelId = recordedIds[CLOUDFLARE_TUNNEL_PREVIOUS_ID_SCOPE_KEY];
+                    // Round 5 finding 2, round 6 finding 2: `dnsTargets` accepts this
+                    // tunnel's current id *or* any id
+                    // {@link CLOUDFLARE_TUNNEL_PREVIOUS_IDS_SCOPE_KEY} still names --
+                    // every id this step has ever replaced, not only the latest -- the
+                    // same match `publishTunnelDnsRecords`'s own repoint logic already
+                    // uses, so a CNAME a domain-less apply's recreate left pointed at
+                    // an old id (never repointed, since that apply never ran the
+                    // DNS-publish block at all), possibly more than one recreate back,
+                    // still gets removed here.
+                    const previousTunnelIds = parseTunnelPreviousIds(recordedIds);
                     const dnsTargets = new Set([`${target.id}.cfargotunnel.com`]);
-                    if (previousTunnelId && previousTunnelId !== target.id) {
-                        dnsTargets.add(`${previousTunnelId}.cfargotunnel.com`);
+                    for (const id of previousTunnelIds) {
+                        if (id !== target.id)
+                            dnsTargets.add(`${id}.cfargotunnel.com`);
+                    }
+                    // Round 6 finding 1: every domain the shared tracked set still
+                    // names, folded together with the authoritative domain/hostnames
+                    // pair (an org that predates that set has only the latter). A
+                    // domain the tracked set names but the authoritative pair does
+                    // not is exactly the case this round exists for -- a hostname
+                    // published under a domain the org has since moved away from
+                    // twice over, never caught by any single "previous domain" field.
+                    const hostnamesByDomain = trackedTunnelDnsByDomain(parseTrackedTunnelDns(recordedIds));
+                    const authoritativeDomain = recordedIds[CLOUDFLARE_TUNNEL_DNS_DOMAIN_SCOPE_KEY];
+                    if (authoritativeDomain) {
+                        const authoritativeHostnamesRaw = recordedIds[CLOUDFLARE_TUNNEL_DNS_HOSTNAMES_SCOPE_KEY];
+                        const merged = new Set(hostnamesByDomain.get(authoritativeDomain) ?? []);
+                        if (authoritativeHostnamesRaw) {
+                            for (const hostname of authoritativeHostnamesRaw.split(','))
+                                merged.add(hostname);
+                        }
+                        hostnamesByDomain.set(authoritativeDomain, [...merged]);
                     }
-                    const dnsSources = [
-                        {
-                            domain: recordedIds[CLOUDFLARE_TUNNEL_DNS_DOMAIN_SCOPE_KEY],
-                            hostnamesRaw: recordedIds[CLOUDFLARE_TUNNEL_DNS_HOSTNAMES_SCOPE_KEY],
-                        },
-                        {
-                            domain: recordedIds[CLOUDFLARE_TUNNEL_DNS_PENDING_DOMAIN_SCOPE_KEY],
-                            hostnamesRaw: recordedIds[CLOUDFLARE_TUNNEL_DNS_PENDING_HOSTNAMES_SCOPE_KEY],
-                        },
-                    ];
-                    for (const { domain: dnsDomain, hostnamesRaw: dnsHostnamesRaw } of dnsSources) {
-                        if (!dnsDomain)
-                            continue;
+                    for (const [dnsDomain, dnsHostnames] of hostnamesByDomain) {
                         try {
                             const zone = await this.ports.cloudflare.findZoneByApex(dnsDomain);
                             if (zone) {
                                 const cf = this.ports.cloudflare;
-                                const records = dnsHostnamesRaw
-                                    ? (await Promise.all(dnsHostnamesRaw
-                                        .split(',')
-                                        .map((hostname) => cf.findDnsRecord(zone.id, hostname, 'CNAME')))).flat()
+                                const records = dnsHostnames.length > 0
+                                    ? (await Promise.all(dnsHostnames.map((hostname) => cf.findDnsRecord(zone.id, hostname, 'CNAME')))).flat()
                                     : await cf.listDnsRecords(zone.id, { type: 'CNAME' });
                                 for (const record of records) {
                                     if (dnsTargets.has(record.content)) {