@wtfalch/foundry 0.70.0 → 0.71.0

package/dist/bootstrap/declare.js

@@ -170,6 +170,8 @@ export function declaredPlatformFor(apex, serverIpv4) {
         // only after Archon runs. Declared on with no pin, it would be reported
         // blocked and the bootstrap would never be complete.
         uptime: { enabled: false, tag: '', sha256: '' },
+        // foundry#1024: off until a human pins an edge release tag and digest, the same reason as uptime.
+        usageTail: { enabled: false, tag: '', sha256: '' },
         heartbeatsProvider: 'self-hosted',
     };
 }

package/dist/index.d.ts

@@ -25,6 +25,8 @@ export { createPostgresAuditLog } from './store/audit.js';
 export type { Color, ReleaseAction, ReleaseEvent, ReleaseEventKind, ReleaseSeries, ReleaseSeriesColor, ReleaseSignal, ReleaseState, WindowState, } from './release/types.js';
 export type { ReleaseStore } from './release/state.js';
 export { readReleaseHistory, readReleaseState, requestReleaseAction } from './release/state.js';
+export type { BuiltinFanOut, FanOutApp, FanOutOrgState } from './release/fanout.js';
+export { readBuiltinFanOut } from './release/fanout.js';
 export type { ReleaseSeriesHost, ReleaseSeriesQuery } from './release/series.js';
 export { readReleaseSeries } from './release/series.js';
 export type { EntrySeriesDeps } from './plan/release-series-reader.js';

package/dist/index.js

@@ -15,6 +15,7 @@ export { StaleLeaseError } from './store.js';
 export { createPostgresAppliedVersionsStore, createPostgresDeclaredOrgStore, createPostgresPlatformStore, createPostgresRequestedChangeStore, DeclaredOrgError, PlatformError, } from './store/postgres.js';
 export { createPostgresAuditLog } from './store/audit.js';
 export { readReleaseHistory, readReleaseState, requestReleaseAction } from './release/state.js';
+export { readBuiltinFanOut } from './release/fanout.js';
 export { readReleaseSeries } from './release/series.js';
 export { readEntryReleaseSeries } from './plan/release-series-reader.js';
 export { checkMigrationCompatibility } from './release/migration-compatibility.js';

package/dist/migrations/0070_platform_usage_tail.sql

@@ -0,0 +1,12 @@
+-- @wtfalch/foundry: the usage tail Worker on the declared platform (foundry#1024).
+-- Mirrors `src/store/schema.ts`'s `foundryDeclaredPlatform` additions.
+-- `usage_tail_tag` and `usage_tail_sha256` are the pin of the edge release
+-- whose built Worker foundry deploys; both empty means nothing is pinned
+-- yet, and the planner reports that instead of planning the step. Unlike the
+-- uptime Worker, this one is off by default: nothing has pinned a release
+-- yet on any existing platform.
+
+alter table foundry_declared_platform
+  add column if not exists usage_tail_enabled boolean not null default false,
+  add column if not exists usage_tail_tag text not null default '',
+  add column if not exists usage_tail_sha256 text not null default '';

package/dist/plan/diff.js

@@ -176,6 +176,18 @@ function liveResourceName(r) {
     const type = r.kind === 'dns_record' && typeof r.config.type === 'string' ? `${r.config.type} ` : '';
     return `${type}${name}, ${r.key}`;
 }
+/**
+ * The provider's own name for a live resource, for a park or provision
+ * step's own prose (#1062, pain point 3): an operator told to "park the
+ * database" acted on the company's Coolify server, named after the same
+ * company, instead of the database resource the step meant. `null` when
+ * there is no live resource yet, or it carries no name -- unlike
+ * {@link liveResourceName} above, this never falls back to the bare key,
+ * since an id is not a name a step's own text should ever show.
+ */
+function liveName(live) {
+    return typeof live?.config.name === 'string' && live.config.name !== '' ? live.config.name : null;
+}
 /**
  * The diff engine: #19's "a plan reads live state from Zitadel, Coolify and
  * Cloudflare and diffs it against the declared org", built on #10's rule
@@ -3166,7 +3178,7 @@ function planKosmos(org, byKey, claimed, recordedIds, issuer, otherSlugs) {
     if (!kosmosOn(org)) {
         // Switched off: park a live app (stop it, keep the database), like CMS and calls. A stopped app is
         // already parked, so the plan reaches empty.
-        const parked = planHostingToggle('coolify', 'kosmos', false, org, byKey, claimed, () => `provision ${org.slug}'s kosmos`, () => `park ${org.slug}'s kosmos (stops serving; its database is retained, not deleted)`, org.id, (live) => !/^(exited|stopped)/.test(String(live.config.status ?? '')));
+        const parked = planHostingToggle('coolify', 'kosmos', false, org, byKey, claimed, () => `provision ${org.slug}'s kosmos`, (live) => `park ${org.slug}'s kosmos${liveName(live) ? ` ${liveName(live)}` : ''} (stops serving; its database is retained, not deleted)`, org.id, (live) => !/^(exited|stopped)/.test(String(live.config.status ?? '')));
         return { step: parked, blocked: null };
     }
     const key = resourceKey('coolify', 'kosmos', org.id);
@@ -3533,7 +3545,11 @@ function planBoule(org, byKey, claimed, recordedIds, issuer, otherSlugs, archonU
  * park-not-delete rule (#7 corrections 1) -- turning it off never deletes
  * the org's own app, database or tunnel configuration, it parks it.
  */
-function planHostingToggle(system, kind, desiredEnabled, org, byKey, claimed, describeProvision, describePark, stepKey = org.id, isOn = (live) => live.config.enabled !== false) {
+function planHostingToggle(system, kind, desiredEnabled, org, byKey, claimed, 
+// #1062: takes the live resource so a caller can name it (Coolify's own
+// name for it, from `config.name`), rather than a generic noun a park or
+// provision step's own text can be told apart from the server it runs on.
+describeProvision, describePark, stepKey = org.id, isOn = (live) => live.config.enabled !== false) {
     const key = resourceKey(system, kind, stepKey);
     const live = byKey.get(key);
     // A live resource is on unless it says it is parked. Providers' own
@@ -3551,7 +3567,7 @@ function planHostingToggle(system, kind, desiredEnabled, org, byKey, claimed, de
             return null;
         return {
             action: { system, kind, key: stepKey, desired: { enabled: false } },
-            step: { description: describePark(), destroys: true, migrates: false },
+            step: { description: describePark(live), destroys: true, migrates: false },
         };
     }
     if (live)
@@ -3562,7 +3578,7 @@ function planHostingToggle(system, kind, desiredEnabled, org, byKey, claimed, de
         return null;
     return {
         action: { system, kind, key: stepKey, desired: { enabled: true } },
-        step: { description: describeProvision(), destroys: false, migrates: false },
+        step: { description: describeProvision(live), destroys: false, migrates: false },
     };
 }
 /**
@@ -3701,7 +3717,7 @@ function planApp(org, entryKey, byKey, claimed, recordedIds, plannedSteps, authI
         });
     }
     const refused = noRepo || composeBuilt;
-    const toggled = planHostingToggle('coolify', 'app', entry !== undefined && !refused, org, byKey, claimed, () => `provision ${org.slug}'s ${label}${volumesNote(entry)}`, () => `park ${org.slug}'s ${label} (stops serving; app data is retained, not deleted -- #7 corrections 1)`, stepKey);
+    const toggled = planHostingToggle('coolify', 'app', entry !== undefined && !refused, org, byKey, claimed, () => `provision ${org.slug}'s ${label}${volumesNote(entry)}`, (live) => `park ${org.slug}'s ${label}${liveName(live) ? ` ${liveName(live)}` : ''} (stops serving; app data is retained, not deleted -- #7 corrections 1)`, stepKey);
     // #447: only the live app's own `git_repository` decides, never the
     // declared entry (a `repo: null` entry can sit over a bespoke app, which is
     // parked as before). Only the `app` key ever held the default app, and an
@@ -4253,7 +4269,7 @@ function planDatabase(org, byKey, claimed, recordedIds) {
     if (!appHome || !legacy || !recordedIds[legacy] || serviceOnServer(org, appHome, recordedIds)) {
         return null;
     }
-    return planHostingToggle('coolify', 'database', org.hosting.entries.some(entryHasDatabase), org, byKey, claimed, () => `provision a database for ${org.slug}'s app`, () => `park ${org.slug}'s database (data is retained, not deleted -- #7 corrections 1)`, org.id, (live) => !/^(exited|stopped)/.test(String(live.config.status ?? '')));
+    return planHostingToggle('coolify', 'database', org.hosting.entries.some(entryHasDatabase), org, byKey, claimed, () => `provision a database for ${org.slug}'s app`, (live) => `park ${org.slug}'s database${liveName(live) ? ` ${liveName(live)}` : ''} (data is retained, not deleted -- #7 corrections 1)`, org.id, (live) => !/^(exited|stopped)/.test(String(live.config.status ?? '')));
 }
 function sortedNames(names) {
     return names.filter(Boolean).sort().join(',');
@@ -4298,7 +4314,7 @@ function planTunnel(org, byKey, claimed, recordedIds, platformApex) {
         appsOn(org) ||
         // A company without a domain still gets a tunnel of its own for an entry at `<slug>.oikos.host`.
         entryRoutes(org, recordedIds).length > 0;
-    const toggled = planHostingToggle('cloudflare', 'tunnel', desired, org, byKey, claimed, () => `provision a Cloudflare tunnel for ${org.slug}'s app`, () => `park ${org.slug}'s Cloudflare tunnel (configuration is retained, not deleted -- #7 corrections 1)`, org.id, 
+    const toggled = planHostingToggle('cloudflare', 'tunnel', desired, org, byKey, claimed, () => `provision a Cloudflare tunnel for ${org.slug}'s app`, (live) => `park ${org.slug}'s Cloudflare tunnel${liveName(live) ? ` ${liveName(live)}` : ''} (configuration is retained, not deleted -- #7 corrections 1)`, org.id, 
     // Cloudflare's status, read against what is declared: a park cannot change
     // it, so judging on/off by status alone would never let the plan empty.
     // Wanted on: only `down` needs a repair (`inactive` is a fresh tunnel

package/dist/plan/edge-access.d.ts

@@ -115,6 +115,15 @@ export interface AccessGateInput {
     emails: string[];
 }
 export declare const isNotFound: (error: unknown) => boolean;
+/**
+ * Throws the operator-facing message when `error` is edge's generic 404 for
+ * a route this SDK expects but the running release predates (#1061, pain
+ * point 12: production edge six days behind the SDK failed foundry's first
+ * apps step with the raw "Route not found" and no hint why). Returns
+ * otherwise, so the caller's own handling of `error` -- a conflict branch,
+ * an `isNotFound`/`isNotFoundError` "already gone" check -- still runs.
+ */
+export declare function edgeTooOld(error: unknown, what: string): void;
 /**
  * Edge answers a policy, delete or create call for a live Access app with no
  * edge row `not_found` or 409 `exists` ("create or adopt it first"). So the

package/dist/plan/edge-access.js

@@ -3,6 +3,26 @@ import { EDGE_DNS_WRITE_CONCURRENCY, limiter, rateLimitRetrier, } from './edge-d
 export const isNotFound = (error) => error instanceof EdgeApiError && error.code === 'not_found';
 /** Edge's 409 for a live app on the domain with no edge row. */
 const isExists = (error) => error instanceof EdgeApiError && error.code === 'conflict' && error.reason === 'exists';
+/**
+ * Edge's generic 404 fallback for a route no release of it answers yet
+ * (`service.ts`'s `errorResponse('not_found', 'Route not found')`), as
+ * opposed to a resource-specific 404 ("Key not found" and the like), which
+ * never carries this exact message.
+ */
+const isRouteMissing = (error) => error instanceof EdgeApiError && error.status === 404 && error.message === 'Route not found';
+/**
+ * Throws the operator-facing message when `error` is edge's generic 404 for
+ * a route this SDK expects but the running release predates (#1061, pain
+ * point 12: production edge six days behind the SDK failed foundry's first
+ * apps step with the raw "Route not found" and no hint why). Returns
+ * otherwise, so the caller's own handling of `error` -- a conflict branch,
+ * an `isNotFound`/`isNotFoundError` "already gone" check -- still runs.
+ */
+export function edgeTooOld(error, what) {
+    if (isRouteMissing(error)) {
+        throw new Error(`${what} is blocked: the running edge has no route for this step; deploy edge to the release foundry expects`);
+    }
+}
 /**
  * Edge answers a policy, delete or create call for a live Access app with no
  * edge row `not_found` or 409 `exists` ("create or adopt it first"). So the

package/dist/plan/executor.d.ts

@@ -2489,6 +2489,22 @@ export declare class FoundryStepExecutor implements StepExecutor, StepDestroyer
         }>;
         down: string[];
     }>;
+    /**
+     * The built-in fan-out's in-place deploy (`release/fanout.ts`): pins the org's Boule or Agora to `sha`, deploys
+     * it and waits at most STAGE_DEPLOY_ATTEMPTS polls (10 minutes) for Coolify to finish. The reasons are fixed
+     * texts. Never touches a copy that runs blue/green: that one has colours and is released by its own runner.
+     */
+    deployBuiltinAt(input: {
+        app: 'boule' | 'agora';
+        org: DeclaredOrg;
+        recordedIds: Readonly<Record<string, string>>;
+        sha: string;
+    }): Promise<{
+        ok: true;
+    } | {
+        ok: false;
+        reason: string;
+    }>;
     /**
      * The release runner's outside calls for one blue/green entry (`plan/release-host.ts`), or `null` while its
      * blue copy is not recorded. The health token is only read here, never minted.

package/dist/plan/executor.js

@@ -30,12 +30,12 @@ import { BACKUP_POSTGRES_DATABASES_SCOPE_KEY, BACKUP_SCOPE_PREFIX, COMPANY_BACKU
 import { LOGIN_COOKIE_DOMAIN_SCOPE_KEY } from './cookie-domain.js';
 import { previewsOn } from './coolify-previews.js';
 import { CORE_FILES_COLLECTION_NAME, FILES_CORE_COLLECTION_SCOPE_KEY, ZITADEL_FILES_PROJECT_SCOPE_KEY, } from './core-files.js';
-import { waitForHealthyVersion, waitForNewDeployment, } from './deploy-health.js';
+import { waitForDeploymentFinished, waitForHealthyVersion, waitForNewDeployment, } from './deploy-health.js';
 import { DESKTOP_AUTH_APP_SCOPE_KEY, DESKTOP_AUTH_DEPLOYED_SCOPE_KEY, DESKTOP_AUTH_DOCKERFILE, DESKTOP_AUTH_HOME, DESKTOP_AUTH_ORIGIN_SCOPE_KEY, DESKTOP_AUTH_READY_PATH, DESKTOP_AUTH_READY_SCOPE_KEY, DESKTOP_AUTH_REDIRECT_SCOPE_KEY, DESKTOP_AUTH_REVISION_SCOPE_KEY, desktopAuthCallbackUri, desktopAuthEnv, desktopAuthInputs, desktopAuthRedirectUnsupported, desktopAuthRevision, } from './desktop-auth.js';
 import { desktopCallbackUri, signInCallbackAppliedValue, signInRedirectUris, unionRedirectUris, zitadelAppDesktopCallbackAppliedScopeKey, } from './desktop-callback.js';
 import { PLATFORM_NETWORK, companyNetworkName, ensureDestination, findDestination, onPlatformNetwork, placed, placedByScope, placedOnPlatform, } from './destinations.js';
 import { DEFAULT_AI_LIMIT_MICROS } from './diff.js';
-import { createEdgeAccess, isNotFound as isEdgeNotFound, } from './edge-access.js';
+import { createEdgeAccess, edgeTooOld, isNotFound as isEdgeNotFound, } from './edge-access.js';
 import { EDGE_DNS_WRITE_CONCURRENCY, createEdgeDns, limiter, } from './edge-dns.js';
 import { createEdgeRateLimits, } from './edge-ratelimits.js';
 import { createEdgeTunnels } from './edge-tunnels.js';
@@ -77,7 +77,7 @@ import { CALLS_MAS_APPLIED_IMAGE_TAG_SCOPE_KEY, CALLS_SYNAPSE_APPLIED_IMAGE_TAG_
 import { AUTH_LOGIN_PORT, PLATFORM_AUTH_BROKER_MAIL_KEY_REF, PLATFORM_AUTH_LOGIN_INTERNAL_NAME, PLATFORM_AUTH_LOGIN_SCOPE_KEY, PLATFORM_AUTH_PROJECT, PLATFORM_MAIL_SERVER_SCOPE_KEY, defaultStalwartForInstance, filesDepsInvalid, filesShareDomains, provisionAuthStack, provisionDatabaseBackup, provisionFilesHost, provisionJobsHost, provisionMailServer, resolveOwnFilesKeys, } from './platform-providers.js';
 import { companyAppPostLogoutUris, postLogoutAppliedValue, postLogoutUrisFor, zitadelAppPostLogoutAppliedScopeKey, } from './post-logout-uris.js';
 import { CLOUDFLARE_TUNNEL_APPS_HOSTNAMES_SCOPE_KEY, CLOUDFLARE_TUNNEL_ENTRY_ROUTES_SCOPE_KEY, CLOUDFLARE_TUNNEL_OIKOS_HOSTNAMES_SCOPE_KEY, ENTRY_CONTAINER_LABELS, appLabelsAppliedScopeKey, builtinRoutes, entryAppsHostnames, entryLabelHash, entryLabelSpec, entryOikosHostnames, entryProxyLabels, entryRoutes, entryRoutesMarker, ownsAppsZoneName, ownsOikosName, routesWithoutToken, sameLabelSets, } from './proxy-labels.js';
-import { createReleaseHost } from './release-host.js';
+import { STAGE_DEPLOY_ATTEMPTS, STAGE_DEPLOY_INTERVAL_MS, createReleaseHost, } from './release-host.js';
 import { TUNNEL_HTTP_INGRESS_SCOPE_KEY, applyRemoteDatabase, mergeTcpIngress, recordedTcpRoutes, } from './remote-database.js';
 import { editSignInBindings } from './sign-in-bindings.js';
 import { ZITADEL_ORG_DOMAIN_SCOPE_KEY, ZITADEL_ORG_REGISTRATION_CLOSED_SCOPE_KEY, closeSignInOrgRegistration, ensureSignInOrg, inviteOwnerWithLink, registerSignInDomainOrBlock, signInOrgDomain, } from './sign-in-org.js';
@@ -3183,6 +3183,40 @@ export class FoundryStepExecutor {
             down,
         };
     }
+    /**
+     * The built-in fan-out's in-place deploy (`release/fanout.ts`): pins the org's Boule or Agora to `sha`, deploys
+     * it and waits at most STAGE_DEPLOY_ATTEMPTS polls (10 minutes) for Coolify to finish. The reasons are fixed
+     * texts. Never touches a copy that runs blue/green: that one has colours and is released by its own runner.
+     */
+    async deployBuiltinAt(input) {
+        const { app, org, sha } = input;
+        if (!/^[0-9a-f]{40}$/.test(sha))
+            return { ok: false, reason: 'not a built commit' };
+        if (org[app]?.blueGreen !== undefined)
+            return { ok: false, reason: 'copy runs blue/green' };
+        const appId = input.recordedIds[`coolify:${app}:${org.id}`];
+        if (!appId)
+            return { ok: false, reason: 'app not recorded' };
+        try {
+            await this.ports.coolify.updateApplication(appId, { git_commit_sha: sha });
+        }
+        catch (error) {
+            console.error(`fanout ${app}: selecting the commit failed (${error instanceof Error ? error.name : 'Error'})`);
+            return { ok: false, reason: 'select commit failed' };
+        }
+        try {
+            await waitForDeploymentFinished(this.ports.coolify, appId, {
+                ...(this.ports.runtime ?? {}),
+                deployAttempts: STAGE_DEPLOY_ATTEMPTS,
+                deployIntervalMs: STAGE_DEPLOY_INTERVAL_MS,
+            }, `coolify:fanout:${app}`, 'the copy stays as it was');
+        }
+        catch (error) {
+            console.error(`fanout ${app}: the deployment did not finish (${error instanceof Error ? error.name : 'Error'})`);
+            return { ok: false, reason: 'deployment did not finish' };
+        }
+        return { ok: true };
+    }
     /**
      * The release runner's outside calls for one blue/green entry (`plan/release-host.ts`), or `null` while its
      * blue copy is not recorded. The health token is only read here, never minted.
@@ -10853,6 +10887,7 @@ export class FoundryStepExecutor {
                     await edge.revokeKey(org.id, id);
                 }
                 catch (error) {
+                    edgeTooOld(error, at);
                     // A 404 means it is already gone -- recorded as retired below,
                     // the same as a successful revoke. Anything else is a real
                     // failure: thrown, not swallowed, so the key stays live and the
@@ -10922,18 +10957,26 @@ export class FoundryStepExecutor {
                 await edge.revokeKey(org.id, staleUndelivered);
             }
             catch (error) {
+                edgeTooOld(error, at);
                 if (!isNotFoundError(error))
                     throw error;
             }
             context.recordScopeId(entryEdgeUndeliveredScopeKey(entryKey), '');
             await context.persistScopeIds();
         }
-        const minted = await edge.mintKey({
-            organisationId: org.id,
-            domain: edgeKeyDomain(org),
-            name: desired.name,
-            expiresDays: EDGE_ENTRY_KEY_DAYS,
-        });
+        let minted;
+        try {
+            minted = await edge.mintKey({
+                organisationId: org.id,
+                domain: edgeKeyDomain(org),
+                name: desired.name,
+                expiresDays: EDGE_ENTRY_KEY_DAYS,
+            });
+        }
+        catch (e) {
+            edgeTooOld(e, at);
+            throw e;
+        }
         context.recordScopeId(entryEdgeUndeliveredScopeKey(entryKey), minted.id);
         await context.persistScopeIds();
         const envs = [
@@ -11094,6 +11137,7 @@ export class FoundryStepExecutor {
             await edge.setOrganisationSlug(org.id, slug);
         }
         catch (e) {
+            edgeTooOld(e, `edge:slug for ${org.slug}`);
             if (e instanceof EdgeApiError && e.code === 'conflict') {
                 throw new Error(`edge:slug for ${org.slug} is blocked: edge says another company already holds the slug '${slug}', so its platform hostnames cannot be accepted; change the slug`);
             }
@@ -11113,7 +11157,13 @@ export class FoundryStepExecutor {
             throw new Error(`edge:reserved-labels for ${org.slug}: this executor was built without an EdgePort (#317)`);
         }
         const labels = action.desired.labels;
-        await edge.setReservedLabels(org.id, labels);
+        try {
+            await edge.setReservedLabels(org.id, labels);
+        }
+        catch (e) {
+            edgeTooOld(e, `edge:reserved-labels for ${org.slug}`);
+            throw e;
+        }
         context.recordScopeId(EDGE_RESERVED_LABELS_APPLIED_SCOPE_KEY, JSON.stringify(labels));
         return {};
     }
@@ -11658,6 +11708,7 @@ export class FoundryStepExecutor {
                         await edge.revokeKey(org.id, target.id);
                     }
                     catch (error) {
+                        edgeTooOld(error, `edge:key '${target.key}' for ${org.slug}`);
                         // Already gone -- the same already-retired key a retire step
                         // handled before org destroy ever ran can still reach here
                         // (#317 review); a 404 is success, not a reason to block

package/dist/plan/own-diff.js

@@ -11,7 +11,7 @@ import { ownHostnames } from './own-hostnames.js';
 import { JOBS_PUBLIC_URL_SCOPE_KEY, OWN_JOBS_ROUTE_TUNNEL_SCOPE_KEY } from './own-jobs-route.js';
 import { OWN_FILES_R2_SCOPE_KEY, OWN_SERVICES_APPLIED_SCOPE_KEY, orgTarget, } from './own-services.js';
 import { OWN_SFU_APPLIED_HOSTNAME_SCOPE_KEY, OWN_SFU_APPLIED_NODE_IP_SCOPE_KEY, OWN_SFU_APPLIED_SLOT_SCOPE_KEY, OWN_SFU_APPLIED_VERSION_SCOPE_KEY, OWN_SFU_DEPLOY_PENDING_SCOPE_KEY, OWN_SFU_KEYS_UNDELIVERED_SCOPE_KEY, SFU_INTERNAL_URL_SCOPE_KEY, SFU_PUBLIC_URL_SCOPE_KEY, } from './own-sfu.js';
-import { AUTH_TUNNEL_ROUTES_VERSION, AUTH_ZITADEL_RUN_OPTIONS, JOBS_DEPLOY_PENDING_SCOPE_KEY, JOBS_INTERNAL_URL_SCOPE_KEY, PLATFORM_AUTH_ADMIN_UNDELIVERED_SCOPE_KEY, PLATFORM_AUTH_ADMIN_USER_SCOPE_KEY, PLATFORM_AUTH_BROKER_SCOPE_KEY, PLATFORM_AUTH_CONNECTOR_APP_SCOPE_KEY, PLATFORM_AUTH_INSTANCE_SCOPE_KEY, PLATFORM_AUTH_INSTANCE_UNDELIVERED_SCOPE_KEY, PLATFORM_AUTH_LOGIN_CLIENT_UNDELIVERED_SCOPE_KEY, PLATFORM_AUTH_LOGIN_CLIENT_USER_SCOPE_KEY, PLATFORM_AUTH_MCP_SCOPE_KEY, PLATFORM_AUTH_SMTP_SCOPE_KEY, PLATFORM_AUTH_TUNNEL_ROUTES_SCOPE_KEY, PLATFORM_AUTH_ZITADEL_RUN_OPTIONS_SCOPE_KEY, PLATFORM_FILES_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_FILES_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_FILES_INTERNAL_URL_SCOPE_KEY, PLATFORM_FILES_INTROSPECTION_SCOPE_KEY, PLATFORM_FILES_INTROSPECTION_UNDELIVERED_SCOPE_KEY, PLATFORM_FILES_MANAGEMENT_KEY_SCOPE_KEY, PLATFORM_FILES_MANAGEMENT_UNDELIVERED_SCOPE_KEY, PLATFORM_MAIL_ADMIN_KEY_SCOPE_KEY, PLATFORM_MAIL_ADMIN_UNDELIVERED_SCOPE_KEY, PLATFORM_MAIL_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_MAIL_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_MAIL_INTERNAL_URL_SCOPE_KEY, orgDeclaresMcp, platformMailMxHost, } from './platform-diff.js';
+import { AUTH_TUNNEL_ROUTES_VERSION, AUTH_ZITADEL_RUN_OPTIONS, JOBS_DEPLOY_PENDING_SCOPE_KEY, JOBS_INTERNAL_URL_SCOPE_KEY, PLATFORM_AUTH_ADMIN_UNDELIVERED_SCOPE_KEY, PLATFORM_AUTH_ADMIN_USER_SCOPE_KEY, PLATFORM_AUTH_BROKER_SCOPE_KEY, PLATFORM_AUTH_CONNECTOR_APP_SCOPE_KEY, PLATFORM_AUTH_INSTANCE_SCOPE_KEY, PLATFORM_AUTH_INSTANCE_UNDELIVERED_SCOPE_KEY, PLATFORM_AUTH_LOGIN_CLIENT_UNDELIVERED_SCOPE_KEY, PLATFORM_AUTH_LOGIN_CLIENT_USER_SCOPE_KEY, PLATFORM_AUTH_LOGIN_ENV_SCOPE_KEY, PLATFORM_AUTH_MCP_SCOPE_KEY, PLATFORM_AUTH_SMTP_SCOPE_KEY, PLATFORM_AUTH_TUNNEL_ROUTES_SCOPE_KEY, PLATFORM_AUTH_ZITADEL_RUN_OPTIONS_SCOPE_KEY, PLATFORM_FILES_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_FILES_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_FILES_INTERNAL_URL_SCOPE_KEY, PLATFORM_FILES_INTROSPECTION_SCOPE_KEY, PLATFORM_FILES_INTROSPECTION_UNDELIVERED_SCOPE_KEY, PLATFORM_FILES_MANAGEMENT_KEY_SCOPE_KEY, PLATFORM_FILES_MANAGEMENT_UNDELIVERED_SCOPE_KEY, PLATFORM_MAIL_ADMIN_KEY_SCOPE_KEY, PLATFORM_MAIL_ADMIN_UNDELIVERED_SCOPE_KEY, PLATFORM_MAIL_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_MAIL_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_MAIL_INTERNAL_URL_SCOPE_KEY, authLoginEnvMarker, orgDeclaresMcp, platformMailMxHost, } from './platform-diff.js';
 import { OWN_FILES_RUNNER_SCOPE_KEY } from './platform-org.js';
 import { JOBS_ADMIN_TOKEN_UNDELIVERED_SCOPE_KEY, JOBS_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_AUTH_DB_SCOPE_KEY, PLATFORM_AUTH_ZITADEL_SCOPE_KEY, PLATFORM_FILES_SHARE_DOMAINS_SCOPE_KEY, filesShareDomains, } from './platform-providers.js';
 import { SIGN_IN_LOGIN_APP_SCOPE_KEY, inAppSignIn } from './sign-in.js';
@@ -217,6 +217,9 @@ export function planOwnAuth(options) {
         recordedIds[PLATFORM_AUTH_SMTP_SCOPE_KEY] &&
         recordedIds[PLATFORM_AUTH_ZITADEL_RUN_OPTIONS_SCOPE_KEY] === AUTH_ZITADEL_RUN_OPTIONS &&
         recordedIds[PLATFORM_AUTH_TUNNEL_ROUTES_SCOPE_KEY] === AUTH_TUNNEL_ROUTES_VERSION &&
+        // The login apps' mail env (`LOGIN_MAIL_FROM`, the Lettermint key); a stack from before it is written again.
+        recordedIds[PLATFORM_AUTH_LOGIN_ENV_SCOPE_KEY] ===
+            authLoginEnvMarker(`noreply@${names.mailDomain}`) &&
         // The code mail carries the company's name and design; an older apply, or a renamed company, writes it again.
         (options.signinMailWritable === false ||
             recordedIds[PLATFORM_AUTH_SIGNIN_MAIL_SCOPE_KEY] === signinMailMarker(org.displayName)) &&

package/dist/plan/plan-apply.d.ts

@@ -234,6 +234,21 @@ export interface StepExecutor {
      * copy is not recorded. Optional: an executor without it runs no release.
      */
     releaseHost?(org: DeclaredOrg, subject: BlueGreenSubject, context: ReleaseContext, renew: () => Promise<void>): ReleaseHost | null;
+    /**
+     * The built-in fan-out's in-place deploy of one org's Boule or Agora at `sha` (`release/fanout.ts`). Optional:
+     * an executor without it runs no fan-out.
+     */
+    deployBuiltinAt?(input: {
+        app: 'boule' | 'agora';
+        org: DeclaredOrg;
+        recordedIds: Readonly<Record<string, string>>;
+        sha: string;
+    }): Promise<{
+        ok: true;
+    } | {
+        ok: false;
+        reason: string;
+    }>;
     /**
      * One release tick for one subject: the recorded stage gate's list follows the people holding `releases:preview`,
      * owners staying. Never throws. Optional: an executor without it leaves every gate as it is.
@@ -337,6 +352,8 @@ export interface PlatformStepExecutor {
         authExecutorMissing?: boolean;
         /** #216 round 5: see `PlanPlatformOptions.blocked.authOtherInputsMissing`. */
         authOtherInputsMissing?: string[];
+        /** The sealed mail key resolved: `planAuth` asks for the login env marker only when it did. */
+        authMailKeyHeld?: boolean;
         files?: string;
         integrations?: string;
         ai?: string;
@@ -933,6 +950,12 @@ export declare class FoundryPlanApply implements PlanApply {
         target: string;
         error?: string;
     }>>;
+    /**
+     * One fan-out pass for the owner org's built-in `app` (`release/fanout.ts`): the other orgs' copies are put on
+     * the commit the owner's copy runs, one after the other, in place, stopping at the first failure. Each state
+     * write takes the owner's lease on its own; no lease is held while a deploy waits.
+     */
+    private runBuiltinFanOut;
     /**
      * Phase 2 for one org: takes the copy lock of each stage copy request under the org's lease, then runs the
      * copies after the lease is released. While a copy runs, every other tick and apply for the org goes on, a

package/dist/plan/plan-apply.js

@@ -4,6 +4,7 @@ import { VERSIONS, floatingTagBlocked } from '../bootstrap/versions.js';
 import { sfuSlotPorts } from '../calls-sfu.js';
 import { ORG_SERVICE_SCHEMAS, heldEnvRef, ownsServices, validateHosting } from '../org.js';
 import { PLATFORM_SERVICE_KEYS } from '../platform.js';
+import { runBuiltinFanOut } from '../release/fanout.js';
 import { recordedFrom, releaseKeys } from '../release/state.js';
 import { AlreadyRunningError, StepFailedError } from '../runner/errors.js';
 import { StaleLeaseError, } from '../store.js';
@@ -1338,6 +1339,16 @@ export class FoundryPlanApply {
                 settle('platform', e);
             }
         }
+        // Between the phases: once the owner's Boule or Agora is fully live, copy it to every other org. Its errors
+        // are reported by app, never as an org.
+        for (const app of ['boule', 'agora']) {
+            try {
+                await this.runBuiltinFanOut(app);
+            }
+            catch (e) {
+                settle(`fanout:${app}`, e);
+            }
+        }
         // Phase 2: the copies, one subject at a time. A copy takes its lock under the org's lease and runs without it.
         for (const { target, run } of copying) {
             try {
@@ -1350,6 +1361,65 @@ export class FoundryPlanApply {
         }
         return results;
     }
+    /**
+     * One fan-out pass for the owner org's built-in `app` (`release/fanout.ts`): the other orgs' copies are put on
+     * the commit the owner's copy runs, one after the other, in place, stopping at the first failure. Each state
+     * write takes the owner's lease on its own; no lease is held while a deploy waits.
+     */
+    async runBuiltinFanOut(app) {
+        const store = this.releaseStore;
+        const platform = await this.platformDeps?.store.get();
+        if (!store || !platform || !this.executor.deployBuiltinAt)
+            return;
+        const ownerOrgId = platform.ownerOrgId;
+        const owner = await this.store.get(ownerOrgId);
+        if (!owner)
+            return;
+        const empty = { orgId: ownerOrgId, steps: [], unmanaged: [] };
+        if ((await this.authz.canApplyPlatform?.(this.actor)) !== true) {
+            await this.refusePlatform(empty, ownerOrgId, 'not authorized to run the release tick');
+            throw new NotAuthorizedError(this.actor, ownerOrgId);
+        }
+        const deploy = this.executor.deployBuiltinAt.bind(this.executor);
+        await runBuiltinFanOut({
+            ownerOrgId,
+            app,
+            ownerDeclares: owner[app]?.blueGreen !== undefined,
+            store,
+            readIds: async () => ({
+                ...((await this.store.getAppliedState(ownerOrgId))?.serviceIds ?? {}),
+            }),
+            write: (writes) => this.withMutationLease(companyScope(ownerOrgId), (reason) => this.refuse(empty, reason), async (fence) => {
+                const previous = await this.store.getAppliedState(ownerOrgId);
+                if (!previous)
+                    throw new Error('the owner org has no applied state');
+                const serviceIds = { ...previous.serviceIds };
+                for (const [key, value] of Object.entries(writes)) {
+                    if (value === null)
+                        delete serviceIds[key];
+                    else
+                        serviceIds[key] = value;
+                }
+                await this.store.putAppliedState({ ...previous, serviceIds }, fence);
+            }),
+            targets: async () => {
+                const others = (await this.store.list()).filter((o) => o.id !== ownerOrgId && o[app] !== undefined);
+                return others.map((o) => ({ orgId: o.id, slug: o.slug }));
+            },
+            // Re-reads the org fresh right before its own deploy, never from a list taken when the pass
+            // started: a deploy can wait many minutes with no lease held on the target org (CLAUDE.md,
+            // "Built-in fan-out"), so an ordinary `apply()` to a later-in-order org can turn its own
+            // blue/green on in that window, and `deployBuiltinAt`'s guard must see that, not a stale copy.
+            deploy: async (target, sha) => {
+                const org = await this.store.get(target.orgId);
+                if (!org || org[app] === undefined)
+                    return { ok: false, reason: 'org not found' };
+                const recordedIds = (await this.store.getAppliedState(org.id))?.serviceIds ?? {};
+                return deploy({ app, org, recordedIds, sha });
+            },
+            now: this.now,
+        });
+    }
     /**
      * Phase 2 for one org: takes the copy lock of each stage copy request under the org's lease, then runs the
      * copies after the lease is released. While a copy runs, every other tick and apply for the org goes on, a

package/dist/plan/platform-diff.d.ts

@@ -68,6 +68,20 @@ export declare const PLATFORM_AUTH_ZITADEL_RUN_OPTIONS_SCOPE_KEY = "zitadel_plat
 export declare const AUTH_TUNNEL_ROUTES_VERSION = "2";
 /** The {@link AUTH_TUNNEL_ROUTES_VERSION} the sign-in tunnel was last written with. */
 export declare const PLATFORM_AUTH_TUNNEL_ROUTES_SCOPE_KEY = "auth_tunnel_routes_applied";
+/**
+ * The version of the env `provisionAuthStack` writes to the login apps (`LOGIN_MAIL_FROM`,
+ * `LETTERMINT_API_TOKEN`). Raise it when a key is added or its meaning changes, so an applied
+ * sign-in plans its step once more and the apps get the env. Never holds a value.
+ * 1: the login apps send their own invite and verification mail (auth 0.35.0, foundry 0.68.0).
+ */
+export declare const AUTH_LOGIN_ENV_VERSION = "1";
+/**
+ * What the login apps' env is written for: the version and the sender address (`LOGIN_MAIL_FROM`).
+ * The address is no secret; a new mail domain gives a new marker, so the step is planned again.
+ */
+export declare function authLoginEnvMarker(smtpSender: string): string;
+/** The {@link authLoginEnvMarker} the login apps' env was last written with, once live on them. */
+export declare const PLATFORM_AUTH_LOGIN_ENV_SCOPE_KEY = "auth_login_env_applied";
 /**
  * `'1'` once the broker holds `MCP_DATABASE_URL` and `MCP_OAUTH_SECRET` (auth ADR 0002), recorded by the
  * sign-in step when an entry of that broker declares `uses.auth.mcp`. A broker entry with `mcp` is not
@@ -977,6 +991,8 @@ export declare const PLATFORM_EDGE_PLATFORM_KEY_SET_SCOPE_KEY = "edge_platform_k
  */
 export declare const PLATFORM_EDGE_APPLIED_HOSTNAME_SCOPE_KEY = "edge_applied_hostname";
 export declare const PLATFORM_EDGE_APPLIED_GIT_REF_SCOPE_KEY = "edge_applied_git_ref";
+/** foundry#1024: what `provisionPlatformEdge` last wrote for {@link usageTailEdgeEnv}; see {@link usageTailEdgeEnvValue}. */
+export declare const PLATFORM_EDGE_APPLIED_USAGE_TAIL_SCOPE_KEY = "edge_applied_usage_tail";
 /**
  * F4.dns: where edge answers on the Coolify network
  * (`http://<custom_internal_name>:3000`), recorded by the edge step once the
@@ -1206,6 +1222,45 @@ export declare const PLATFORM_UPTIME_ACCESS_TOKEN_ID_SCOPE_KEY = "uptime_access_
 export declare function uptimeAccessTokenConflict(): string;
 /** Whether anything of the uptime Worker is recorded that disabling it must remove. */
 export declare function uptimeRecorded(recordedIds: Readonly<Record<string, string>>): boolean;
+/**
+ * foundry#1024: the usage tail Worker that meters every company Worker's
+ * use, one `cloudflare:worker` step keyed {@link PLATFORM_USAGE_TAIL_KEY}.
+ * `runPlatformSteps` records the script name under
+ * `cloudflare:worker:platform:usage-tail` once it succeeds, which is what
+ * stops `planUsageTail` planning it again.
+ */
+export declare const PLATFORM_USAGE_TAIL_KEY = "platform:usage-tail";
+/** The Worker script's name at Cloudflare. */
+export declare const PLATFORM_USAGE_TAIL_SCRIPT_NAME = "platform-usage-tail";
+/**
+ * The held ref of the Worker's one secret, `EDGE_USAGE_KEY` -- the narrow
+ * `usage:ingest` credential edge's ingest route accepts. Minted once (rule
+ * 1): the hash is never secret, so (the same pattern `provisionPlatformEdge`
+ * uses for `EDGE_PLATFORM_KEY_HASH`) it doubles as both the undelivered
+ * marker and the final recorded value, with nothing extra to compute or a
+ * live read that could fall back to a blank value.
+ */
+export declare const PLATFORM_USAGE_TAIL_KEY_REF = "usage-tail:worker:key";
+export declare const PLATFORM_USAGE_TAIL_KEY_UNDELIVERED_SCOPE_KEY = "usage_tail_key_undelivered";
+export declare const PLATFORM_USAGE_TAIL_KEY_SET_SCOPE_KEY = "usage_tail_key_set";
+export declare const PLATFORM_USAGE_TAIL_APPLIED_TAG_SCOPE_KEY = "usage_tail_applied_tag";
+export declare const PLATFORM_USAGE_TAIL_APPLIED_SHA256_SCOPE_KEY = "usage_tail_applied_sha256";
+export declare const PLATFORM_USAGE_TAIL_REMOVED_SCOPE_KEY = "usage_tail_removed";
+/**
+ * What `platform:edge`'s own env should hold for the usage tail Worker
+ * (`USAGE_TAIL_WORKER`, `EDGE_USAGE_KEY_HASH`), or `null` while the Worker is
+ * not deployed or its key is not held yet. `provisionPlatformEdge` writes
+ * these two names on every run alongside `EDGE_PLATFORM_KEY_HASH`; `planEdge`
+ * reads {@link usageTailEdgeEnvValue} to replan edge once this changes.
+ */
+export declare function usageTailEdgeEnv(recordedIds: Readonly<Record<string, string>>): {
+    USAGE_TAIL_WORKER: string;
+    EDGE_USAGE_KEY_HASH: string;
+} | null;
+/** The value `provisionPlatformEdge` records once it has written {@link usageTailEdgeEnv} to edge's env; empty while there is nothing to write. */
+export declare function usageTailEdgeEnvValue(recordedIds: Readonly<Record<string, string>>): string;
+/** Whether anything of the usage tail Worker is recorded that disabling it must remove. */
+export declare function usageTailRecorded(recordedIds: Readonly<Record<string, string>>): boolean;
 /**
  * Options for {@link planPlatform}, replacing four positional arguments that
  * shifted on every merge that added a platform service (#186, #187, #188) --
@@ -1265,6 +1320,11 @@ export interface PlanPlatformOptions {
          * these. Default `[]`.
          */
         authOtherInputsMissing?: string[];
+        /**
+         * Whether the sealed Lettermint key resolved. `provisionAuthStack` records the login env marker only
+         * then, so `planAuth` asks for the marker only when this is not false. Unset: asked for.
+         */
+        authMailKeyHeld?: boolean;
         /** C3: why the owner's invite cannot be made (no work address, no mail port). The rest of the step still plans. */
         authOwnerInvite?: string;
         files?: string;
@@ -1341,6 +1401,8 @@ export interface PlanPlatformOptions {
             pending: boolean;
             blocked?: string;
         };
+        /** foundry#1024: why the usage tail Worker is blocked, or absent when it is not. */
+        usageTail?: string;
         /** Why the organisation's release webhook step cannot run (the token is not held), or absent. */
         githubReleaseHook?: string;
         /**