@wtfalch/foundry 0.70.1 → 0.71.0

package/dist/index.d.ts

@@ -25,6 +25,8 @@ export { createPostgresAuditLog } from './store/audit.js';
 export type { Color, ReleaseAction, ReleaseEvent, ReleaseEventKind, ReleaseSeries, ReleaseSeriesColor, ReleaseSignal, ReleaseState, WindowState, } from './release/types.js';
 export type { ReleaseStore } from './release/state.js';
 export { readReleaseHistory, readReleaseState, requestReleaseAction } from './release/state.js';
+export type { BuiltinFanOut, FanOutApp, FanOutOrgState } from './release/fanout.js';
+export { readBuiltinFanOut } from './release/fanout.js';
 export type { ReleaseSeriesHost, ReleaseSeriesQuery } from './release/series.js';
 export { readReleaseSeries } from './release/series.js';
 export type { EntrySeriesDeps } from './plan/release-series-reader.js';

package/dist/index.js

@@ -15,6 +15,7 @@ export { StaleLeaseError } from './store.js';
 export { createPostgresAppliedVersionsStore, createPostgresDeclaredOrgStore, createPostgresPlatformStore, createPostgresRequestedChangeStore, DeclaredOrgError, PlatformError, } from './store/postgres.js';
 export { createPostgresAuditLog } from './store/audit.js';
 export { readReleaseHistory, readReleaseState, requestReleaseAction } from './release/state.js';
+export { readBuiltinFanOut } from './release/fanout.js';
 export { readReleaseSeries } from './release/series.js';
 export { readEntryReleaseSeries } from './plan/release-series-reader.js';
 export { checkMigrationCompatibility } from './release/migration-compatibility.js';

package/dist/plan/executor.d.ts

@@ -2489,6 +2489,22 @@ export declare class FoundryStepExecutor implements StepExecutor, StepDestroyer
         }>;
         down: string[];
     }>;
+    /**
+     * The built-in fan-out's in-place deploy (`release/fanout.ts`): pins the org's Boule or Agora to `sha`, deploys
+     * it and waits at most STAGE_DEPLOY_ATTEMPTS polls (10 minutes) for Coolify to finish. The reasons are fixed
+     * texts. Never touches a copy that runs blue/green: that one has colours and is released by its own runner.
+     */
+    deployBuiltinAt(input: {
+        app: 'boule' | 'agora';
+        org: DeclaredOrg;
+        recordedIds: Readonly<Record<string, string>>;
+        sha: string;
+    }): Promise<{
+        ok: true;
+    } | {
+        ok: false;
+        reason: string;
+    }>;
     /**
      * The release runner's outside calls for one blue/green entry (`plan/release-host.ts`), or `null` while its
      * blue copy is not recorded. The health token is only read here, never minted.

package/dist/plan/executor.js

@@ -30,7 +30,7 @@ import { BACKUP_POSTGRES_DATABASES_SCOPE_KEY, BACKUP_SCOPE_PREFIX, COMPANY_BACKU
 import { LOGIN_COOKIE_DOMAIN_SCOPE_KEY } from './cookie-domain.js';
 import { previewsOn } from './coolify-previews.js';
 import { CORE_FILES_COLLECTION_NAME, FILES_CORE_COLLECTION_SCOPE_KEY, ZITADEL_FILES_PROJECT_SCOPE_KEY, } from './core-files.js';
-import { waitForHealthyVersion, waitForNewDeployment, } from './deploy-health.js';
+import { waitForDeploymentFinished, waitForHealthyVersion, waitForNewDeployment, } from './deploy-health.js';
 import { DESKTOP_AUTH_APP_SCOPE_KEY, DESKTOP_AUTH_DEPLOYED_SCOPE_KEY, DESKTOP_AUTH_DOCKERFILE, DESKTOP_AUTH_HOME, DESKTOP_AUTH_ORIGIN_SCOPE_KEY, DESKTOP_AUTH_READY_PATH, DESKTOP_AUTH_READY_SCOPE_KEY, DESKTOP_AUTH_REDIRECT_SCOPE_KEY, DESKTOP_AUTH_REVISION_SCOPE_KEY, desktopAuthCallbackUri, desktopAuthEnv, desktopAuthInputs, desktopAuthRedirectUnsupported, desktopAuthRevision, } from './desktop-auth.js';
 import { desktopCallbackUri, signInCallbackAppliedValue, signInRedirectUris, unionRedirectUris, zitadelAppDesktopCallbackAppliedScopeKey, } from './desktop-callback.js';
 import { PLATFORM_NETWORK, companyNetworkName, ensureDestination, findDestination, onPlatformNetwork, placed, placedByScope, placedOnPlatform, } from './destinations.js';
@@ -77,7 +77,7 @@ import { CALLS_MAS_APPLIED_IMAGE_TAG_SCOPE_KEY, CALLS_SYNAPSE_APPLIED_IMAGE_TAG_
 import { AUTH_LOGIN_PORT, PLATFORM_AUTH_BROKER_MAIL_KEY_REF, PLATFORM_AUTH_LOGIN_INTERNAL_NAME, PLATFORM_AUTH_LOGIN_SCOPE_KEY, PLATFORM_AUTH_PROJECT, PLATFORM_MAIL_SERVER_SCOPE_KEY, defaultStalwartForInstance, filesDepsInvalid, filesShareDomains, provisionAuthStack, provisionDatabaseBackup, provisionFilesHost, provisionJobsHost, provisionMailServer, resolveOwnFilesKeys, } from './platform-providers.js';
 import { companyAppPostLogoutUris, postLogoutAppliedValue, postLogoutUrisFor, zitadelAppPostLogoutAppliedScopeKey, } from './post-logout-uris.js';
 import { CLOUDFLARE_TUNNEL_APPS_HOSTNAMES_SCOPE_KEY, CLOUDFLARE_TUNNEL_ENTRY_ROUTES_SCOPE_KEY, CLOUDFLARE_TUNNEL_OIKOS_HOSTNAMES_SCOPE_KEY, ENTRY_CONTAINER_LABELS, appLabelsAppliedScopeKey, builtinRoutes, entryAppsHostnames, entryLabelHash, entryLabelSpec, entryOikosHostnames, entryProxyLabels, entryRoutes, entryRoutesMarker, ownsAppsZoneName, ownsOikosName, routesWithoutToken, sameLabelSets, } from './proxy-labels.js';
-import { createReleaseHost } from './release-host.js';
+import { STAGE_DEPLOY_ATTEMPTS, STAGE_DEPLOY_INTERVAL_MS, createReleaseHost, } from './release-host.js';
 import { TUNNEL_HTTP_INGRESS_SCOPE_KEY, applyRemoteDatabase, mergeTcpIngress, recordedTcpRoutes, } from './remote-database.js';
 import { editSignInBindings } from './sign-in-bindings.js';
 import { ZITADEL_ORG_DOMAIN_SCOPE_KEY, ZITADEL_ORG_REGISTRATION_CLOSED_SCOPE_KEY, closeSignInOrgRegistration, ensureSignInOrg, inviteOwnerWithLink, registerSignInDomainOrBlock, signInOrgDomain, } from './sign-in-org.js';
@@ -3183,6 +3183,40 @@ export class FoundryStepExecutor {
             down,
         };
     }
+    /**
+     * The built-in fan-out's in-place deploy (`release/fanout.ts`): pins the org's Boule or Agora to `sha`, deploys
+     * it and waits at most STAGE_DEPLOY_ATTEMPTS polls (10 minutes) for Coolify to finish. The reasons are fixed
+     * texts. Never touches a copy that runs blue/green: that one has colours and is released by its own runner.
+     */
+    async deployBuiltinAt(input) {
+        const { app, org, sha } = input;
+        if (!/^[0-9a-f]{40}$/.test(sha))
+            return { ok: false, reason: 'not a built commit' };
+        if (org[app]?.blueGreen !== undefined)
+            return { ok: false, reason: 'copy runs blue/green' };
+        const appId = input.recordedIds[`coolify:${app}:${org.id}`];
+        if (!appId)
+            return { ok: false, reason: 'app not recorded' };
+        try {
+            await this.ports.coolify.updateApplication(appId, { git_commit_sha: sha });
+        }
+        catch (error) {
+            console.error(`fanout ${app}: selecting the commit failed (${error instanceof Error ? error.name : 'Error'})`);
+            return { ok: false, reason: 'select commit failed' };
+        }
+        try {
+            await waitForDeploymentFinished(this.ports.coolify, appId, {
+                ...(this.ports.runtime ?? {}),
+                deployAttempts: STAGE_DEPLOY_ATTEMPTS,
+                deployIntervalMs: STAGE_DEPLOY_INTERVAL_MS,
+            }, `coolify:fanout:${app}`, 'the copy stays as it was');
+        }
+        catch (error) {
+            console.error(`fanout ${app}: the deployment did not finish (${error instanceof Error ? error.name : 'Error'})`);
+            return { ok: false, reason: 'deployment did not finish' };
+        }
+        return { ok: true };
+    }
     /**
      * The release runner's outside calls for one blue/green entry (`plan/release-host.ts`), or `null` while its
      * blue copy is not recorded. The health token is only read here, never minted.

package/dist/plan/plan-apply.d.ts

@@ -234,6 +234,21 @@ export interface StepExecutor {
      * copy is not recorded. Optional: an executor without it runs no release.
      */
     releaseHost?(org: DeclaredOrg, subject: BlueGreenSubject, context: ReleaseContext, renew: () => Promise<void>): ReleaseHost | null;
+    /**
+     * The built-in fan-out's in-place deploy of one org's Boule or Agora at `sha` (`release/fanout.ts`). Optional:
+     * an executor without it runs no fan-out.
+     */
+    deployBuiltinAt?(input: {
+        app: 'boule' | 'agora';
+        org: DeclaredOrg;
+        recordedIds: Readonly<Record<string, string>>;
+        sha: string;
+    }): Promise<{
+        ok: true;
+    } | {
+        ok: false;
+        reason: string;
+    }>;
     /**
      * One release tick for one subject: the recorded stage gate's list follows the people holding `releases:preview`,
      * owners staying. Never throws. Optional: an executor without it leaves every gate as it is.
@@ -935,6 +950,12 @@ export declare class FoundryPlanApply implements PlanApply {
         target: string;
         error?: string;
     }>>;
+    /**
+     * One fan-out pass for the owner org's built-in `app` (`release/fanout.ts`): the other orgs' copies are put on
+     * the commit the owner's copy runs, one after the other, in place, stopping at the first failure. Each state
+     * write takes the owner's lease on its own; no lease is held while a deploy waits.
+     */
+    private runBuiltinFanOut;
     /**
      * Phase 2 for one org: takes the copy lock of each stage copy request under the org's lease, then runs the
      * copies after the lease is released. While a copy runs, every other tick and apply for the org goes on, a

package/dist/plan/plan-apply.js

@@ -4,6 +4,7 @@ import { VERSIONS, floatingTagBlocked } from '../bootstrap/versions.js';
 import { sfuSlotPorts } from '../calls-sfu.js';
 import { ORG_SERVICE_SCHEMAS, heldEnvRef, ownsServices, validateHosting } from '../org.js';
 import { PLATFORM_SERVICE_KEYS } from '../platform.js';
+import { runBuiltinFanOut } from '../release/fanout.js';
 import { recordedFrom, releaseKeys } from '../release/state.js';
 import { AlreadyRunningError, StepFailedError } from '../runner/errors.js';
 import { StaleLeaseError, } from '../store.js';
@@ -1338,6 +1339,16 @@ export class FoundryPlanApply {
                 settle('platform', e);
             }
         }
+        // Between the phases: once the owner's Boule or Agora is fully live, copy it to every other org. Its errors
+        // are reported by app, never as an org.
+        for (const app of ['boule', 'agora']) {
+            try {
+                await this.runBuiltinFanOut(app);
+            }
+            catch (e) {
+                settle(`fanout:${app}`, e);
+            }
+        }
         // Phase 2: the copies, one subject at a time. A copy takes its lock under the org's lease and runs without it.
         for (const { target, run } of copying) {
             try {
@@ -1350,6 +1361,65 @@ export class FoundryPlanApply {
         }
         return results;
     }
+    /**
+     * One fan-out pass for the owner org's built-in `app` (`release/fanout.ts`): the other orgs' copies are put on
+     * the commit the owner's copy runs, one after the other, in place, stopping at the first failure. Each state
+     * write takes the owner's lease on its own; no lease is held while a deploy waits.
+     */
+    async runBuiltinFanOut(app) {
+        const store = this.releaseStore;
+        const platform = await this.platformDeps?.store.get();
+        if (!store || !platform || !this.executor.deployBuiltinAt)
+            return;
+        const ownerOrgId = platform.ownerOrgId;
+        const owner = await this.store.get(ownerOrgId);
+        if (!owner)
+            return;
+        const empty = { orgId: ownerOrgId, steps: [], unmanaged: [] };
+        if ((await this.authz.canApplyPlatform?.(this.actor)) !== true) {
+            await this.refusePlatform(empty, ownerOrgId, 'not authorized to run the release tick');
+            throw new NotAuthorizedError(this.actor, ownerOrgId);
+        }
+        const deploy = this.executor.deployBuiltinAt.bind(this.executor);
+        await runBuiltinFanOut({
+            ownerOrgId,
+            app,
+            ownerDeclares: owner[app]?.blueGreen !== undefined,
+            store,
+            readIds: async () => ({
+                ...((await this.store.getAppliedState(ownerOrgId))?.serviceIds ?? {}),
+            }),
+            write: (writes) => this.withMutationLease(companyScope(ownerOrgId), (reason) => this.refuse(empty, reason), async (fence) => {
+                const previous = await this.store.getAppliedState(ownerOrgId);
+                if (!previous)
+                    throw new Error('the owner org has no applied state');
+                const serviceIds = { ...previous.serviceIds };
+                for (const [key, value] of Object.entries(writes)) {
+                    if (value === null)
+                        delete serviceIds[key];
+                    else
+                        serviceIds[key] = value;
+                }
+                await this.store.putAppliedState({ ...previous, serviceIds }, fence);
+            }),
+            targets: async () => {
+                const others = (await this.store.list()).filter((o) => o.id !== ownerOrgId && o[app] !== undefined);
+                return others.map((o) => ({ orgId: o.id, slug: o.slug }));
+            },
+            // Re-reads the org fresh right before its own deploy, never from a list taken when the pass
+            // started: a deploy can wait many minutes with no lease held on the target org (CLAUDE.md,
+            // "Built-in fan-out"), so an ordinary `apply()` to a later-in-order org can turn its own
+            // blue/green on in that window, and `deployBuiltinAt`'s guard must see that, not a stale copy.
+            deploy: async (target, sha) => {
+                const org = await this.store.get(target.orgId);
+                if (!org || org[app] === undefined)
+                    return { ok: false, reason: 'org not found' };
+                const recordedIds = (await this.store.getAppliedState(org.id))?.serviceIds ?? {};
+                return deploy({ app, org, recordedIds, sha });
+            },
+            now: this.now,
+        });
+    }
     /**
      * Phase 2 for one org: takes the copy lock of each stage copy request under the org's lease, then runs the
      * copies after the lease is released. While a copy runs, every other tick and apply for the org goes on, a

package/dist/platform.d.ts

@@ -20,6 +20,9 @@ export interface PlatformArchonService extends PlatformService {
     blueGreen?: {
         steps?: number[];
         stepHoldMinutes?: number;
+        advance?: 'time' | 'signal' | 'approval';
+        /** Archon's switch only: `{ steps: [100], staffFirst: true }` deploys the idle copy and waits for `open`. */
+        staffFirst?: boolean;
     };
 }
 /**

package/dist/release/fanout.d.ts

@@ -0,0 +1,77 @@
+import { type ReleaseStore } from './state.js';
+/** The built-in apps Company Included releases from Archon and then copies to every customer org. */
+export type FanOutApp = 'boule' | 'agora';
+export type FanOutOrgState = 'waiting' | 'deploying' | 'done' | 'failed';
+/** C1: what Archon shows for one app's fan-out. */
+export interface BuiltinFanOut {
+    digest: string | null;
+    done: boolean;
+    orgs: Array<{
+        orgId: string;
+        slug: string;
+        state: FanOutOrgState;
+        at: string | null;
+        reason: string | null;
+    }>;
+}
+/** Applied-state scope keys of one app's fan-out (on the owner org). */
+export declare function fanOutKeys(app: FanOutApp): {
+    /** The commit being copied. */
+    digest: string;
+    /** The commit whose copy finished on every org. */
+    done: string;
+    orgPrefix: string;
+    /** `<state>|<ISO time>|<org id>|<fixed reason>` */
+    org: (slug: string) => string;
+};
+/** C1: where the fan-out of the owner's Boule or Agora stands, from the owner org's applied state. */
+export declare function readBuiltinFanOut(store: Pick<ReleaseStore, 'getScopeIds'>, ownerOrgId: string, app: FanOutApp): Promise<BuiltinFanOut>;
+/**
+ * The commit the owner's copy runs when it is fully live: one colour at 100 %, the other at 0, no release
+ * running and no open step window. Null otherwise.
+ */
+export declare function fullyLiveDigest(ids: Readonly<Record<string, string>>, app: FanOutApp): string | null;
+export type FanOutDeployResult = {
+    ok: true;
+} | {
+    ok: false;
+    reason: string;
+};
+export interface FanOutDeps {
+    ownerOrgId: string;
+    app: FanOutApp;
+    /** Whether the owner's `org.<app>.blueGreen` is declared. */
+    ownerDeclares: boolean;
+    store: Pick<ReleaseStore, 'openRequests' | 'consumeRequests' | 'appendEvents'>;
+    /** The owner org's recorded ids now. */
+    readIds(): Promise<Record<string, string>>;
+    /** Persists scope keys on the owner org (null removes one), under a short lease. */
+    write(writes: Record<string, string | null>): Promise<void>;
+    /** Every other org that has the built-in app, in any order. Never the owner. */
+    targets(): Promise<Array<{
+        orgId: string;
+        slug: string;
+    }>>;
+    /** Puts the org's copy on `sha` in place and waits for the deployment. Fixed-text reasons only. */
+    deploy(target: {
+        orgId: string;
+        slug: string;
+    }, sha: string): Promise<FanOutDeployResult>;
+    now: () => Date;
+    /** Each deploy is abandoned after this long (default 12 minutes). */
+    deployTimeoutMs?: number;
+}
+export declare const FANOUT_DEPLOY_TIMEOUT_MS: number;
+export interface FanOutRun {
+    /** Why nothing deployed, or `deployed`, `done`, `stopped`. Fixed text. */
+    outcome: string;
+    deployed: string[];
+    failed: string | null;
+}
+/**
+ * One fan-out pass for one app (release tick, after the owner's release ran). Deploys each other org's copy at
+ * the owner's live commit, in slug order, and stops at the first failure. Every org is deployed in this pass
+ * unless one fails; a pass that dies halfway leaves the finished orgs `done` and the interrupted one `deploying`,
+ * which the next pass deploys again (a deploy of the same commit is safe to repeat).
+ */
+export declare function runBuiltinFanOut(deps: FanOutDeps): Promise<FanOutRun>;

package/dist/release/fanout.js

@@ -0,0 +1,177 @@
+import { recordedFrom } from './state.js';
+const COMMIT = /^[0-9a-f]{40}$/;
+const SLUG = /^[a-z0-9][a-z0-9-]*$/;
+const STATES = ['waiting', 'deploying', 'done', 'failed'];
+/** Applied-state scope keys of one app's fan-out (on the owner org). */
+export function fanOutKeys(app) {
+    const p = `fanout.${app}`;
+    return {
+        /** The commit being copied. */
+        digest: `${p}.digest`,
+        /** The commit whose copy finished on every org. */
+        done: `${p}.done`,
+        orgPrefix: `${p}.org.`,
+        /** `<state>|<ISO time>|<org id>|<fixed reason>` */
+        org: (slug) => `${p}.org.${slug}`,
+    };
+}
+function assertApp(app) {
+    if (app !== 'boule' && app !== 'agora')
+        throw new Error('invalid fan-out app');
+}
+function parseOrg(ids, key, slug) {
+    if (!SLUG.test(slug))
+        return null;
+    const [state, at, orgId, ...rest] = (ids[key] ?? '').split('|');
+    if (!state || !STATES.includes(state) || !orgId)
+        return null;
+    return {
+        orgId,
+        slug,
+        state: state,
+        at: at || null,
+        reason: rest.join('|') || null,
+    };
+}
+function readFrom(ids, app) {
+    const k = fanOutKeys(app);
+    const orgs = Object.keys(ids)
+        .filter((key) => key.startsWith(k.orgPrefix))
+        .map((key) => parseOrg(ids, key, key.slice(k.orgPrefix.length)))
+        .filter((o) => o !== null)
+        .sort((a, b) => (a.slug < b.slug ? -1 : a.slug > b.slug ? 1 : 0));
+    const digest = ids[k.digest] ?? null;
+    return { digest, done: digest !== null && ids[k.done] === digest, orgs };
+}
+/** C1: where the fan-out of the owner's Boule or Agora stands, from the owner org's applied state. */
+export async function readBuiltinFanOut(store, ownerOrgId, app) {
+    assertApp(app);
+    return readFrom(await store.getScopeIds(ownerOrgId), app);
+}
+/**
+ * The commit the owner's copy runs when it is fully live: one colour at 100 %, the other at 0, no release
+ * running and no open step window. Null otherwise.
+ */
+export function fullyLiveDigest(ids, app) {
+    const r = recordedFrom({ ...ids }, app);
+    if (!r || r.release !== null)
+        return null;
+    const other = r.live === 'blue' ? 'green' : 'blue';
+    if (r.weights[r.live] !== 100 || r.weights[other] !== 0)
+        return null;
+    if (!['idle', 'passed', 'rolled_back'].includes(r.window.kind))
+        return null;
+    const digest = r.digests[r.live];
+    return digest && COMMIT.test(digest) ? digest : null;
+}
+export const FANOUT_DEPLOY_TIMEOUT_MS = 12 * 60_000;
+async function bounded(run, ms) {
+    let timer;
+    const timeout = new Promise((resolve) => {
+        timer = setTimeout(() => resolve({ ok: false, reason: 'deployment timed out' }), ms);
+    });
+    try {
+        return await Promise.race([run, timeout]);
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+/**
+ * One fan-out pass for one app (release tick, after the owner's release ran). Deploys each other org's copy at
+ * the owner's live commit, in slug order, and stops at the first failure. Every org is deployed in this pass
+ * unless one fails; a pass that dies halfway leaves the finished orgs `done` and the interrupted one `deploying`,
+ * which the next pass deploys again (a deploy of the same commit is safe to repeat).
+ */
+export async function runBuiltinFanOut(deps) {
+    const { app } = deps;
+    assertApp(app);
+    const k = fanOutKeys(app);
+    const none = (outcome) => ({ outcome, deployed: [], failed: null });
+    const retries = (await deps.store.openRequests({ orgId: deps.ownerOrgId, entryKey: app })).filter((r) => r.action.kind === 'fanout_retry');
+    if (retries.length > 0) {
+        const failedNow = readFrom(await deps.readIds(), app).orgs.filter((o) => o.state === 'failed');
+        const writes = {};
+        for (const o of failedNow)
+            writes[k.org(o.slug)] = `waiting|${deps.now().toISOString()}|${o.orgId}|`;
+        if (Object.keys(writes).length > 0)
+            await deps.write(writes);
+        await deps.store.consumeRequests(retries.map((r) => r.id));
+    }
+    if (!deps.ownerDeclares)
+        return none('owner has no blue/green copy');
+    const ids = await deps.readIds();
+    const digest = fullyLiveDigest(ids, app);
+    if (!digest)
+        return none('owner copy is not fully live on one commit');
+    if (ids[k.done] === digest)
+        return none('commit already copied');
+    const targets = [...(await deps.targets())]
+        .filter((t) => t.orgId !== deps.ownerOrgId && SLUG.test(t.slug))
+        .sort((a, b) => (a.slug < b.slug ? -1 : a.slug > b.slug ? 1 : 0));
+    const stamp = deps.now().toISOString();
+    const current = readFrom(ids, app);
+    const fresh = current.digest !== digest;
+    const sync = {};
+    if (fresh) {
+        sync[k.digest] = digest;
+        sync[k.done] = null;
+    }
+    const states = new Map(current.orgs.map((o) => [o.slug, o]));
+    for (const o of current.orgs) {
+        if (!targets.some((t) => t.slug === o.slug))
+            sync[k.org(o.slug)] = null;
+    }
+    for (const t of targets) {
+        const have = states.get(t.slug);
+        if (fresh || !have || have.orgId !== t.orgId) {
+            sync[k.org(t.slug)] = `waiting|${stamp}|${t.orgId}|`;
+            states.set(t.slug, { ...t, state: 'waiting', at: stamp, reason: null });
+        }
+    }
+    if (Object.keys(sync).length > 0)
+        await deps.write(sync);
+    const run = { outcome: 'done', deployed: [], failed: null };
+    for (const t of targets) {
+        const state = states.get(t.slug)?.state;
+        if (state === 'done')
+            continue;
+        if (state === 'failed')
+            return { ...run, outcome: 'stopped', failed: t.slug };
+        await deps.write({ [k.org(t.slug)]: `deploying|${deps.now().toISOString()}|${t.orgId}|` });
+        let result;
+        try {
+            result = await bounded(deps.deploy(t, digest), deps.deployTimeoutMs ?? FANOUT_DEPLOY_TIMEOUT_MS);
+        }
+        catch (error) {
+            console.error(`fanout ${app}: deploy threw ${error instanceof Error ? error.name : 'Error'}`);
+            result = { ok: false, reason: 'deployment failed' };
+        }
+        const at = deps.now().toISOString();
+        if (result.ok) {
+            await deps.write({ [k.org(t.slug)]: `done|${at}|${t.orgId}|` });
+            run.deployed.push(t.slug);
+            await record(deps, { at, kind: 'fanout_done', reason: t.slug });
+            continue;
+        }
+        const reason = result.reason.replace(/\|/g, ' ').slice(0, 120);
+        await deps.write({ [k.org(t.slug)]: `failed|${at}|${t.orgId}|${reason}` });
+        await record(deps, { at, kind: 'fanout_failed', reason: `${t.slug}: ${reason}` });
+        return { ...run, outcome: 'stopped', failed: t.slug };
+    }
+    await deps.write({ [k.done]: digest });
+    return run;
+}
+async function record(deps, event) {
+    try {
+        await deps.store.appendEvents({
+            orgId: deps.ownerOrgId,
+            entryKey: deps.app,
+            events: [event],
+        });
+    }
+    catch {
+        // The history is a record, not a gate: the state above already says what happened.
+        console.error(`fanout ${deps.app}: recording a history line failed`);
+    }
+}

package/dist/release/run.js

@@ -28,7 +28,8 @@ export async function runRelease(input) {
         };
     }
     const k = releaseKeys(entryKey);
-    const all = await store.openRequests({ orgId, entryKey });
+    // `fanout_retry` belongs to the built-in fan-out (`fanout.ts`), which consumes it; this run leaves it open.
+    const all = (await store.openRequests({ orgId, entryKey })).filter((r) => r.action.kind !== 'fanout_retry');
     // A stage request whose result was persisted before a crash is consumed, never run a second time.
     const handledId = ids[k.stageCopyHandled];
     const replayed = all.filter((r) => isStageCopyAction(r.action) && String(r.id) === handledId);

package/dist/release/state.d.ts

@@ -66,7 +66,7 @@ export declare function releaseKeys(entry: string): {
 };
 export declare function formatWeights(w: Record<Color, number>): string;
 export declare function parseWeights(value: string | undefined): Record<Color, number> | null;
-export declare const RELEASE_EVENT_KINDS: readonly ["promote", "step", "check", "pause", "resume", "step_now", "approve", "open", "rollback", "passed", "hard_fail"];
+export declare const RELEASE_EVENT_KINDS: readonly ["promote", "step", "check", "pause", "resume", "step_now", "approve", "open", "rollback", "passed", "hard_fail", "fanout_done", "fanout_failed"];
 /** What `foundry_release_events.payload` holds: a line's fields besides `at` and `kind`. */
 export declare const releaseEventPayload: z.ZodObject<{
     percent: z.ZodOptional<z.ZodNumber>;
@@ -100,6 +100,8 @@ export declare const releaseEventKind: z.ZodEnum<{
     resume: "resume";
     step_now: "step_now";
     approve: "approve";
+    fanout_done: "fanout_done";
+    fanout_failed: "fanout_failed";
 }>;
 /** The window JSON is the C2 state plus the active release's `release` field. */
 export declare function parseWindow(value: string | undefined): {

package/dist/release/state.js

@@ -78,6 +78,8 @@ export const RELEASE_EVENT_KINDS = [
     'rollback',
     'passed',
     'hard_fail',
+    'fanout_done',
+    'fanout_failed',
 ];
 /** What `foundry_release_events.payload` holds: a line's fields besides `at` and `kind`. */
 export const releaseEventPayload = z.object({
@@ -216,6 +218,7 @@ const actionSchema = z.discriminatedUnion('kind', [
     z.object({ kind: z.literal('deploy_stage'), ref: z.string().regex(STAGE_REF_PATTERN) }),
     z.object({ kind: z.literal('copy_prod_to_stage') }),
     z.object({ kind: z.literal('undo_stage_copy') }),
+    z.object({ kind: z.literal('fanout_retry') }),
 ]);
 /** C2: ask the runner to apply an action on its next run. Archon audits it. */
 export async function requestReleaseAction(store, input) {

package/dist/release/types.d.ts

@@ -107,8 +107,16 @@ export type ReleaseAction = {
     kind: 'copy_prod_to_stage';
 } | {
     kind: 'undo_stage_copy';
+}
+/** Owner org, entry key `boule` or `agora`: continue the built-in fan-out from the org that failed. */
+ | {
+    kind: 'fanout_retry';
 };
-export type ReleaseEventKind = 'promote' | 'step' | 'check' | 'pause' | 'resume' | 'step_now' | 'approve' | 'open' | 'rollback' | 'passed' | 'hard_fail';
+export type ReleaseEventKind = 'promote' | 'step' | 'check' | 'pause' | 'resume' | 'step_now' | 'approve' | 'open' | 'rollback' | 'passed' | 'hard_fail'
+/** The built-in fan-out put an org's copy on the live commit (`reason` is the org slug). */
+ | 'fanout_done'
+/** The built-in fan-out stopped at an org (`reason` is `<slug>: <fixed text>`). */
+ | 'fanout_failed';
 /** C1: one line of a release's history, written by the release run (`foundry_release_events`). */
 export interface ReleaseEvent {
     at: string;

package/dist/store/schema.d.ts

@@ -2173,6 +2173,8 @@ export declare const foundryDeclaredPlatform: import("drizzle-orm/pg-core").PgTa
             data: {
                 steps?: number[];
                 stepHoldMinutes?: number;
+                advance?: "time" | "signal" | "approval";
+                staffFirst?: boolean;
             } | null;
             driverParam: unknown;
             notNull: false;
@@ -2188,6 +2190,8 @@ export declare const foundryDeclaredPlatform: import("drizzle-orm/pg-core").PgTa
             $type: {
                 steps?: number[];
                 stepHoldMinutes?: number;
+                advance?: "time" | "signal" | "approval";
+                staffFirst?: boolean;
             } | null;
         }>;
         edgeEnabled: import("drizzle-orm/pg-core").PgColumn<{

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wtfalch/foundry",
-  "version": "0.70.1",
+  "version": "0.71.0",
   "description": "Foundry's SDK: the declared org's types, the plan and apply contracts, and the state store's interface.",
   "repository": {
     "type": "git",