@wtfalch/foundry 0.62.0 → 0.63.0

package/dist/plan/platform-registry.d.ts

@@ -0,0 +1,67 @@
+import type { DeclaredPlatform } from '../platform.js';
+import type { CoolifyPort } from './executor.js';
+import type { PlatformStepExecutionContext } from './plan-apply.js';
+import { type PlatformDnsFactory, type PlatformTunnelsFactory } from './platform-providers.js';
+export interface PlatformRegistryRuntime {
+    fetch?: typeof fetch;
+    sleep?: (ms: number) => Promise<void>;
+    now?: () => number;
+    readyTimeoutMs?: number;
+    readyIntervalMs?: number;
+    /** Tries at handing each password to the host (default 3). */
+    deliverAttempts?: number;
+    /** Told of every password this step makes or reads, so the host can scrub it from logs and audit rows. */
+    onSecret?: (value: string) => void;
+}
+/**
+ * Everything the registry steps need beyond {@link PlatformStepExecutionContext}. `deliverSecret`
+ * and `heldKeys` are the platform's held-key writes and reads (edge's pattern); `listServers` and
+ * `raw` reach every server Coolify manages, not only the platform's.
+ */
+export interface PlatformRegistryDeps {
+    coolify: Pick<CoolifyPort, 'findProjectByName' | 'createProject' | 'findOrCreateEnvironment' | 'listServers' | 'findApplicationByName' | 'getApplication' | 'createDockerImageApplication' | 'createDockerfileApplication' | 'listStorages' | 'createPersistentStorage' | 'createFileStorage' | 'updateFileStorage' | 'bulkUpsertEnvs' | 'listEnvs' | 'updateApplication' | 'deploy' | 'deployAndWait' | 'raw'>;
+    /** F4.dns: every DNS record call, through edge at its recorded address. */
+    dns: PlatformDnsFactory;
+    /** F4.tunnels: every tunnel call, through edge at its recorded address. */
+    tunnels: PlatformTunnelsFactory;
+    deliverSecret: (ref: string, secret: string) => Promise<void>;
+    /** Reads a held key back. The login step reads the `pull` password this way. */
+    heldKeys?: {
+        resolve(ref: string): Promise<string | null>;
+    };
+    runtime?: PlatformRegistryRuntime;
+}
+/** One htpasswd line (`user:bcrypt hash`). Docker's `distribution` accepts bcrypt only. */
+export declare function htpasswdLine(user: string, password: string): string;
+/**
+ * The registry app: `registry:3` (by digest), the data volume, the htpasswd file, the tunnel and
+ * the DNS record. Safe to run again at any point:
+ * - The two passwords are made once. The htpasswd file is written before they are recorded as
+ *   undelivered, so the only window that blocks a re-run is the one the other platform steps
+ *   have (CLAUDE.md, rule 1).
+ * - The app is deployed when it was created, is not running, its env changed, the declared
+ *   hostname or tag changed, or the passwords were made in this run.
+ * - The applied hostname and tag are recorded last.
+ */
+export declare function provisionPlatformRegistry(context: PlatformStepExecutionContext, deps: PlatformRegistryDeps): Promise<{
+    resourceId?: string;
+}>;
+/**
+ * Logs every Coolify server in to the registry as `pull` (`POST /servers/{uuid}/registries`,
+ * which runs `docker login` on the server and replaces an earlier login to the same host).
+ * - A server already recorded as logged in to this host with these passwords is left alone.
+ * - A server Coolify marks unusable or unreachable is skipped and named under
+ *   {@link PLATFORM_REGISTRY_LOGINS_SKIPPED_SCOPE_KEY}; `driftedSteps` plans the step again
+ *   once it is reachable.
+ * - A login that fails is counted; the others still run, each success is recorded at once, and
+ *   the step then throws a text naming the server ids, so the next apply tries only those again.
+ * - Every Coolify call has a timeout. The password is read from the held keys and goes only
+ *   into the request body.
+ */
+export declare function ensureRegistryLogins(context: PlatformStepExecutionContext, deps: PlatformRegistryDeps): Promise<Record<string, never>>;
+/**
+ * True when a usable, reachable server has no login recorded for the current registry host and
+ * passwords: a server added, or back up, since the last login run. A read that fails counts as
+ * not drifted, like the Archon drift reads.
+ */
+export declare function registryLoginsDrifted(deps: Pick<PlatformRegistryDeps, 'coolify'>, platform: DeclaredPlatform, recordedIds: Readonly<Record<string, string>>): Promise<boolean>;

package/dist/plan/platform-registry.js

@@ -0,0 +1,298 @@
+import { createHash, randomBytes } from 'node:crypto';
+import bcrypt from 'bcryptjs';
+import { coolifyImageFor } from '../bootstrap/versions.js';
+import { CallError, withTimeout } from './call-timeout.js';
+import { ensureCloudflaredConnector } from './cloudflared-connector.js';
+import { waitForNewDeployment } from './deploy-health.js';
+import { commandHealthCheckSettings, tcpProbe } from './health-check-settings.js';
+import { envsDiffer, findOrCreatePlatformApp, platformDns, platformTunnels, resolvePlatformServer, } from './platform-providers.js';
+import { PLATFORM_REGISTRY_APPLIED_GIT_REF_SCOPE_KEY, PLATFORM_REGISTRY_APPLIED_HOSTNAME_SCOPE_KEY, PLATFORM_REGISTRY_APP_SCOPE_KEY, PLATFORM_REGISTRY_CONNECTOR_APP_SCOPE_KEY, PLATFORM_REGISTRY_INTERNAL_URL_SCOPE_KEY, PLATFORM_REGISTRY_LOGINS_APPLIED_SCOPE_KEY, PLATFORM_REGISTRY_LOGINS_SKIPPED_SCOPE_KEY, PLATFORM_REGISTRY_PASSWORDS_SET_SCOPE_KEY, PLATFORM_REGISTRY_PASSWORDS_UNDELIVERED_SCOPE_KEY, PLATFORM_REGISTRY_PROJECT_SCOPE_KEY, PLATFORM_REGISTRY_PULL_PASSWORD_REF, PLATFORM_REGISTRY_PULL_USER, PLATFORM_REGISTRY_PUSH_PASSWORD_REF, PLATFORM_REGISTRY_PUSH_USER, PLATFORM_REGISTRY_TUNNEL_SCOPE_KEY, registryImageTagBlocked, registryLoginVersion, registryServerLoginScopeKey, } from './platform-registry-keys.js';
+const REGISTRY_IMAGE = 'registry';
+const REGISTRY_PROJECT = 'platform';
+const REGISTRY_APP_NAME = 'platform-registry';
+const REGISTRY_INTERNAL_NAME = 'platform-registry';
+const REGISTRY_PORT = '5000';
+const REGISTRY_TUNNEL_NAME = 'platform-registry-tunnel';
+const REGISTRY_CLOUDFLARED_APP_NAME = 'platform-registry-cloudflared';
+const REGISTRY_DATA_DIR = '/var/lib/registry';
+const REGISTRY_HTPASSWD_PATH = '/auth/htpasswd';
+/** Coolify's per-call limit on a registry login. It runs `docker login` on the server over SSH. */
+const REGISTRY_LOGIN_TIMEOUT_MS = 60_000;
+const REGISTRY_SERVER_LIST_TIMEOUT_MS = 30_000;
+const REGISTRY_HELD_READ_TIMEOUT_MS = 30_000;
+const DEFAULT_DELIVER_ATTEMPTS = 3;
+/** One htpasswd line (`user:bcrypt hash`). Docker's `distribution` accepts bcrypt only. */
+export function htpasswdLine(user, password) {
+    return `${user}:${bcrypt.hashSync(password, 10)}`;
+}
+function newPassword() {
+    return randomBytes(32).toString('base64url');
+}
+const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+async function deliverWithRetry(deliverSecret, ref, secret, attempts, sleep) {
+    for (let attempt = 1;; attempt += 1) {
+        try {
+            await deliverSecret(ref, secret);
+            return;
+        }
+        catch {
+            if (attempt >= attempts) {
+                throw new Error(`platform:registry: the password held at ${ref} could not be delivered after ${attempts} tries; the next plan reports it as blocked and mints no new one`);
+            }
+            await sleep(1000 * attempt);
+        }
+    }
+}
+/**
+ * The registry app: `registry:3` (by digest), the data volume, the htpasswd file, the tunnel and
+ * the DNS record. Safe to run again at any point:
+ * - The two passwords are made once. The htpasswd file is written before they are recorded as
+ *   undelivered, so the only window that blocks a re-run is the one the other platform steps
+ *   have (CLAUDE.md, rule 1).
+ * - The app is deployed when it was created, is not running, its env changed, the declared
+ *   hostname or tag changed, or the passwords were made in this run.
+ * - The applied hostname and tag are recorded last.
+ */
+export async function provisionPlatformRegistry(context, deps) {
+    const { platform } = context;
+    const { hostname, gitRef } = platform.registry;
+    const tagBlocked = registryImageTagBlocked(gitRef);
+    if (tagBlocked)
+        throw new Error(tagBlocked);
+    const recorded = context.recordedIds();
+    const runtime = deps.runtime ?? {};
+    const sleep = runtime.sleep ?? defaultSleep;
+    const record = async (key, id) => {
+        context.recordScopeId(key, id);
+        await context.persist?.();
+    };
+    if (recorded[PLATFORM_REGISTRY_PASSWORDS_UNDELIVERED_SCOPE_KEY] &&
+        !recorded[PLATFORM_REGISTRY_PASSWORDS_SET_SCOPE_KEY]) {
+        throw new Error(`platform:registry: the passwords were made and their delivery is not confirmed; remove the recorded id ${PLATFORM_REGISTRY_PASSWORDS_UNDELIVERED_SCOPE_KEY} to make new ones`);
+    }
+    let projectUuid = recorded[PLATFORM_REGISTRY_PROJECT_SCOPE_KEY];
+    if (!projectUuid) {
+        const found = await deps.coolify.findProjectByName(REGISTRY_PROJECT);
+        projectUuid = found ? found.uuid : (await deps.coolify.createProject(REGISTRY_PROJECT)).uuid;
+        await record(PLATFORM_REGISTRY_PROJECT_SCOPE_KEY, projectUuid);
+    }
+    const { environment } = await deps.coolify.findOrCreateEnvironment(projectUuid, 'production');
+    const server = await resolvePlatformServer(deps.coolify);
+    // ---- the app ---------------------------------------------------------------------------
+    const app = await findOrCreatePlatformApp(deps.coolify, REGISTRY_APP_NAME, environment, () => deps.coolify.createDockerImageApplication({
+        project_uuid: projectUuid,
+        server_uuid: server.uuid,
+        environment_name: environment.name,
+        environment_uuid: environment.uuid,
+        ...coolifyImageFor('registry', REGISTRY_IMAGE, gitRef),
+        ports_exposes: REGISTRY_PORT,
+        name: REGISTRY_APP_NAME,
+        description: `The platform's image registry (C5, ${REGISTRY_IMAGE}:${gitRef})`,
+        instant_deploy: false,
+    }));
+    const appId = app.appId;
+    await record(PLATFORM_REGISTRY_APP_SCOPE_KEY, appId);
+    // On every run: a changed tag reaches an app made on an earlier one.
+    await deps.coolify.updateApplication(appId, {
+        custom_internal_name: REGISTRY_INTERNAL_NAME,
+        ...coolifyImageFor('registry', REGISTRY_IMAGE, gitRef),
+        // The registry answers 401 on every route without a password, so Coolify's HTTP check cannot pass: a TCP connect.
+        ...commandHealthCheckSettings(tcpProbe(REGISTRY_PORT)),
+    });
+    // ---- the data volume ---------------------------------------------------------------------
+    const storages = await deps.coolify.listStorages(appId);
+    let storageAdded = false;
+    if (!(storages.persistent_storages ?? []).some((s) => s.mount_path === REGISTRY_DATA_DIR)) {
+        await deps.coolify.createPersistentStorage(appId, {
+            name: `${REGISTRY_APP_NAME}-data`,
+            mount_path: REGISTRY_DATA_DIR,
+        });
+        storageAdded = true;
+    }
+    // ---- the passwords, once -----------------------------------------------------------------
+    let mintedNow = false;
+    let set = recorded[PLATFORM_REGISTRY_PASSWORDS_SET_SCOPE_KEY] ?? '';
+    if (!set) {
+        const push = newPassword();
+        const pull = newPassword();
+        runtime.onSecret?.(push);
+        runtime.onSecret?.(pull);
+        set = createHash('sha256').update(`${push}\n${pull}`).digest('hex');
+        // The file first: with the passwords only in memory, a crash before they are recorded makes new ones
+        // and writes the file again, so the file and the held passwords never disagree.
+        const content = `${htpasswdLine(PLATFORM_REGISTRY_PUSH_USER, push)}\n${htpasswdLine(PLATFORM_REGISTRY_PULL_USER, pull)}\n`;
+        const existing = (storages.file_storages ?? []).find((s) => s.mount_path === REGISTRY_HTPASSWD_PATH);
+        if (existing) {
+            await deps.coolify.updateFileStorage(appId, existing.uuid, content);
+        }
+        else {
+            await deps.coolify.createFileStorage(appId, {
+                mount_path: REGISTRY_HTPASSWD_PATH,
+                content,
+            });
+        }
+        await record(PLATFORM_REGISTRY_PASSWORDS_UNDELIVERED_SCOPE_KEY, set);
+        const attempts = runtime.deliverAttempts ?? DEFAULT_DELIVER_ATTEMPTS;
+        await deliverWithRetry(deps.deliverSecret, PLATFORM_REGISTRY_PUSH_PASSWORD_REF, push, attempts, sleep);
+        await deliverWithRetry(deps.deliverSecret, PLATFORM_REGISTRY_PULL_PASSWORD_REF, pull, attempts, sleep);
+        await record(PLATFORM_REGISTRY_PASSWORDS_SET_SCOPE_KEY, set);
+        mintedNow = true;
+    }
+    // ---- env, no secret in it -----------------------------------------------------------------
+    const envs = [
+        { key: 'REGISTRY_AUTH', value: 'htpasswd' },
+        { key: 'REGISTRY_AUTH_HTPASSWD_REALM', value: 'Registry' },
+        { key: 'REGISTRY_AUTH_HTPASSWD_PATH', value: REGISTRY_HTPASSWD_PATH },
+        { key: 'REGISTRY_HTTP_ADDR', value: `:${REGISTRY_PORT}` },
+        // The tunnel and Cloudflare sit in front: the registry answers with paths, not with its own address.
+        { key: 'REGISTRY_HTTP_RELATIVEURLS', value: 'true' },
+        { key: 'REGISTRY_STORAGE_DELETE_ENABLED', value: 'false' },
+    ];
+    const envsChanged = await envsDiffer(deps.coolify, appId, envs);
+    await deps.coolify.bulkUpsertEnvs(appId, envs);
+    const declaredChanged = recorded[PLATFORM_REGISTRY_APPLIED_HOSTNAME_SCOPE_KEY] !== hostname ||
+        recorded[PLATFORM_REGISTRY_APPLIED_GIT_REF_SCOPE_KEY] !== gitRef;
+    const internalUrl = `http://${REGISTRY_INTERNAL_NAME}:${REGISTRY_PORT}`;
+    if (app.justCreated ||
+        !app.running ||
+        envsChanged ||
+        declaredChanged ||
+        mintedNow ||
+        storageAdded) {
+        // The registry answers 401 to a caller with no password, so the deployment record is the check.
+        await waitForNewDeployment(deps.coolify, appId, { kind: 'deployment-only' }, runtime, sleep, 'platform:registry', `the registry stays at ${recorded[PLATFORM_REGISTRY_APPLIED_GIT_REF_SCOPE_KEY] ?? 'no recorded ref'}`);
+    }
+    await record(PLATFORM_REGISTRY_INTERNAL_URL_SCOPE_KEY, internalUrl);
+    // ---- tunnel and DNS ------------------------------------------------------------------------
+    const tunnels = await platformTunnels(deps, context.recordedIds(), 'platform:registry');
+    const existingTunnel = await tunnels.findTunnelByName(REGISTRY_TUNNEL_NAME);
+    const tunnel = existingTunnel ?? (await tunnels.createTunnel(REGISTRY_TUNNEL_NAME));
+    await record(PLATFORM_REGISTRY_TUNNEL_SCOPE_KEY, tunnel.id);
+    await tunnels.putTunnelConfig(tunnel.id, {
+        ingress: [{ hostname, service: internalUrl }, { service: 'http_status:404' }],
+    });
+    await ensureCloudflaredConnector({
+        coolify: deps.coolify,
+        tunnels,
+        tunnelId: tunnel.id,
+        appName: REGISTRY_CLOUDFLARED_APP_NAME,
+        projectUuid,
+        environment,
+        resolveServer: async () => server,
+        recordedConnectorAppId: recorded[PLATFORM_REGISTRY_CONNECTOR_APP_SCOPE_KEY] ?? null,
+        recordAppId: (connectorAppId) => record(PLATFORM_REGISTRY_CONNECTOR_APP_SCOPE_KEY, connectorAppId),
+    });
+    const dns = await platformDns(deps, context.recordedIds(), 'platform:registry');
+    const dnsTarget = `${tunnel.id}.cfargotunnel.com`;
+    const [dnsRecord] = await dns.findDnsRecord(platform.apex, hostname, 'CNAME');
+    if (!dnsRecord) {
+        await dns.createDnsRecord(platform.apex, {
+            type: 'CNAME',
+            name: hostname,
+            content: dnsTarget,
+            proxied: true,
+        });
+    }
+    else if (dnsRecord.content !== dnsTarget) {
+        await dns.updateDnsRecord(platform.apex, dnsRecord.id, {
+            type: 'CNAME',
+            name: hostname,
+            content: dnsTarget,
+            proxied: true,
+        });
+    }
+    // Last: a run that failed anywhere above is planned again.
+    context.recordScopeId(PLATFORM_REGISTRY_APPLIED_HOSTNAME_SCOPE_KEY, hostname);
+    await record(PLATFORM_REGISTRY_APPLIED_GIT_REF_SCOPE_KEY, gitRef);
+    return { resourceId: appId };
+}
+/**
+ * Logs every Coolify server in to the registry as `pull` (`POST /servers/{uuid}/registries`,
+ * which runs `docker login` on the server and replaces an earlier login to the same host).
+ * - A server already recorded as logged in to this host with these passwords is left alone.
+ * - A server Coolify marks unusable or unreachable is skipped and named under
+ *   {@link PLATFORM_REGISTRY_LOGINS_SKIPPED_SCOPE_KEY}; `driftedSteps` plans the step again
+ *   once it is reachable.
+ * - A login that fails is counted; the others still run, each success is recorded at once, and
+ *   the step then throws a text naming the server ids, so the next apply tries only those again.
+ * - Every Coolify call has a timeout. The password is read from the held keys and goes only
+ *   into the request body.
+ */
+export async function ensureRegistryLogins(context, deps) {
+    const { hostname } = context.platform.registry;
+    const recorded = context.recordedIds();
+    const set = recorded[PLATFORM_REGISTRY_PASSWORDS_SET_SCOPE_KEY];
+    if (!set) {
+        throw new Error('platform:registry-logins: the registry passwords are not set yet; platform:registry runs first');
+    }
+    if (!deps.heldKeys) {
+        throw new Error(`platform:registry-logins: the host cannot read held keys, so the pull password at ${PLATFORM_REGISTRY_PULL_PASSWORD_REF} cannot be read`);
+    }
+    const heldKeys = deps.heldKeys;
+    const pull = await withTimeout('platform:registry-logins: reading the pull password', REGISTRY_HELD_READ_TIMEOUT_MS, () => heldKeys.resolve(PLATFORM_REGISTRY_PULL_PASSWORD_REF));
+    if (!pull) {
+        throw new Error(`platform:registry-logins: no password is held at ${PLATFORM_REGISTRY_PULL_PASSWORD_REF}`);
+    }
+    deps.runtime?.onSecret?.(pull);
+    const record = async (key, id) => {
+        context.recordScopeId(key, id);
+        await context.persist?.();
+    };
+    const version = registryLoginVersion(hostname, set);
+    const servers = await withTimeout('platform:registry-logins: listing the servers', REGISTRY_SERVER_LIST_TIMEOUT_MS, () => deps.coolify.listServers());
+    const skipped = [];
+    const failed = [];
+    for (const server of servers) {
+        if (!server.is_usable || !server.is_reachable) {
+            skipped.push(server.uuid);
+            continue;
+        }
+        const key = registryServerLoginScopeKey(server.uuid);
+        if (recorded[key] === version)
+            continue;
+        try {
+            await withTimeout(`platform:registry-logins: logging in server ${server.uuid}`, REGISTRY_LOGIN_TIMEOUT_MS, () => deps.coolify.raw('POST', `/servers/${server.uuid}/registries`, {
+                registry: hostname,
+                username: PLATFORM_REGISTRY_PULL_USER,
+                password: pull,
+            }));
+        }
+        catch (error) {
+            // The text of a CallError is fixed; anything else is dropped without reading it.
+            if (!(error instanceof CallError))
+                throw new Error('platform:registry-logins: a login call failed');
+            failed.push(server.uuid);
+            continue;
+        }
+        await record(key, version);
+    }
+    await record(PLATFORM_REGISTRY_LOGINS_SKIPPED_SCOPE_KEY, skipped.join(','));
+    if (failed.length > 0) {
+        throw new Error(`platform:registry-logins: could not log in ${failed.length} of ${servers.length} servers (${failed.join(', ')}); the next apply tries those again`);
+    }
+    await record(PLATFORM_REGISTRY_LOGINS_APPLIED_SCOPE_KEY, version);
+    return {};
+}
+/**
+ * True when a usable, reachable server has no login recorded for the current registry host and
+ * passwords: a server added, or back up, since the last login run. A read that fails counts as
+ * not drifted, like the Archon drift reads.
+ */
+export async function registryLoginsDrifted(deps, platform, recordedIds) {
+    const set = recordedIds[PLATFORM_REGISTRY_PASSWORDS_SET_SCOPE_KEY];
+    if (!set)
+        return false;
+    const version = registryLoginVersion(platform.registry.hostname, set);
+    // Planned already (never applied for this host and these passwords): not a drift.
+    if (recordedIds[PLATFORM_REGISTRY_LOGINS_APPLIED_SCOPE_KEY] !== version)
+        return false;
+    let servers;
+    try {
+        servers = await withTimeout('platform:registry-logins: listing the servers', REGISTRY_SERVER_LIST_TIMEOUT_MS, () => deps.coolify.listServers());
+    }
+    catch {
+        console.error('platform:registry-logins: could not list the servers; treating the logins as current');
+        return false;
+    }
+    return servers.some((s) => s.is_usable && s.is_reachable && recordedIds[registryServerLoginScopeKey(s.uuid)] !== version);
+}

package/dist/plan/release-host.d.ts

@@ -1,8 +1,9 @@
-import { type ReleaseDeps } from '../release/run.js';
-import type { ReleaseStore } from '../release/state.js';
+import { type ReleaseDeps, type StageCopyJob, type StageCopyOutcome } from '../release/run.js';
+import { type ReleaseStore } from '../release/state.js';
 import { type BlueGreenSubject, type PinHeldKeys } from './blue-green.js';
 import { type DeployHealthRuntime } from './deploy-health.js';
 import type { CoolifyPort } from './executor.js';
+import { type StageCopyHostPorts } from './stage-copy-host.js';
 /** The recorded ids a release run reads and writes: an org's apply context, or the platform's. */
 export interface ReleaseContext {
     recordedIds(): Readonly<Record<string, string>>;
@@ -40,7 +41,12 @@ export interface ReleaseHostInput {
     /** Extends the mutation lease; called before each outside call that can take long. */
     renew: () => Promise<void>;
     label: string;
+    /** What the stage copy needs; absent: copy requests are refused with `no-copy-port`. */
+    stageCopy?: StageCopyHostPorts;
 }
+/** A stage deploy waits at most this many polls of this long (10 minutes) for Coolify's deployment to finish. */
+export declare const STAGE_DEPLOY_ATTEMPTS = 60;
+export declare const STAGE_DEPLOY_INTERVAL_MS = 10000;
 export declare function createReleaseHost(input: ReleaseHostInput): ReleaseHost;
 /**
  * One release run for one subject, writing into `context` (the live recorded ids of the run that holds
@@ -57,7 +63,30 @@ export declare function runSubjectRelease(input: {
     context: ReleaseContext;
     now: Date;
     primaryUuid: string;
+    /** Leave stage copy requests open: an apply never runs one, and the tick's first phase defers them. */
+    deferCopies?: boolean;
 }): Promise<{
     blocked: string[];
     refused: string[];
+    deferredCopy: boolean;
+    /** The stage copy this run decided on. The caller runs it, after this run's writes and weights are done. */
+    copy: StageCopyJob | null;
+}>;
+/**
+ * Runs the stage copy a release run decided on and records what came of it. Call it after
+ * {@link runSubjectRelease} returned (that run's writes are persisted and its weights written), so a rollback
+ * decided in the same run never waits for the copy. The outcome goes into `context`, is persisted, and only then
+ * is the request consumed. `refused` is the operator text of a refusal. A host without ports refuses with `no-copy-port`; a lost lock throws and leaves the
+ * request open.
+ */
+export declare function finishStageCopy(input: {
+    store: Pick<ReleaseStore, 'consumeRequests'>;
+    host: ReleaseHost | null | undefined;
+    subject: Pick<BlueGreenSubject, 'id'>;
+    context: ReleaseContext;
+    now: Date;
+    job: StageCopyJob;
+}): Promise<{
+    outcome: StageCopyOutcome;
+    refused: string[];
 }>;

package/dist/plan/release-host.js

@@ -1,11 +1,16 @@
 import { randomBytes } from 'node:crypto';
 import { checkColorAddress, readReleaseHealth } from '../release/health.js';
-import { runRelease } from '../release/run.js';
+import { runRelease, runStageCopy, settleStageCopy, } from '../release/run.js';
+import { releaseKeys } from '../release/state.js';
 import { applyBlueGreenWeights, copiedEnv } from './blue-green-apply.js';
 import { ACTIONS_KEY_ENV, DEFAULT_WEIGHTS, LB_COLOR_HEADER, LB_PIN_HEADER, bgKey, formatWeights, mintPinKey, parseWeights, readPinKey, reportingEnv, } from './blue-green.js';
-import { waitForNewDeployment } from './deploy-health.js';
+import { waitForDeploymentFinished, waitForNewDeployment, } from './deploy-health.js';
 import { isNotFoundError } from './executor.js';
+import { runStageCopyRequest } from './stage-copy-host.js';
 const COMMIT = /^[0-9a-f]{40}$/;
+/** A stage deploy waits at most this many polls of this long (10 minutes) for Coolify's deployment to finish. */
+export const STAGE_DEPLOY_ATTEMPTS = 60;
+export const STAGE_DEPLOY_INTERVAL_MS = 10_000;
 /** Coolify reports a restarting container as `restarting:...`; a crash loop is that. */
 function statusOf(status) {
     if (status?.startsWith('running'))
@@ -129,6 +134,65 @@ export function createReleaseHost(input) {
             await waitForNewDeployment(coolify, appId, { kind: 'deployment-only' }, input.runtime, input.runtime.sleep ?? defaultSleep, `${input.label} ${color}`, `${color} stays as it was`);
         }
     };
+    /**
+     * Deploys the stage app at `ref` (a full sha, or a branch's head) and records the outcome as
+     * `bg.<id>.stagedeploy`. A failure is recorded, not thrown: the request is then consumed with the run, and the
+     * record is what tells the person. The text recorded is fixed, never an error message (it can echo env).
+     */
+    const deployStage = async (ref) => {
+        const at = new Date((input.runtime.now ?? Date.now)()).toISOString();
+        const fail = (step) => {
+            console.error(`${input.label}: deploying stage at ${ref} failed at ${step}`);
+            context.recordScopeId(bgKey.stageDeploy(id), `${at}|failed|${step}`);
+        };
+        const stageId = recorded()[bgKey.app(id, input.primaryUuid, 'stage')];
+        if (!stageId)
+            return fail('no stage app');
+        await input.renew();
+        try {
+            // A sha pins the commit; a branch goes back to its head. Env is not touched.
+            await coolify.updateApplication(stageId, COMMIT.test(ref) ? { git_commit_sha: ref } : { git_branch: ref, git_commit_sha: 'HEAD' });
+        }
+        catch {
+            return fail('select commit');
+        }
+        let deploymentUuid;
+        try {
+            ({ deploymentUuid } = await waitForDeploymentFinished(coolify, stageId, {
+                ...input.runtime,
+                deployAttempts: input.runtime.deployAttempts ?? STAGE_DEPLOY_ATTEMPTS,
+                deployIntervalMs: input.runtime.deployIntervalMs ?? STAGE_DEPLOY_INTERVAL_MS,
+            }, `${input.label} stage`, 'stage stays as it was'));
+        }
+        catch {
+            return fail('deploy');
+        }
+        let commit = COMMIT.test(ref) ? ref : '';
+        if (!commit && coolify.listApplicationDeployments) {
+            try {
+                const rows = await coolify.listApplicationDeployments(stageId, 5);
+                const row = rows.find((r) => r.deployment_uuid === deploymentUuid);
+                if (row?.commit && COMMIT.test(row.commit))
+                    commit = row.commit;
+            }
+            catch {
+                // The deploy worked; the commit is read from the deployment list on the next run.
+            }
+        }
+        if (!commit)
+            return fail('select commit');
+        if (!COMMIT.test(ref)) {
+            // Pinned to what was built, so a later redeploy of stage (an env change) never floats to the branch head.
+            try {
+                await coolify.updateApplication(stageId, { git_commit_sha: commit });
+            }
+            catch {
+                return fail('select commit');
+            }
+        }
+        context.recordScopeId(bgKey.stage(id), commit);
+        context.recordScopeId(bgKey.stageDeploy(id), `${at}|ok|${commit}`);
+    };
     const stop = async (color) => {
         for (const appId of colorApps(color)) {
             await input.renew();
@@ -174,12 +238,14 @@ export function createReleaseHost(input) {
                 }),
             };
         },
+        stageCopy: (request) => runStageCopyRequest({ subject, context, ports: input.stageCopy, renew: input.renew }, request),
         async effect(effect) {
             if (effect.kind === 'deploy')
                 await deployTo(effect.color, effect.digest);
             else if (effect.kind === 'stop')
                 await stop(effect.color);
-            // deploy_stage is executed by the stage deploy task (F4); until then it is passed on and ignored here.
+            else if (effect.kind === 'deploy_stage')
+                await deployStage(effect.ref);
         },
     };
     return {
@@ -244,8 +310,9 @@ export async function runSubjectRelease(input) {
     const { subject, context } = input;
     const id = subject.id;
     const ids = () => context.recordedIds();
-    if (!ids()[bgKey.applied(id, input.primaryUuid)])
-        return { blocked: [], refused: [] };
+    if (!ids()[bgKey.applied(id, input.primaryUuid)]) {
+        return { blocked: [], refused: [], deferredCopy: false, copy: null };
+    }
     if (!ids()[bgKey.live(id)]) {
         context.recordScopeId(bgKey.live(id), 'blue');
         if (!ids()[bgKey.weights(id)])
@@ -271,6 +338,7 @@ export async function runSubjectRelease(input) {
         now: input.now,
         deps: input.host.deps,
         ...(subject.reporting ? {} : { hardOnly: true }),
+        ...(input.deferCopies ? { deferCopies: true } : {}),
     });
     for (const [key, value] of Object.entries(result.writes)) {
         if (value === null)
@@ -285,5 +353,45 @@ export async function runSubjectRelease(input) {
     if (after && (formatWeights(after) !== before || balancerLags(ids(), id, formatWeights(after)))) {
         await input.host.writeWeights();
     }
-    return { blocked: result.blocked, refused: result.refused };
+    return {
+        blocked: result.blocked,
+        refused: result.refused,
+        deferredCopy: result.deferredCopy,
+        copy: result.copy,
+    };
+}
+/**
+ * Runs the stage copy a release run decided on and records what came of it. Call it after
+ * {@link runSubjectRelease} returned (that run's writes are persisted and its weights written), so a rollback
+ * decided in the same run never waits for the copy. The outcome goes into `context`, is persisted, and only then
+ * is the request consumed. `refused` is the operator text of a refusal. A host without ports refuses with `no-copy-port`; a lost lock throws and leaves the
+ * request open.
+ */
+export async function finishStageCopy(input) {
+    const { context, job } = input;
+    const outcome = input.host
+        ? await runStageCopy(input.host.deps, job)
+        : { outcome: 'refused', reason: 'no-copy-port' };
+    const settled = settleStageCopy({
+        ids: context.recordedIds(),
+        entryKey: input.subject.id,
+        now: input.now,
+        job,
+        outcome,
+    });
+    for (const [key, value] of Object.entries(settled.writes)) {
+        if (value === null)
+            context.forgetId(key);
+        else
+            context.recordScopeId(key, value);
+    }
+    // A request that never reached the copy (no port) still holds the lock it took.
+    const lockKey = releaseKeys(input.subject.id).stageCopyLock;
+    if (context.recordedIds()[lockKey]?.split('|')[1] === String(job.requestId)) {
+        context.forgetId(lockKey);
+    }
+    await context.persist();
+    if (settled.consume)
+        await input.store.consumeRequests([job.requestId]);
+    return { outcome, refused: settled.refused };
 }