@wtfalch/foundry 0.42.1 → 0.43.0

package/dist/plan/remote-database.js

@@ -0,0 +1,373 @@
+import { pinnedRef } from '../bootstrap/versions.js';
+import { PRIMARY_SERVER_SCOPE_KEY, splitServers } from './blue-green.js';
+import { isNotFoundError } from './executor.js';
+import { appNameFor } from './platform-diff.js';
+/**
+ * A blue/green app on a server other than the database's reaches its one database through a
+ * Cloudflare tunnel TCP route behind an Access service token (William, 2026-10-15). The tunnel has a
+ * `tcp://<org>-db:5432` rule on an internal hostname; Access lets only the service token in; a
+ * `cloudflared access tcp` sidecar on each other server listens on 5432 under the name
+ * {@link remoteDatabaseHost}.
+ */
+export const REMOTE_DATABASE_PORT = 5432;
+const DELIVERY_ATTEMPTS = 3;
+/** The sidecar's name on the Docker network of `serverUuid`, or `null` on the database's own server. */
+export function remoteDatabaseHost(entryKey, serverUuid, primaryUuid) {
+    return serverUuid === primaryUuid ? null : `${entryKey}-db-access`;
+}
+/** The `host:port` a color on a server other than the database's connects to. */
+export const remoteDatabaseAddress = (host) => `${host}:${REMOTE_DATABASE_PORT}`;
+/**
+ * The facts for {@link planRemoteDatabase}, from what the planner already read. The zone is the one the org's
+ * tunnel names live in (`org.domain`); `access` says the host can reach Access, tunnels, DNS and zones.
+ */
+export function remoteDatabaseFactsFrom(input) {
+    const { up } = splitServers(input.servers);
+    return {
+        serverUuids: up,
+        primaryUuid: input.recorded[PRIMARY_SERVER_SCOPE_KEY] ?? up[0] ?? null,
+        zone: input.org.domain ?? null,
+        access: input.access,
+    };
+}
+/** The scope ids one entry's step keeps. None holds a secret. */
+export const remoteDatabaseKeys = (entryKey) => ({
+    /** The service token's id, from its mint until its secret reached every sidecar. No `:`. */
+    undelivered: `bgdb.${entryKey}.undelivered`,
+    /** The service token's id once delivered. */
+    token: `bgdb.${entryKey}.token`,
+    access: `bgdb.${entryKey}.access`,
+    route: `bgdb.${entryKey}.route`,
+    applied: `bgdb.${entryKey}.applied`,
+    sidecar: (serverUuid) => `bgdb.${entryKey}.sidecar.${serverUuid}`,
+});
+/** The tunnel's HTTP rules as the org tunnel step last wrote them (JSON), so a later write keeps them. */
+export const TUNNEL_HTTP_INGRESS_SCOPE_KEY = 'cloudflare_tunnel_http_ingress';
+const ROUTE_KEY = /^bgdb\.[^.:]+\.route$/;
+/** Every TCP route recorded for the org, in key order. */
+export function recordedTcpRoutes(recorded) {
+    const routes = [];
+    for (const key of Object.keys(recorded).sort()) {
+        if (!ROUTE_KEY.test(key) || !recorded[key])
+            continue;
+        try {
+            const parsed = JSON.parse(recorded[key]);
+            if (typeof parsed.hostname === 'string' && typeof parsed.service === 'string') {
+                routes.push({ hostname: parsed.hostname, service: parsed.service });
+            }
+        }
+        catch {
+            // A record that is not JSON is ignored; the next apply of the step writes it again.
+        }
+    }
+    return routes;
+}
+/** The HTTP rules recorded by the tunnel step, catch-all included; `[]` if none or unreadable. */
+export function recordedHttpIngress(recorded) {
+    try {
+        const parsed = JSON.parse(recorded[TUNNEL_HTTP_INGRESS_SCOPE_KEY] ?? '[]');
+        return Array.isArray(parsed) ? parsed : [];
+    }
+    catch {
+        return [];
+    }
+}
+/**
+ * The tunnel's whole ingress: the TCP rules first (their names are exact), then every HTTP rule, the
+ * catch-all last. `putTunnelConfig` replaces the whole list, so every writer builds it through this.
+ */
+export function mergeTcpIngress(http, tcp) {
+    const names = new Set(tcp.map((r) => r.hostname));
+    const rules = http.filter((r) => !(r.hostname && names.has(r.hostname)));
+    const catchAll = rules.filter((r) => !r.hostname);
+    return [
+        ...tcp.map((r) => ({ hostname: r.hostname, service: r.service })),
+        ...rules.filter((r) => r.hostname),
+        ...(catchAll.length > 0 ? catchAll : [{ service: 'http_status:404' }]),
+    ];
+}
+/** The entry that gets a remote route: blue/green with the org's database. */
+function routedEntry(org) {
+    if (!org.hosting.enabled)
+        return undefined;
+    return org.hosting.entries.find((e) => e.database && e.blueGreen);
+}
+const appliedMarker = (d) => JSON.stringify({ hostname: d.hostname, database: d.database, servers: d.servers });
+/** Whether the route step has been applied for `serverUuid`: its marker names the server. */
+export function remoteDatabaseAppliedFor(recorded, entryKey, serverUuid) {
+    try {
+        const marker = JSON.parse(recorded[remoteDatabaseKeys(entryKey).applied] ?? 'null');
+        return Array.isArray(marker?.servers) && marker.servers.includes(serverUuid);
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Plans the remote database route for the org's blue/green database entry. One server, or no facts,
+ * plans nothing. A missing zone, port, tunnel or primary is a blocked line (rule 2), and so is a token
+ * that was minted and never delivered (rule 1): it is never minted again.
+ */
+export function planRemoteDatabase(org, facts, recorded) {
+    const entry = routedEntry(org);
+    if (!entry || !facts || facts.serverUuids.length < 2)
+        return { step: null, blocked: null };
+    const key = `cloudflare:db-route:${org.id}`;
+    const block = (reason) => ({
+        step: null,
+        blocked: {
+            description: `${org.slug}'s remote database route for ${entry.key} is blocked, not planned: ${reason}`,
+            key,
+        },
+    });
+    const keys = remoteDatabaseKeys(entry.key);
+    const lost = recorded[keys.undelivered];
+    if (lost) {
+        return block(`its Access service token ${lost} was minted and its secret never reached every sidecar; delete the token in Cloudflare and clear the recorded id before minting again`);
+    }
+    if (!facts.access)
+        return block('the host supplies no Access, tunnel, DNS or zone port');
+    if (!facts.zone)
+        return block("the platform's edge has no internal zone");
+    if (!facts.primaryUuid)
+        return block('no primary server is recorded');
+    if (!recorded[`cloudflare:tunnel:${org.id}`]) {
+        return block(`${org.slug}'s Cloudflare tunnel is not applied yet`);
+    }
+    const database = recorded[`coolify:database:${org.id}`];
+    if (!database)
+        return block(`${org.slug}'s database is not applied yet`);
+    const app = appNameFor(org, entry);
+    const desired = {
+        entry: entry.key,
+        app,
+        hostname: `${app}-db.${facts.zone}`,
+        database,
+        primary: facts.primaryUuid,
+        servers: facts.serverUuids.filter((s) => s !== facts.primaryUuid).sort(),
+        zone: facts.zone,
+    };
+    // A server added after the token was minted gets its sidecar's secret copied from another sidecar;
+    // with no sidecar to copy from there is nothing to give it.
+    if (recorded[keys.token] && desired.servers.every((server) => !recorded[keys.sidecar(server)])) {
+        return block(`service token ${recorded[keys.token]} cannot be shown again and no sidecar holds it to copy; delete the token in Cloudflare and clear the recorded ids to mint it again`);
+    }
+    if (recorded[keys.applied] === appliedMarker(desired))
+        return { step: null, blocked: null };
+    return {
+        blocked: null,
+        step: {
+            action: {
+                system: 'cloudflare',
+                kind: 'db-route',
+                key: org.id,
+                desired: { ...desired },
+            },
+            step: {
+                description: `reach ${org.slug}'s database from ${desired.servers.length} other server(s) through a tunnel route behind an Access service token`,
+                destroys: false,
+                migrates: false,
+            },
+        },
+    };
+}
+/** The Dockerfile of the sidecar: the pinned `cloudflared`, listening on 5432 for the Access-protected route. */
+export function remoteDatabaseDockerfile(hostname) {
+    return [
+        `FROM cloudflare/cloudflared:${pinnedRef('cloudflared')}`,
+        `CMD ["access", "tcp", "--hostname", "${hostname}", "--url", "0.0.0.0:${REMOTE_DATABASE_PORT}"]`,
+        // The image is distroless (no shell, no nc): the exec-form check proves only that the binary runs.
+        'HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 CMD ["cloudflared", "version"]',
+        '',
+    ].join('\n');
+}
+const TOKEN_ID_ENV = 'TUNNEL_SERVICE_TOKEN_ID';
+const TOKEN_SECRET_ENV = 'TUNNEL_SERVICE_TOKEN_SECRET';
+/**
+ * Applies one entry's route. The Access app and its service-token policy come first, so the tunnel route
+ * and the DNS record never exist without the gate in front of them. The service token is minted once
+ * (rule 1): its id is recorded and persisted before delivery, the secret goes only to the sidecars' env
+ * (rule 6), the delivered id replaces it, and the applied marker is the last write. A failure halfway
+ * leaves a repair for the next run: the Access app, sidecars, route and DNS are found again; a minted
+ * token is never minted again. A sidecar made after the mint takes its secret from an existing sidecar's
+ * env. Every outside call is bounded by the ports' own timeouts (edge SDK and Coolify client).
+ */
+export async function applyRemoteDatabase(deps, run) {
+    const { desired, org } = run;
+    const keys = remoteDatabaseKeys(desired.entry);
+    const tunnelId = run.recorded()[`cloudflare:tunnel:${org.id}`];
+    if (!tunnelId)
+        throw new Error(`remote database for ${org.slug}: the Cloudflare tunnel is not applied`);
+    const zone = await deps.zones.findZoneByApex(desired.zone);
+    if (!zone)
+        throw new Error(`remote database for ${org.slug}: zone ${desired.zone} is not live`);
+    let accessAppId = run.recorded()[keys.access];
+    if (!accessAppId) {
+        const live = (await deps.access.listAccessApps()).find((a) => a.domain === desired.hostname);
+        accessAppId = (live ??
+            (await deps.access.createAccessApp({
+                name: `${desired.app}-db-access`,
+                domain: desired.hostname,
+                type: 'self_hosted',
+            }))).id;
+        run.record(keys.access, accessAppId);
+        await run.persist();
+    }
+    // Sidecars exist before the secret does, so the secret has somewhere to go the moment it is minted.
+    const { environment } = await deps.coolify.findOrCreateEnvironment(run.projectUuid, 'production');
+    const sidecars = [];
+    for (const server of desired.servers) {
+        const host = remoteDatabaseHost(desired.app, server, desired.primary);
+        if (!host)
+            continue;
+        const recordedId = run.recorded()[keys.sidecar(server)];
+        let appId = null;
+        if (recordedId) {
+            try {
+                await deps.coolify.getApplication(recordedId);
+                appId = recordedId;
+            }
+            catch (error) {
+                if (!isNotFoundError(error))
+                    throw error;
+            }
+        }
+        let fresh = false;
+        if (!appId) {
+            const created = await deps.coolify.createDockerfileApplication({
+                project_uuid: run.projectUuid,
+                server_uuid: server,
+                environment_name: environment.name,
+                environment_uuid: environment.uuid,
+                dockerfile: remoteDatabaseDockerfile(desired.hostname),
+                name: host,
+                description: `cloudflared access sidecar: ${org.slug}'s database over the tunnel`,
+                instant_deploy: false,
+                autogenerate_domain: false,
+            });
+            appId = created.uuid;
+            fresh = true;
+            // The internal name is what `<app>-db-access` resolves to on the server's Docker network.
+            await deps.coolify.updateApplication(appId, {
+                custom_internal_name: host,
+                health_check_enabled: false,
+            });
+            run.record(keys.sidecar(server), appId);
+            await run.persist();
+        }
+        sidecars.push({ server, appId, fresh });
+    }
+    // The token: found by recorded id first. A token of the name that foundry has no id for is not
+    // minted over; the step fails with fixed text.
+    const lost = run.recorded()[keys.undelivered];
+    if (lost) {
+        throw new Error(`remote database for ${org.slug}: service token ${lost} was minted and not delivered; delete it in Cloudflare and clear the recorded id before minting again`);
+    }
+    let tokenId = run.recorded()[keys.token];
+    let secrets = null;
+    let deliverTo = sidecars;
+    if (!tokenId) {
+        const name = `${desired.app}-db-access`;
+        if (await deps.access.findServiceTokenByName(name)) {
+            throw new Error(`remote database for ${org.slug}: a service token named ${name} exists and foundry holds no id for it; delete it in Cloudflare before applying again`);
+        }
+        const minted = await deps.access.createServiceToken(name);
+        deps.secret(minted.clientSecret);
+        deps.secret(minted.clientId);
+        run.record(keys.undelivered, minted.id);
+        await run.persist();
+        tokenId = minted.id;
+        secrets = { clientId: minted.clientId, clientSecret: minted.clientSecret };
+    }
+    else {
+        // A sidecar made after the token was minted takes the secret from a sidecar that already holds it.
+        deliverTo = sidecars.filter((sidecar) => sidecar.fresh);
+        if (deliverTo.length > 0) {
+            secrets = await secretsOf(deps, sidecars.filter((sidecar) => !sidecar.fresh));
+            if (!secrets) {
+                throw new Error(`remote database for ${org.slug}: the sidecar on server ${deliverTo[0]?.server} is new and no existing sidecar holds service token ${tokenId} to copy; delete the token in Cloudflare and clear the recorded ids to mint it again`);
+            }
+        }
+    }
+    if (secrets) {
+        const failed = [];
+        for (const sidecar of deliverTo) {
+            if (!(await deliver(deps, sidecar.appId, secrets)))
+                failed.push(sidecar.server);
+        }
+        if (failed.length > 0) {
+            // Ids only: the cause of a failed write can echo the request body.
+            throw new Error(`remote database for ${org.slug}: service token ${tokenId} could not be delivered to the sidecar on server ${failed.join(', ')} after ${DELIVERY_ATTEMPTS} attempts; delete the token in Cloudflare and clear the recorded id before minting again`);
+        }
+        if (run.recorded()[keys.undelivered]) {
+            run.record(keys.token, tokenId);
+            run.record(keys.undelivered, '');
+            await run.persist();
+        }
+    }
+    const policies = await deps.access.listAccessPolicies(accessAppId);
+    const hasPolicy = policies.some((p) => p.decision === 'non_identity' &&
+        (p.include ?? []).some((r) => r.service_token?.token_id === tokenId));
+    if (!hasPolicy) {
+        await deps.access.createAccessPolicy(accessAppId, {
+            name: 'database sidecars',
+            decision: 'non_identity',
+            include: [{ service_token: { token_id: tokenId } }],
+        });
+    }
+    // The gate exists: now the route and the name that reach the database.
+    run.record(keys.route, JSON.stringify({
+        hostname: desired.hostname,
+        service: `tcp://${run.databaseHost}:${REMOTE_DATABASE_PORT}`,
+    }));
+    await deps.tunnels.putTunnelConfig(tunnelId, {
+        ingress: mergeTcpIngress(recordedHttpIngress(run.recorded()), recordedTcpRoutes(run.recorded())),
+    });
+    if ((await deps.dns.findDnsRecord(zone.name, desired.hostname, 'CNAME')).length === 0) {
+        await deps.dns.createDnsRecord(zone.name, {
+            type: 'CNAME',
+            name: desired.hostname,
+            content: `${tunnelId}.cfargotunnel.com`,
+            proxied: true,
+        });
+    }
+    for (const sidecar of sidecars) {
+        const live = await deps.coolify.getApplication(sidecar.appId);
+        const up = Boolean(live.status?.startsWith('running') || live.status?.startsWith('starting'));
+        if (secrets || sidecar.fresh || !up)
+            await deps.coolify.deploy(sidecar.appId);
+    }
+    run.record(keys.applied, appliedMarker(desired));
+    await run.persist();
+}
+/** The service token an existing sidecar already holds in its env, registered for redaction; `null` when none does. */
+async function secretsOf(deps, sidecars) {
+    for (const sidecar of sidecars) {
+        const envs = await deps.coolify.listEnvs(sidecar.appId);
+        const envValue = (key) => envs.find((e) => e.key === key && !e.is_preview)?.value;
+        const clientId = envValue(TOKEN_ID_ENV);
+        const clientSecret = envValue(TOKEN_SECRET_ENV);
+        if (clientId && clientSecret) {
+            deps.secret(clientId);
+            deps.secret(clientSecret);
+            return { clientId, clientSecret };
+        }
+    }
+    return null;
+}
+/** Writes the token to one sidecar's env, up to {@link DELIVERY_ATTEMPTS} times; the error is dropped. */
+async function deliver(deps, appId, secrets) {
+    for (let attempt = 1; attempt <= DELIVERY_ATTEMPTS; attempt++) {
+        try {
+            await deps.coolify.bulkUpsertEnvs(appId, [
+                { key: TOKEN_ID_ENV, value: secrets.clientId, literal: true },
+                { key: TOKEN_SECRET_ENV, value: secrets.clientSecret, literal: true },
+            ]);
+            return true;
+        }
+        catch {
+            // Retried; the error is dropped in case it echoes the secret.
+        }
+    }
+    return false;
+}

package/dist/platform.d.ts

@@ -12,6 +12,16 @@ export interface PlatformService {
     hostname: string;
     gitRef: string;
 }
+/**
+ * `archon`'s own extra field: presence turns blue/green on for platform-archon, through the same code as a hosting
+ * entry (`plan/blue-green.ts`). The tunnel moves to the balancer only once a balancer is recorded.
+ */
+export interface PlatformArchonService extends PlatformService {
+    blueGreen?: {
+        steps?: number[];
+        stepHoldMinutes?: number;
+    };
+}
 /**
  * `sfu`'s own extra field. A host that changes what the SFU VM is reached
  * with (its SSH key, host key or tunnel token) sets a new `connectionRevision`
@@ -97,7 +107,7 @@ export interface DeclaredPlatform {
     ai: PlatformService;
     jobs: PlatformService;
     integrations: PlatformService;
-    archon: PlatformService;
+    archon: PlatformArchonService;
     /** #317 (F7). No deploy step reads this service's own `gitRef` yet -- `N1` is deployed by hand until it exists. */
     edge: PlatformService;
     /**

package/dist/platform.js

@@ -1,5 +1,5 @@
 import { isIPv4 } from 'node:net';
-import { validateDomain } from './org.js';
+import { validateBlueGreen, validateDomain } from './org.js';
 /** Where Archon publishes the Worker script for `tag` (a private repo; the host's `fetchArtifact` fetches it). */
 export function uptimeArtifactUrl(tag) {
     return `https://github.com/wtfalch/archon/releases/download/${tag}/archon-uptime-worker.js`;
@@ -69,6 +69,9 @@ export function validatePlatform(platform) {
             return `${key}.gitRef must not be empty once ${key}.enabled is true`;
         }
     }
+    const blueGreenReason = validateBlueGreen(platform.archon.blueGreen, 'archon');
+    if (blueGreenReason)
+        return blueGreenReason;
     const revision = platform.sfu.connectionRevision;
     if (revision != null && !/^[A-Za-z0-9:.-]{0,64}$/.test(revision)) {
         return 'sfu.connectionRevision must be at most 64 characters of A-Z, a-z, 0-9, ":", "." and "-"';

package/dist/release/decide.d.ts

@@ -0,0 +1,95 @@
+import { type Color, type ReleaseAction, type ReleaseSignal, type WindowState } from './types.js';
+/** ADR 0001 rule 5: the default steps and the least time on each. */
+export declare const DEFAULT_STEPS: readonly number[];
+export declare const DEFAULT_STEP_HOLD_MINUTES = 60;
+/** Rule 7: errors this share of requests, with at least HARD_FAIL_MIN_REQUESTS, roll back at any level. */
+export declare const HARD_FAIL_ERROR_RATE = 0.9;
+export declare const HARD_FAIL_MIN_REQUESTS = 2;
+/** Rule 8: a relative fail needs this many requests and this many failed checks in a row. */
+export declare const RELATIVE_MIN_REQUESTS = 20;
+export declare const RELATIVE_FAILED_CHECKS = 3;
+/** A relative check counts only this long after the last counted one, so runs close together (applies also run releases) are one check. */
+export declare const CHECK_INTERVAL_MINUTES = 10;
+/** The new color matches the baseline while its error rate is at most baseline plus this (absolute). */
+export declare const ERROR_RATE_MARGIN = 0.02;
+/** Rules 12 and 13: the window at 100% closes after this long, and at most this long. */
+export declare const WINDOW_MIN_HOURS = 24;
+export declare const WINDOW_CAP_DAYS = 7;
+/** The new color must be running and answer on its own address within this time of the deploy. */
+export declare const DEPLOY_TIMEOUT_MINUTES = 30;
+export interface ActiveRelease {
+    from: Color;
+    to: Color;
+    digest: string;
+    /** Relative-fail checks in a row. */
+    fails: number;
+    /** When the last counted relative check ran (ISO). Absent: none yet in this step. */
+    checkedAt?: string | undefined;
+}
+export interface Recorded {
+    live: Color;
+    digests: Record<Color, string | null>;
+    stageDigest: string | null;
+    weights: Record<Color, number>;
+    window: WindowState;
+    /** Set while window.kind is step, inconclusive, window or cap. */
+    release: ActiveRelease | null;
+    held: string[];
+}
+export interface ColorCounts {
+    requests: number;
+    errors: number;
+}
+/** The reporting health read (C1), once validated. */
+export interface HealthRead {
+    colors: Record<Color, ColorCounts>;
+    daily: {
+        days: number;
+        requestsPerDay: number;
+        errorsPerDay: number;
+    };
+}
+export interface Observation {
+    /** null when the health read failed: counted as inconclusive, never as a pass. */
+    health: HealthRead | null;
+    /** The new color's Coolify status. */
+    status: 'running' | 'crash_loop' | 'other';
+    /** The new color's own address, checked over HTTP. */
+    address: 'ok' | 'failed' | 'unchecked';
+}
+export type ReleaseEffect = {
+    kind: 'deploy';
+    color: Color;
+    digest: string;
+} | {
+    kind: 'stop';
+    color: Color;
+};
+export interface Decision {
+    recorded: Recorded;
+    signal: ReleaseSignal | null;
+    effects: ReleaseEffect[];
+    /** Blocked items for the operator, fixed text. */
+    blocked: string[];
+    /** Actions that changed nothing, with a fixed reason. */
+    refused: string[];
+}
+export declare function normalizeSteps(steps: readonly number[] | undefined): number[];
+/** The time a health read should cover: from the step or window start to now. */
+export declare function healthRange(r: Recorded, now: Date): {
+    from: string;
+    to: string;
+} | null;
+/**
+ * The whole release decision for one run: from the recorded state, the
+ * actions requested, one observation and the time, the next recorded state.
+ * Pure: it reads nothing and writes nothing (ADR 0001 rules 5, 7-13, 16).
+ */
+export declare function decide(input: {
+    recorded: Recorded;
+    actions: readonly ReleaseAction[];
+    steps?: readonly number[] | undefined;
+    stepHoldMinutes?: number | undefined;
+    now: Date;
+    observation: Observation;
+}): Decision;