@wtfalch/foundry 0.42.1 → 0.44.0

package/dist/plan/worker-app.js

@@ -0,0 +1,792 @@
+import { createHash, randomBytes } from 'node:crypto';
+import { WORKER_APP_HYPERDRIVE_BINDING, WORKER_APP_KEYS_SECRET, WORKER_APP_KEYS_URL_VAR, workerAppHostId, } from '../worker-app-config.js';
+import { entryAuthEnv } from './broker-entries.js';
+import { workerAppHostname } from './hostnames.js';
+import { appStepKey } from './platform-diff.js';
+import { remoteDatabaseKeys } from './remote-database.js';
+import { applyWorkerAppDatabase, deleteWorkerAppDatabaseRoute, deleteWorkerAppHyperdrive, workerAppDbRouteMarkerScopeKey, workerAppHyperdriveScopeKey, workerAppMigrationsScopeKey, } from './worker-app-database.js';
+import { WORKER_COMPATIBILITY_DATE } from './worker.js';
+/**
+ * ADR 0002: a hosting entry that declares `workerApp` runs as one Cloudflare
+ * Worker with Durable Objects and Containers on the platform account. One step
+ * per entry, `cloudflare:worker-app`, keyed `<orgId>:<entryKey>`; its resource
+ * id is the script name. The module is fetched, checked and uploaded behind
+ * {@link WorkerAppPort}; foundry builds and checks out nothing.
+ *
+ * Order on apply, so a failure at any point leaves a state the next apply
+ * repairs and the Worker is never routed before it can serve:
+ * trace -> upload (bindings, vars, only the unapplied migrations) -> secrets ->
+ * containers -> DNS record (own domain only) -> route -> applied marker.
+ * The marker is written last, so a run that fails anywhere is planned again.
+ */
+// ---- scope keys. None starts with `edge_worker_`, which `liveWorkers` scans. ----
+export const workerAppAppliedScopeKey = (entryKey) => `workerapp_applied_${entryKey}`;
+/** The script name, recorded and persisted before the upload, so a crash mid-deploy leaves a trail to it. */
+export const workerAppUploadedScopeKey = (entryKey) => `workerapp_uploaded_${entryKey}`;
+/** The migration tags applied, as a JSON list, recorded after the upload that carried them. */
+export const workerAppTagsScopeKey = (entryKey) => `workerapp_tags_${entryKey}`;
+/** Secret name -> delivered version, as JSON. Versions only, never a value. */
+export const workerAppSecretsScopeKey = (entryKey) => `workerapp_secrets_${entryKey}`;
+/** The delivered custody credential, as `{hostId, credentialId, expiresAt}` JSON. Never the credential. */
+export const workerAppKeysScopeKey = (entryKey) => `workerapp_keys_${entryKey}`;
+/** Class name -> `{id, image, instanceType, maxInstances}` of its container application, as JSON. */
+export const workerAppContainersScopeKey = (entryKey) => `workerapp_containers_${entryKey}`;
+/** The route: `{zoneId, apex, pattern, id}` as JSON. */
+export const workerAppRouteScopeKey = (entryKey) => `workerapp_route_${entryKey}`;
+/** A route about to be created, same shape without `id`; recorded and persisted before `createRoute`. */
+export const workerAppRoutePendingScopeKey = (entryKey) => `workerapp_route_pending_${entryKey}`;
+/** The proxied record in the org's zone: `{zone, id, name}` as JSON. */
+export const workerAppDnsScopeKey = (entryKey) => `workerapp_dns_${entryKey}`;
+/** The script name retired, once a retire step is done. */
+export const workerAppRetiredScopeKey = (entryKey) => `workerapp_retired_${entryKey}`;
+/** Every per-entry scope key that marks something deployed, half deployed or handled. */
+export const workerAppTraceScopeKeys = (entryKey) => [
+    workerAppAppliedScopeKey(entryKey),
+    workerAppUploadedScopeKey(entryKey),
+    workerAppTagsScopeKey(entryKey),
+    workerAppSecretsScopeKey(entryKey),
+    workerAppKeysScopeKey(entryKey),
+    workerAppContainersScopeKey(entryKey),
+    workerAppRouteScopeKey(entryKey),
+    workerAppRoutePendingScopeKey(entryKey),
+    workerAppDnsScopeKey(entryKey),
+    workerAppRetiredScopeKey(entryKey),
+    workerAppHyperdriveScopeKey(entryKey),
+    workerAppMigrationsScopeKey(entryKey),
+    workerAppDbRouteMarkerScopeKey(entryKey),
+    ...['route', 'access', 'token', 'undelivered'].map((name) => remoteDatabaseKeys(entryKey)[name]),
+];
+/** The recorded id of the step: the script name. */
+export const workerAppResourceKey = (stepKey) => `cloudflare:worker-app:${stepKey}`;
+export const WORKER_APP_ORG_HASH_LENGTH = 8;
+/** `${8 hex of the org id's sha256}-${entry key}-app`. */
+export const workerAppName = (org, entry) => `${createHash('sha256').update(org.id).digest('hex').slice(0, WORKER_APP_ORG_HASH_LENGTH)}-${entry.key}-app`;
+/** The zone that holds the routes of an org with no domain of its own. */
+export const WORKER_APP_OIKOS_ZONE = 'oikos.host';
+/** The record a Worker route needs on the org's own zone: Cloudflare's documented dummy, proxied. */
+export const WORKER_APP_DNS_RECORD = {
+    type: 'AAAA',
+    content: '100::',
+};
+/** True once the Worker app holds the broker key (`authEnv` delivered it). */
+export const brokerKeyDelivered = (recorded, entryKey) => Boolean(parseJson(recorded[workerAppSecretsScopeKey(entryKey)], {})
+    .AUTH_APP_KEY);
+/**
+ * The broker key of a Worker app with `authEnv`, read from the held-key store
+ * (it was put there before it was delivered to the Worker). Throws, naming no
+ * value, when it is missing or shorter than the broker accepts.
+ */
+export async function heldBrokerKey(heldKeys, org, entryKey) {
+    const key = await heldKeys?.resolve(workerAppSecretRef(org.id, entryKey, WORKER_APP_AUTH_KEY_SECRET));
+    if (!key || key.length < 32) {
+        throw new Error(`coolify:broker-entry for ${org.slug}: the Worker app ${entryKey} has no held AUTH_APP_KEY yet`);
+    }
+    return key;
+}
+/** The foundry-written secret the broker entry shares with the app when `authEnv` is on. */
+export const WORKER_APP_AUTH_KEY_SECRET = 'AUTH_APP_KEY';
+/**
+ * Where foundry keeps a generated secret (the held-key store), so the value
+ * is minted once and delivered again unchanged after a crash. The entry key
+ * and name are hashed: a held ref is at most 128 characters.
+ */
+export const workerAppSecretRef = (orgId, entryKey, name) => `foundry:worker-app:${orgId}:${createHash('sha256').update(`${entryKey}:${name}`).digest('hex').slice(0, 32)}`;
+const secretVersion = (value) => `g1-${createHash('sha256').update(value).digest('hex').slice(0, 16)}`;
+/** Foundry mints a new credential this long before the held one expires. */
+export const WORKER_APP_CREDENTIAL_RENEW_BEFORE_MS = 30 * 24 * 60 * 60 * 1000;
+/**
+ * True when the delivered custody credential is within 30 days of expiry (or
+ * its record is unreadable), so the plan renews it.
+ */
+export function custodyRenewalDue(recorded, entryKey, now) {
+    const current = parseJson(recorded[workerAppKeysScopeKey(entryKey)], null);
+    if (!current)
+        return false; // Never delivered: the first apply mints it.
+    const expires = Date.parse(current.expiresAt ?? '');
+    return Number.isNaN(expires) || expires - WORKER_APP_CREDENTIAL_RENEW_BEFORE_MS <= now.getTime();
+}
+const sortedEntries = (record) => Object.entries(record ?? {}).sort(([a], [b]) => a.localeCompare(b));
+/** The zone apex the entry's route lives in. */
+export const workerAppApex = (org) => org.hosting.usesDomain && org.domain ? org.domain : WORKER_APP_OIKOS_ZONE;
+/** True when the want has a secret that must be held (a generated one, the broker key, a custody credential). */
+const holdsGenerated = (want) => want.secrets.length > 0;
+/**
+ * What the Worker must look like, or the inputs still missing (names only).
+ * `issuer` is the sign-in issuer the plan read; `keysUrl` the custody Worker.
+ */
+export function wantedWorkerApp(org, entry, recorded, input) {
+    const app = entry.workerApp;
+    const missing = [];
+    const vars = { ...app.vars };
+    const secrets = [];
+    for (const [name, source] of Object.entries(app.secrets ?? {})) {
+        if (source.from === 'keysCustody') {
+            const hostId = workerAppHostId(org.id, entry.key);
+            if (input.keysUrl === undefined)
+                missing.push('the keys custody port');
+            else
+                vars[WORKER_APP_KEYS_URL_VAR] = input.keysUrl;
+            if (hostId === null)
+                missing.push('a valid custody host id');
+            else
+                secrets.push({ name, source: 'keysCustody', hostId });
+        }
+        else {
+            secrets.push({ name, source: 'generated' });
+        }
+    }
+    if (app.authEnv) {
+        const env = entryAuthEnv(org, entry, recorded, input.issuer);
+        for (const [name, what] of [
+            ['AUTH_ORGANIZATION_ID', 'its sign-in organisation id'],
+            ['AUTH_CLIENT_ID', 'its sign-in client id'],
+            ['AUTH_ISSUER', 'the sign-in issuer'],
+        ]) {
+            if (env[name])
+                vars[name] = env[name];
+            else
+                missing.push(what);
+        }
+        secrets.push({ name: WORKER_APP_AUTH_KEY_SECRET, source: 'generated' });
+        if (entry.uses?.auth?.mcp) {
+            for (const name of ['MCP_AUDIENCE', 'MCP_RESOURCE', 'MCP_ISSUER']) {
+                if (env[name])
+                    vars[name] = env[name];
+                else if (!missing.includes('its MCP audience project id')) {
+                    missing.push('its MCP audience project id');
+                }
+            }
+        }
+    }
+    if (missing.length > 0)
+        return { missing };
+    return {
+        want: {
+            name: workerAppName(org, entry),
+            artifactUrl: app.artifactUrl,
+            artifactSha256: app.artifactSha256,
+            compatibilityDate: app.compatibilityDate ?? WORKER_COMPATIBILITY_DATE,
+            compatibilityFlags: [...(app.compatibilityFlags ?? [])].sort(),
+            host: workerAppHostname(org, entry),
+            apex: workerAppApex(org),
+            classes: Object.entries(app.durableObjects.classes)
+                .map(([binding, className]) => ({ binding, className }))
+                .sort((a, b) => a.binding.localeCompare(b.binding)),
+            migrations: app.durableObjects.migrations,
+            containers: (app.containers ?? [])
+                .map((c) => ({
+                className: c.className,
+                image: `${c.image.repository}@${c.image.digest}`,
+                instanceType: c.instanceType,
+                maxInstances: c.maxInstances,
+            }))
+                .sort((a, b) => a.className.localeCompare(b.className)),
+            vars: sortedEntries(vars),
+            ...(app.database
+                ? {
+                    database: {
+                        appendOnly: [...(app.database.appendOnly ?? [])].sort(),
+                        ...(app.database.migrationsSha256
+                            ? { migrationsSha256: app.database.migrationsSha256 }
+                            : {}),
+                    },
+                }
+                : {}),
+            secrets: secrets.sort((a, b) => a.name.localeCompare(b.name)),
+        },
+    };
+}
+const parseJson = (raw, fallback) => {
+    if (!raw)
+        return fallback;
+    try {
+        return JSON.parse(raw);
+    }
+    catch {
+        return fallback;
+    }
+};
+/**
+ * The migration tags the recorded list says are applied must be the first tags
+ * of the declared list, in order. `null` when they are; else the refusal.
+ */
+export function migrationPrefixError(applied, declared) {
+    for (const [index, tag] of applied.entries()) {
+        if (declared[index] === tag)
+            continue;
+        return declared.includes(tag)
+            ? `migration '${tag}' was applied and has been reordered; migrations are append-only`
+            : `migration '${tag}' was applied and has been removed; migrations are append-only`;
+    }
+    return null;
+}
+/** The unapplied migrations after the last tag in `applied`. */
+export const unappliedMigrations = (declared, applied) => declared.slice(applied.length);
+/** The registry prefix a container image of this platform must start with. */
+export const workerAppRegistryPrefix = (accountId) => `registry.cloudflare.com/${accountId}/`;
+/** Every entry key with a deployed or half-deployed Worker app recorded. */
+export function liveWorkerApps(orgId, recordedIds) {
+    const mainPrefix = `cloudflare:worker-app:${orgId}:`;
+    const entryKeys = new Set();
+    for (const [name, id] of Object.entries(recordedIds)) {
+        if (!id)
+            continue;
+        if (name.startsWith(mainPrefix))
+            entryKeys.add(name.slice(mainPrefix.length));
+        for (const prefix of [
+            'workerapp_uploaded_',
+            'workerapp_route_',
+            'workerapp_route_pending_',
+            'workerapp_containers_',
+            'workerapp_keys_',
+            'workerapp_hyperdrive_',
+            'workerapp_dbroute_',
+        ]) {
+            if (name.startsWith(prefix))
+                entryKeys.add(name.slice(prefix.length));
+        }
+    }
+    const live = new Map();
+    for (const entryKey of entryKeys) {
+        const mainId = recordedIds[`${mainPrefix}${entryKey}`] ?? '';
+        const retired = mainId !== '' && recordedIds[workerAppRetiredScopeKey(entryKey)] === mainId;
+        const partial = recordedIds[workerAppUploadedScopeKey(entryKey)] ||
+            recordedIds[workerAppRouteScopeKey(entryKey)] ||
+            recordedIds[workerAppRoutePendingScopeKey(entryKey)] ||
+            recordedIds[workerAppContainersScopeKey(entryKey)] ||
+            recordedIds[workerAppKeysScopeKey(entryKey)] ||
+            recordedIds[workerAppHyperdriveScopeKey(entryKey)] ||
+            recordedIds[workerAppDbRouteMarkerScopeKey(entryKey)] ||
+            recordedIds[workerAppDnsScopeKey(entryKey)];
+        if (!partial && (!mainId || retired))
+            continue;
+        live.set(entryKey, mainId || recordedIds[workerAppUploadedScopeKey(entryKey)] || '');
+    }
+    return live;
+}
+/**
+ * One step per Worker app whose declaration or inputs changed, plus a retire
+ * step per recorded Worker app no longer declared. Blocked, not planned
+ * (`CLAUDE.md` platform rule 2), while the host lacks a port or a value, while
+ * the declared migrations contradict the applied ones, and for each input the
+ * Worker needs that does not exist yet. A migration that deletes or renames a
+ * class makes the step `destroys` or `migrates`, so the org's name must be typed
+ * to apply it.
+ */
+export function planWorkerApps(org, recordedIds, input) {
+    const steps = [];
+    const blocked = [];
+    const declared = new Set(org.hosting.enabled ? org.hosting.entries.filter((e) => e.workerApp).map((e) => e.key) : []);
+    for (const entryKey of liveWorkerApps(org.id, recordedIds).keys()) {
+        if (declared.has(entryKey))
+            continue;
+        const key = workerAppResourceKey(`${org.id}:${entryKey}`);
+        if (!input.available) {
+            blocked.push({
+                description: `${org.slug}'s ${entryKey} Worker app is blocked, not planned: the host supplies no WorkerAppPort`,
+                key,
+            });
+            continue;
+        }
+        steps.push({
+            action: {
+                system: 'cloudflare',
+                kind: 'worker-app',
+                key: `${org.id}:${entryKey}`,
+                desired: { mode: 'retire' },
+            },
+            step: {
+                description: `delete ${org.slug}'s ${entryKey} Worker app, its route and its containers: workerApp is no longer declared`,
+                destroys: true,
+                migrates: false,
+            },
+        });
+    }
+    if (!org.hosting.enabled)
+        return { steps, blocked };
+    for (const entry of org.hosting.entries) {
+        const app = entry.workerApp;
+        if (!app)
+            continue;
+        const stepKey = appStepKey(org, entry.key);
+        const key = workerAppResourceKey(stepKey);
+        const at = `${org.slug}'s ${entry.key} Worker app`;
+        const block = (why) => blocked.push({
+            description: `${at} is blocked, not planned: ${why}`,
+            key,
+        });
+        if (!input.available) {
+            block('the host supplies no WorkerAppPort');
+            continue;
+        }
+        const wanted = wantedWorkerApp(org, entry, recordedIds, input);
+        if ('missing' in wanted) {
+            block(`${wanted.missing.join(', ')} not recorded yet`);
+            continue;
+        }
+        const { want } = wanted;
+        const stray = (app.containers ?? []).find((c) => input.registryAccountId === undefined ||
+            !c.image.repository.startsWith(workerAppRegistryPrefix(input.registryAccountId)) ||
+            c.image.repository.split('/').some((part) => /^\.+$/.test(part)));
+        if (stray) {
+            block(input.registryAccountId === undefined
+                ? 'the host supplies no Cloudflare account id for the container registry'
+                : `container '${stray.className}' image repository must start with ${workerAppRegistryPrefix(input.registryAccountId)} and hold no '..' segment`);
+            continue;
+        }
+        if (want.database) {
+            const db = input.database;
+            const why = !db
+                ? 'the host supplies no WorkerAppDatabasePort'
+                : !db.accessReady
+                    ? 'the host supplies no Access, tunnel, DNS or zone port'
+                    : want.apex === WORKER_APP_OIKOS_ZONE
+                        ? "its database route needs the org's own domain: the shared oikos.host zone has no per-org tunnel"
+                        : !db.databaseReady
+                            ? `${org.slug}'s database is not applied yet`
+                            : !db.tunnelReady
+                                ? `${org.slug}'s Cloudflare tunnel is not applied yet`
+                                : recordedIds[remoteDatabaseKeys(entry.key).undelivered]
+                                    ? `its Access service token ${recordedIds[remoteDatabaseKeys(entry.key).undelivered]} was minted and its secret never held; delete the token in Cloudflare and clear the recorded id before minting again`
+                                    : !input.canHoldSecrets
+                                        ? 'its database credentials need a held-key store that can put, and the host has none'
+                                        : null;
+            if (why) {
+                block(why);
+                continue;
+            }
+        }
+        if (holdsGenerated(want) && !input.canHoldSecrets) {
+            block('its generated secrets need a held-key store that can put, and the host has none');
+            continue;
+        }
+        const appliedTags = parseJson(recordedIds[workerAppTagsScopeKey(entry.key)], []);
+        const declaredTags = want.migrations.map((m) => m.tag);
+        const prefixError = migrationPrefixError(appliedTags, declaredTags);
+        if (prefixError) {
+            block(prefixError);
+            continue;
+        }
+        const renew = want.secrets.some((s) => s.source === 'keysCustody') &&
+            custodyRenewalDue(recordedIds, entry.key, input.now ?? new Date());
+        if (!renew && recordedIds[workerAppAppliedScopeKey(entry.key)] === JSON.stringify(want))
+            continue;
+        const pending = unappliedMigrations(want.migrations, appliedTags);
+        const deletes = pending.flatMap((m) => m.deletedClasses ?? []);
+        const renames = pending.flatMap((m) => m.renamedClasses ?? []);
+        const risk = deletes.length > 0 || renames.length > 0
+            ? ` (migration ${deletes.length > 0 ? `deletes class ${deletes.join(', ')}` : ''}${deletes.length > 0 && renames.length > 0 ? ' and ' : ''}${renames.length > 0 ? `renames ${renames.map((r) => `${r.from} to ${r.to}`).join(', ')}` : ''}: type the org's name to apply it)`
+            : '';
+        const first = !recordedIds[workerAppAppliedScopeKey(entry.key)];
+        steps.push({
+            action: {
+                system: 'cloudflare',
+                kind: 'worker-app',
+                key: stepKey,
+                desired: { mode: 'apply', ...want },
+            },
+            step: {
+                description: `${first ? 'deploy' : 'update'} ${at} at ${want.host}${risk}${renew ? ' (renews its keys custody credential, which expires within 30 days)' : ''}`,
+                destroys: deletes.length > 0,
+                migrates: renames.length > 0,
+            },
+        });
+    }
+    return { steps, blocked };
+}
+/** An outside call that has not answered in this long fails the step; the next apply repairs it. */
+export const WORKER_APP_CALL_TIMEOUT_MS = 120_000;
+export const WORKER_APP_KEYS_DELIVERY_ATTEMPTS = 3;
+/**
+ * Races `task` against a timer. An error names the call, never the cause.
+ */
+export function withCallTimeout(ms, label, task) {
+    return new Promise((resolve, reject) => {
+        const timer = setTimeout(() => reject(new Error(`${label}: no answer within ${ms} ms`)), ms);
+        task().then((value) => {
+            clearTimeout(timer);
+            resolve(value);
+        }, (error) => {
+            clearTimeout(timer);
+            // The cause is dropped: a provider error can echo a request body. The
+            // HTTP status stays, because callers tell "already gone" by it.
+            const failure = new Error(`${label} failed`);
+            const status = error?.status;
+            if (typeof status === 'number')
+                failure.status = status;
+            reject(failure);
+        });
+    });
+}
+const isGone = (error) => {
+    const status = error?.status;
+    return typeof status === 'number' && status === 404;
+};
+/**
+ * Makes the Worker app true. Throws a fixed message on a failure, naming the
+ * step and never a value or a provider's cause for anything that carried a
+ * secret. Every recorded id is persisted before the call that could lose it.
+ */
+export async function applyWorkerApp(deps, org, action, context) {
+    const entryKey = action.key.slice(org.id.length + 1);
+    const at = `cloudflare:worker-app for ${org.slug} entry '${entryKey}'`;
+    const ms = deps.timeoutMs ?? WORKER_APP_CALL_TIMEOUT_MS;
+    const call = (label, task) => withCallTimeout(ms, `${at}: ${label}`, task);
+    const mode = action.desired.mode;
+    if (mode === 'retire')
+        return retireWorkerApp(deps, org, entryKey, context, call);
+    const want = action.desired;
+    const entry = org.hosting.entries.find((e) => e.key === entryKey);
+    if (!entry?.workerApp)
+        throw new Error(`${at}: the entry no longer declares workerApp`);
+    const { workers } = deps;
+    // Route and zone first: a foreign route is refused before anything is uploaded.
+    const zone = await call('find the zone', () => deps.zones.findZoneByApex(want.apex));
+    if (!zone) {
+        throw new Error(`${at}: no Cloudflare zone found for ${want.apex} -- the zone must exist first`);
+    }
+    const pattern = `${want.host}/*`;
+    const existingRoute = await call('find the route', () => workers.findRoute(zone.id, pattern));
+    if (existingRoute && existingRoute.script !== want.name) {
+        throw new Error(`${at}: route '${pattern}' is already bound to Worker '${existingRoute.script}', not foundry's own '${want.name}'; foundry does not adopt or replace it`);
+    }
+    const recorded0 = context.recordedIds();
+    const everUploaded = Boolean(recorded0[workerAppAppliedScopeKey(entryKey)] ||
+        recorded0[workerAppUploadedScopeKey(entryKey)] ||
+        recorded0[workerAppResourceKey(action.key)]);
+    if (!everUploaded &&
+        (await call('check the script name', () => workers.scriptExists(want.name)))) {
+        throw new Error(`${at}: a Worker script named '${want.name}' already exists and foundry never created it; refuses to overwrite it`);
+    }
+    // 0. The database, before the upload: a failed migration leaves the live Worker as it was.
+    let hyperdriveId;
+    if (want.database) {
+        if (!deps.database) {
+            throw new Error(`${at}: this executor was built without a WorkerAppDatabasePort`);
+        }
+        ({ hyperdriveId } = await applyWorkerAppDatabase({ ...deps.database, heldKeys: deps.heldKeys, timeoutMs: ms }, org, entry, want.name, { artifactUrl: want.artifactUrl, database: want.database }, context, want.apex));
+    }
+    // 1. Trace, then upload with only the migrations the script does not hold.
+    context.recordScopeId(workerAppUploadedScopeKey(entryKey), want.name);
+    context.recordScopeId(workerAppRetiredScopeKey(entryKey), '');
+    await context.persistScopeIds();
+    const declaredTags = want.migrations.map((m) => m.tag);
+    const recordedTags = parseJson(recorded0[workerAppTagsScopeKey(entryKey)], []);
+    // The live tag is the fact; the record can be behind it after a crash between upload and record.
+    const liveTag = await call('read the migration tag', () => workers.liveMigrationTag(want.name));
+    let appliedCount;
+    if (liveTag === undefined)
+        appliedCount = 0;
+    else {
+        const index = declaredTags.indexOf(liveTag);
+        if (index < 0) {
+            throw new Error(`${at}: the live Worker holds migration '${liveTag}', which is not in the declared list; migrations are append-only and foundry will not guess`);
+        }
+        appliedCount = index + 1;
+    }
+    const recordedError = migrationPrefixError(recordedTags, declaredTags);
+    if (recordedError)
+        throw new Error(`${at}: ${recordedError}`);
+    const pending = want.migrations.slice(appliedCount);
+    const migrations = pending.length > 0
+        ? {
+            ...(liveTag !== undefined ? { oldTag: liveTag } : {}),
+            newTag: pending[pending.length - 1]?.tag,
+            steps: pending.map((m) => ({
+                ...(m.newSqliteClasses ? { newSqliteClasses: m.newSqliteClasses } : {}),
+                ...(m.deletedClasses ? { deletedClasses: m.deletedClasses } : {}),
+                ...(m.renamedClasses ? { renamedClasses: m.renamedClasses } : {}),
+            })),
+        }
+        : undefined;
+    await call('upload the script', () => workers.uploadWorkerApp({
+        name: want.name,
+        artifactUrl: want.artifactUrl,
+        artifactSha256: want.artifactSha256,
+        compatibilityDate: want.compatibilityDate,
+        compatibilityFlags: want.compatibilityFlags,
+        bindings: [
+            ...want.classes.map((c) => ({
+                type: 'durable_object_namespace',
+                name: c.binding,
+                class_name: c.className,
+            })),
+            ...want.vars.map(([name, text]) => ({
+                type: 'plain_text',
+                name,
+                text,
+            })),
+            ...(hyperdriveId
+                ? [{ type: 'hyperdrive', name: WORKER_APP_HYPERDRIVE_BINDING, id: hyperdriveId }]
+                : []),
+        ],
+        ...(migrations ? { migrations } : {}),
+    }));
+    context.recordScopeId(workerAppTagsScopeKey(entryKey), JSON.stringify(declaredTags));
+    await context.persistScopeIds();
+    // A Worker that no longer declares a database no longer binds its config: delete it now.
+    if (!want.database) {
+        await deleteWorkerAppHyperdrive(deps.database?.port, entryKey, context.recordedIds(), context, call);
+        await deleteWorkerAppDatabaseRoute(deps.database, deps.heldKeys, org.id, entryKey, context.recordedIds(), context, call);
+    }
+    // 2. Secrets: before the route, so the Worker never serves without them.
+    await deliverSecrets(deps, org, entryKey, want, context, at, call);
+    // 3. Containers.
+    await syncContainers(deps, entryKey, want, context, at, call);
+    // 4. The record on the org's own zone, 5. the route.
+    if (want.apex !== WORKER_APP_OIKOS_ZONE) {
+        await ensureRecord(deps, entryKey, zone.name, want.host, context, at, call);
+    }
+    let routeId = existingRoute?.id;
+    const routeZone = {
+        zoneId: zone.id,
+        apex: want.apex,
+        pattern,
+    };
+    if (!routeId) {
+        context.recordScopeId(workerAppRoutePendingScopeKey(entryKey), JSON.stringify(routeZone));
+        await context.persistScopeIds();
+        routeId = (await call('create the route', () => workers.createRoute(zone.id, pattern, want.name))).id;
+    }
+    const previous = parseJson(context.recordedIds()[workerAppRouteScopeKey(entryKey)], null);
+    if (previous?.id && (previous.id !== routeId || previous.zoneId !== zone.id)) {
+        await call('delete the previous route', () => workers.deleteRoute(previous.zoneId, previous.id));
+    }
+    context.recordScopeId(workerAppRouteScopeKey(entryKey), JSON.stringify({ ...routeZone, id: routeId }));
+    context.recordScopeId(workerAppRoutePendingScopeKey(entryKey), '');
+    // 6. Last: a run that failed above plans the step again.
+    const { mode: _mode, ...marker } = want;
+    context.recordScopeId(workerAppAppliedScopeKey(entryKey), JSON.stringify(marker));
+    context.recordScopeId(workerAppUploadedScopeKey(entryKey), '');
+    return { resourceId: want.name };
+}
+async function deliverSecrets(deps, org, entryKey, want, context, at, call) {
+    const { workers, heldKeys } = deps;
+    const delivered = parseJson(context.recordedIds()[workerAppSecretsScopeKey(entryKey)], {});
+    const wanted = new Set(want.secrets.map((s) => s.name));
+    // A secret no longer declared is deleted from the Worker.
+    for (const name of Object.keys(delivered)) {
+        if (wanted.has(name))
+            continue;
+        await call(`delete secret ${name}`, () => workers.deleteSecret(want.name, name));
+        delete delivered[name];
+        context.recordScopeId(workerAppSecretsScopeKey(entryKey), JSON.stringify(delivered));
+        await context.persistScopeIds();
+    }
+    if (!wanted.has(WORKER_APP_KEYS_SECRET) &&
+        context.recordedIds()[workerAppKeysScopeKey(entryKey)]) {
+        // The custody Worker has no revoke; the credential expires on its own and the held copy stays.
+        context.recordScopeId(workerAppKeysScopeKey(entryKey), '');
+        await context.persistScopeIds();
+    }
+    // Generated secrets: held first, so a crash hands the same value out again.
+    const generated = [];
+    for (const secret of want.secrets) {
+        if (secret.source !== 'generated')
+            continue;
+        if (!heldKeys?.put) {
+            throw new Error(`${at}: generated secret ${secret.name} needs a held-key store that can put`);
+        }
+        const ref = workerAppSecretRef(org.id, entryKey, secret.name);
+        let value = await call('read a held secret', () => heldKeys.resolve(ref)).then((v) => v ?? '');
+        if (!value) {
+            value = randomBytes(32).toString('base64url');
+            const minted = value;
+            await call('hold a generated secret', () => heldKeys.put(ref, minted));
+        }
+        const version = secretVersion(value);
+        if (delivered[secret.name] !== version)
+            generated.push({ name: secret.name, value, version });
+    }
+    if (generated.length > 0) {
+        await call('deliver secrets', () => workers.setSecrets(want.name, generated));
+        for (const { name, version } of generated)
+            delivered[name] = version;
+        context.recordScopeId(workerAppSecretsScopeKey(entryKey), JSON.stringify(delivered));
+        await context.persistScopeIds();
+    }
+    const custody = want.secrets.find((s) => s.source === 'keysCustody');
+    if (custody?.hostId) {
+        await deliverCustodyCredential(deps, org, entryKey, want, custody.hostId, delivered, context, at, call);
+    }
+}
+const parseHeldCredential = (raw) => {
+    const held = parseJson(raw ?? undefined, null);
+    return held &&
+        typeof held.credentialId === 'string' &&
+        typeof held.credential === 'string' &&
+        held.credential !== '' &&
+        typeof held.expiresAt === 'string' &&
+        !Number.isNaN(Date.parse(held.expiresAt))
+        ? held
+        : null;
+};
+/**
+ * Mints the custody credential once, holds it (the held-key store, JSON with
+ * the expiry) before delivery, and delivers the held value again unchanged
+ * after a crash. Mints again only when the held one is within 30 days of
+ * expiry. The credential never reaches an error, a log or a recorded id: the
+ * recorded `workerapp_keys_<entry>` carries the host id, credential id and
+ * expiry only.
+ */
+async function deliverCustodyCredential(deps, org, entryKey, want, hostId, delivered, context, at, call) {
+    const { keys, heldKeys } = deps;
+    if (!keys)
+        throw new Error(`${at}: a keysCustody secret needs the host's keys custody port`);
+    if (!heldKeys?.put) {
+        throw new Error(`${at}: the custody credential needs a held-key store that can put`);
+    }
+    const put = heldKeys.put;
+    const ref = workerAppSecretRef(org.id, entryKey, WORKER_APP_KEYS_SECRET);
+    const now = (deps.now ?? (() => new Date()))().getTime();
+    let held = parseHeldCredential(await call('read the held credential', () => heldKeys.resolve(ref)));
+    if (!held || Date.parse(held.expiresAt) - WORKER_APP_CREDENTIAL_RENEW_BEFORE_MS <= now) {
+        const minted = await call('mint a custody credential', () => keys.mintDerivedCredential(hostId));
+        const fresh = parseHeldCredential(JSON.stringify(minted));
+        if (!fresh) {
+            throw new Error(`${at}: the custody Worker returned a credential without an id, a value or an expiry`);
+        }
+        await call('hold the custody credential', () => put(ref, JSON.stringify(fresh)));
+        held = fresh;
+    }
+    const credential = held;
+    if (delivered[WORKER_APP_KEYS_SECRET] !== credential.credentialId) {
+        let failed = false;
+        for (let attempt = 1; attempt <= WORKER_APP_KEYS_DELIVERY_ATTEMPTS; attempt++) {
+            try {
+                await call('deliver the custody credential', () => deps.workers.setSecrets(want.name, [
+                    {
+                        name: WORKER_APP_KEYS_SECRET,
+                        value: credential.credential,
+                        version: credential.credentialId,
+                    },
+                ]));
+                failed = false;
+                break;
+            }
+            catch {
+                failed = true; // The cause may echo the value: dropped.
+            }
+        }
+        if (failed) {
+            throw new Error(`${at}: the custody credential ${credential.credentialId} is held but could not be delivered after ${WORKER_APP_KEYS_DELIVERY_ATTEMPTS} attempts; the next apply delivers it again`);
+        }
+        delivered[WORKER_APP_KEYS_SECRET] = credential.credentialId;
+        context.recordScopeId(workerAppSecretsScopeKey(entryKey), JSON.stringify(delivered));
+    }
+    context.recordScopeId(workerAppKeysScopeKey(entryKey), JSON.stringify({
+        hostId,
+        credentialId: credential.credentialId,
+        expiresAt: credential.expiresAt,
+    }));
+    await context.persistScopeIds();
+}
+async function syncContainers(deps, entryKey, want, context, at, call) {
+    const key = workerAppContainersScopeKey(entryKey);
+    const recorded = parseJson(context.recordedIds()[key], {});
+    const save = async () => {
+        context.recordScopeId(key, Object.keys(recorded).length > 0 ? JSON.stringify(recorded) : '');
+        await context.persistScopeIds();
+    };
+    const wanted = new Set(want.containers.map((c) => c.className));
+    for (const [className, held] of Object.entries(recorded)) {
+        if (wanted.has(className))
+            continue;
+        await call(`delete the ${className} container`, () => deps.workers.deleteContainerApplication(held.id));
+        delete recorded[className];
+        await save();
+    }
+    for (const container of want.containers) {
+        const held = recorded[container.className];
+        if (held &&
+            held.image === container.image &&
+            held.instanceType === container.instanceType &&
+            held.maxInstances === container.maxInstances) {
+            continue;
+        }
+        const { id } = await call(`ensure the ${container.className} container`, () => deps.workers.ensureContainerApplication({
+            ...container,
+            scriptName: want.name,
+        }));
+        recorded[container.className] = { ...container, id };
+        await save();
+    }
+    void at;
+}
+async function ensureRecord(deps, entryKey, zoneName, host, context, at, call) {
+    const key = workerAppDnsScopeKey(entryKey);
+    const previous = parseJson(context.recordedIds()[key], null);
+    if (previous?.name === host && previous.zone === zoneName)
+        return;
+    const found = await call('look for the DNS record', () => deps.dns.findDnsRecord(zoneName, host));
+    let id;
+    const ours = found.find((r) => r.type === WORKER_APP_DNS_RECORD.type && r.content === WORKER_APP_DNS_RECORD.content);
+    if (ours)
+        id = ours.id;
+    else if (found.length > 0) {
+        throw new Error(`${at}: ${host} already has a DNS record that foundry did not make; it does not replace it`);
+    }
+    else {
+        id = (await call('create the DNS record', () => deps.dns.createDnsRecord(zoneName, {
+            ...WORKER_APP_DNS_RECORD,
+            name: host,
+            proxied: true,
+        }))).id;
+    }
+    if (previous && (previous.name !== host || previous.zone !== zoneName)) {
+        await call('delete the previous DNS record', () => deps.dns.deleteDnsRecord(previous.zone, previous.id));
+    }
+    context.recordScopeId(key, JSON.stringify({ zone: zoneName, id, name: host }));
+    await context.persistScopeIds();
+}
+/**
+ * Tears a Worker app down in the order that never leaves a route to nothing:
+ * route, record, containers, then the script (its secrets go with it).
+ * Held generated secrets stay, so redeclaring reuses them. Used by retire and
+ * by org destroy.
+ */
+export async function teardownWorkerApp(deps, entryKey, scriptName, recorded, call, orgId, context = null) {
+    const { workers } = deps;
+    const route = parseJson(recorded[workerAppRouteScopeKey(entryKey)], null);
+    if (route?.id)
+        await call('delete the route', () => workers.deleteRoute(route.zoneId, route.id));
+    const pending = parseJson(recorded[workerAppRoutePendingScopeKey(entryKey)], null);
+    if (pending) {
+        const found = await call('find the pending route', () => workers.findRoute(pending.zoneId, pending.pattern));
+        if (found && found.script === scriptName) {
+            await call('delete the pending route', () => workers.deleteRoute(pending.zoneId, found.id));
+        }
+    }
+    const record = parseJson(recorded[workerAppDnsScopeKey(entryKey)], null);
+    if (record) {
+        try {
+            await call('delete the DNS record', () => deps.dns.deleteDnsRecord(record.zone, record.id));
+        }
+        catch (error) {
+            if (!isGone(error))
+                throw error;
+        }
+    }
+    const containers = parseJson(recorded[workerAppContainersScopeKey(entryKey)], {});
+    for (const [className, held] of Object.entries(containers)) {
+        await call(`delete the ${className} container`, () => workers.deleteContainerApplication(held.id));
+    }
+    if (scriptName)
+        await call('delete the script', () => workers.deleteScript(scriptName));
+    // After the script: Cloudflare refuses to delete a config a live Worker binds.
+    await deleteWorkerAppHyperdrive(deps.database?.port, entryKey, recorded, null, call);
+    // After the config: it presents the service token and dials the route.
+    await deleteWorkerAppDatabaseRoute(deps.database, deps.heldKeys, orgId, entryKey, recorded, context, call);
+}
+async function retireWorkerApp(deps, org, entryKey, context, call) {
+    const recorded = context.recordedIds();
+    const mainId = recorded[workerAppResourceKey(`${org.id}:${entryKey}`)];
+    const scriptName = mainId || recorded[workerAppUploadedScopeKey(entryKey)];
+    await teardownWorkerApp(deps, entryKey, scriptName, recorded, call, org.id, context);
+    for (const scopeKey of workerAppTraceScopeKeys(entryKey))
+        context.recordScopeId(scopeKey, '');
+    // `runSteps` writes the main id only from a truthy `resourceId`, so the marker keeps the scan from retiring it forever.
+    if (mainId)
+        context.recordScopeId(workerAppRetiredScopeKey(entryKey), mainId);
+    return {};
+}