@substrat-run/control-plane-api 0.48.1 → 0.50.0

package/dist/api.js

@@ -1,14 +1,15 @@
 import { Hono } from 'hono';
-import { adminAction, channelName, createTenantInput, entitlementGrantInput, hostname as hostnameSchema, hostnameRegion, hostnameStatus, identityLink, listPageQuery, pageOf, principalId as principalIdSchema, promotionAcknowledgement, provisionableJurisdiction, publishVersionInput, queryScopeInput, readScopeTableInput, registerVerticalInput, scopeDump, scopeId as scopeIdSchema, scopeStatus, storageShape, surfaceName, tenantId as tenantIdSchema, tenantStatus, z, } from '@substrat-run/contracts';
+import { adminAction, ASSET_PART_PREFIX, assetHash, channelName, createTenantInput, entitlementGrantInput, hostname as hostnameSchema, hostnameRegion, hostnameStatus, identityLink, listPageQuery, pageOf, principalId as principalIdSchema, promotionAcknowledgement, provisionableJurisdiction, publishVersionInput, queryScopeInput, readScopeTableInput, registerVerticalInput, scopeDump, scopeId as scopeIdSchema, scopeStatus, storageShape, surfaceName, tenantId as tenantIdSchema, tenantStatus, z, } from '@substrat-run/contracts';
 import { migrationProgress, ulid } from '@substrat-run/kernel';
 import { TENANT_HEADER } from './auth.js';
 import { ControlPlaneError } from './client.js';
 import { provisionSiblingScope } from './platform-drain.js';
 import { mapError } from './errors.js';
-import { maskDump } from './mask.js';
+import { maskDump, maskRecords } from './mask.js';
 import { assertSandboxContract, deployManifest, storedDeployManifest, deploymentRefFor, stableDeploymentRefFor, nextMigrationTag, upstreamStatusOf, } from './deploy.js';
 import { blobStoreBindings, collectBlobStoreHandles, collectTenantStoreHandles, tenantStoreBindings, } from './tenant-stores.js';
 import { mintPushToken, pushActorFor } from './push-token.js';
+import { backupDirectoryIfDue } from './directory-backup.js';
 import { isCustomHostname, validateBindableHostname, } from './custom-hostnames.js';
 // -- request schemas ---------------------------------------------------------
 // Parse, don't trust: every input crosses Zod at the boundary. The ids stay
@@ -142,6 +143,31 @@ const snapshotScopeBody = z.object({
     kind: z.string().min(1).optional(),
     expiresAt: z.string().datetime({ offset: true }).optional(),
 });
+// A reap request (#493). `backup` is deliberately TRI-STATE, not a defaulted boolean:
+//   true      — back up or refuse (the console always sends this, so a control plane
+//               deployed without a backup store fails loudly instead of quietly wiping)
+//   false     — the explicit "I accept an unrecoverable wipe"
+//   undefined — back up if a store is configured, proceed without one if not, which is
+//               what keeps every pre-#493 caller (and self-host) working unchanged
+const reapScopeBody = z.object({
+    backup: z.boolean().optional(),
+});
+// A directory-restore request (#40). `capturedAt` addresses the copy — there is one
+// directory, so that is its whole address. `overwrite` is the guard against the
+// dangerous case: replaying a restore onto a control plane that already recovered.
+const restoreDirectoryBody = z.object({
+    capturedAt: z.string().min(1),
+    overwrite: z.boolean().optional(),
+});
+/**
+ * How a stored backup is named in the admin log and to callers: the route that fetches
+ * it. Store-neutral by construction — the R2 key scheme stays private to the store — and
+ * an operator reading a reap entry gets an address they can actually GET, rather than a
+ * bucket path they would have to know the platform's internals to use.
+ */
+function backupRefOf(b) {
+    return `/tenants/${b.tenantId}/scopes/${b.scopeId}/backups/${b.capturedAt}`;
+}
 // A per-PR preview request (preview-and-snapshots.md §2/§9 — the "run a new version
 // against a fork of prod" slice). `tag` is a short DNS-safe label (`pr-123`): it names
 // the preview both in its scope slug (`<vertical>--<tag>`) and in its hostname
@@ -391,6 +417,7 @@ export function createControlPlaneApi(options) {
     app.post('/tenants/:tenantId/reap', async (c) => {
         const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
         const actor = c.get('actor');
+        const { backup: wantsBackup } = reapScopeBody.parse(await c.req.json().catch(() => ({})));
         const tenant = await admin.getTenant(actor, tenantId);
         if (!tenant)
             return c.json({ error: `unknown tenant: ${tenantId}` }, 404);
@@ -405,12 +432,22 @@ export function createControlPlaneApi(options) {
                     continue;
                 if (scope.status !== 'archived')
                     await admin.archiveScope(actor, tenantId, scope.id);
+                // A backup here is OPT-IN, the inverse of the per-scope reap's default (#493).
+                // A scope reap is operational cleanup, so leaving a copy is the safe default; a
+                // TENANT reap is the deletion of a customer, and §4.8 exists partly to serve an
+                // Art. 17 erasure — silently writing that customer's data to a bucket the reap
+                // does not clear would defeat the request it was made to satisfy. Staff who are
+                // retiring (not erasing) a tenant pass `backup: true` deliberately.
+                const backup = wantsBackup === true ? await backupScope(c, tenantId, scope) : null;
                 const vertical = await verticalForScope(c, scope);
                 if (vertical)
                     await vertical.deleteScope({ scopeId: scope.id });
                 // Tenant teardown reaps every scope and releases every name by design — force past
                 // the bound-hostname guard (which fences the interactive per-scope reap route below).
-                await admin.reapScope(actor, tenantId, scope.id, { force: true });
+                await admin.reapScope(actor, tenantId, scope.id, {
+                    force: true,
+                    ...(backup ? { backupRef: backupRefOf(backup) } : {}),
+                });
             }
             await admin.reapTenant(actor, tenantId);
             return c.json(await admin.getTenant(actor, tenantId));
@@ -1051,6 +1088,154 @@ export function createControlPlaneApi(options) {
             throw e;
         }
     });
+    // -- backups (#493) --------------------------------------------------------
+    // The recoverable copy a reap leaves behind. Delegation mirrors the export route
+    // exactly: `admin.exportScope` is the canonical call — it writes the K-24 access-log
+    // entry, and IS the bytes when the host is co-located — and a vertical-held scope's
+    // real tables overlay it. FULL fidelity, never masked: a masked dump cannot restore,
+    // and a backup that cannot restore is a false promise (see `backups.ts`).
+    const backupScope = async (c, tenantId, scope) => {
+        const store = options.scopeBackups;
+        if (!store) {
+            throw new ControlPlaneError(501, 'no backup target configured — this control plane cannot store a scope backup ' +
+                '(bind one, or reap with backup=false to accept an unrecoverable wipe)');
+        }
+        // Residency (K-7/K-32): the platform bucket is global, so writing a jurisdiction-
+        // pinned scope's bytes into it would move them out of the region the scope was
+        // promised. Refuse rather than back up to the wrong place — and, because the reap
+        // aborts with it, rather than wipe a scope we cannot legally copy.
+        if (scope.jurisdiction !== 'global') {
+            throw new ControlPlaneError(409, `scope ${scope.id} is pinned to '${scope.jurisdiction}' — the platform backup ` +
+                `store is global, so a backup would move its data out of that jurisdiction ` +
+                `(K-32); refused until a per-jurisdiction store exists`);
+        }
+        const dump = await admin.exportScope(c.get('actor'), tenantId, scope.id);
+        const vertical = await verticalForScope(c, scope);
+        const tables = vertical ? await vertical.exportScope(scope.id) : dump.tables;
+        return store.put({ vertical: scope.vertical, dump: { ...dump, tables } });
+    };
+    // The copies held for one scope — metadata only, so listing a reaped scope's backups
+    // is cheap and hands out no bytes. Readable AFTER the reap (that is the point): the
+    // directory row survives as a tombstone, and this is what tells the operator a
+    // recoverable copy exists and when it was taken.
+    app.get('/tenants/:tenantId/scopes/:scopeId/backups', async (c) => {
+        const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
+        const scopeId = scopeIdSchema.parse(c.req.param('scopeId'));
+        if (!options.scopeBackups)
+            return c.json({ error: 'no backup target configured' }, 501);
+        // Reads the directory, not the store, for the tenant cross-check: the store is keyed
+        // by (tenant, scope) but nothing there proves the caller's tenant owns the scope.
+        const scope = await admin.getScopeRecord(c.get('actor'), tenantId, scopeId);
+        if (!scope)
+            return c.json({ error: `unknown scope for tenant: (${tenantId}, ${scopeId})` }, 404);
+        return c.json(await options.scopeBackups.list({ tenantId, scopeId }));
+    });
+    // One backup's DUMP — the restore source. Staff-only like the export it came from
+    // (not in BUILDER_ROUTES), and full-fidelity, so it is the same governed-pull posture:
+    // K-3 cross-checked above, and `POST …/restore` is where it goes back.
+    app.get('/tenants/:tenantId/scopes/:scopeId/backups/:capturedAt', async (c) => {
+        const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
+        const scopeId = scopeIdSchema.parse(c.req.param('scopeId'));
+        const capturedAt = c.req.param('capturedAt');
+        if (!options.scopeBackups)
+            return c.json({ error: 'no backup target configured' }, 501);
+        const scope = await admin.getScopeRecord(c.get('actor'), tenantId, scopeId);
+        if (!scope)
+            return c.json({ error: `unknown scope for tenant: (${tenantId}, ${scopeId})` }, 404);
+        const dump = await options.scopeBackups.get({ tenantId, scopeId, capturedAt });
+        if (!dump)
+            return c.json({ error: `no backup for scope ${scopeId} at ${capturedAt}` }, 404);
+        return c.json(dump);
+    });
+    // Take a backup WITHOUT reaping — the standalone copy (a pre-migration checkpoint, an
+    // export-to-keep). The reap route below takes its own; this is the same act made
+    // available on its own, so "back up" is not something only a destructive path can do.
+    app.post('/tenants/:tenantId/scopes/:scopeId/backups', async (c) => {
+        const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
+        const scopeId = scopeIdSchema.parse(c.req.param('scopeId'));
+        const scope = await admin.getScopeRecord(c.get('actor'), tenantId, scopeId);
+        if (!scope)
+            return c.json({ error: `unknown scope for tenant: (${tenantId}, ${scopeId})` }, 404);
+        try {
+            return c.json(await backupScope(c, tenantId, scope), 201);
+        }
+        catch (e) {
+            if (e instanceof ControlPlaneError) {
+                return c.json({ error: e.message }, e.status);
+            }
+            throw e;
+        }
+    });
+    // -- directory backups (#40) -----------------------------------------------
+    // The platform's own disaster recovery, on the same store posture as the scope
+    // backups above and deliberately on a different axis from them: a scope has ~30-day
+    // point-in-time recovery, the directory has one Durable Object and no second copy of
+    // the mapping that makes every scope addressable. These routes are the manual arms of
+    // the cron phase (`backupDirectoryIfDue`) — take one now, see what is held, and the
+    // break-glass restore. Staff-only: none is in BUILDER_ROUTES, and none is per-tenant,
+    // because the subject of all three is every tenant at once.
+    // What copies exist, newest first — metadata only, so this is cheap and hands out no
+    // bytes. This is also the answer to "is the backup actually running?", which is the
+    // question an unrehearsed backup story never has a way to ask.
+    app.get('/directory/backups', async (c) => {
+        if (!options.directoryBackups)
+            return c.json({ error: 'no directory backup target configured' }, 501);
+        return c.json(await options.directoryBackups.list());
+    });
+    // Take one NOW, cadence ignored — the pre-migration checkpoint an operator takes by
+    // hand before touching the directory, and the way a fresh deployment gets its first
+    // copy without waiting a day for the cron.
+    app.post('/directory/backups', async (c) => {
+        if (!options.directoryBackups)
+            return c.json({ error: 'no directory backup target configured' }, 501);
+        const result = await backupDirectoryIfDue({
+            admin,
+            store: options.directoryBackups,
+            actor: c.get('actor'),
+            force: true,
+        });
+        return c.json(result.taken, 201);
+    });
+    // One copy's DUMP — the restore source, and the off-platform escape hatch: an operator
+    // who wants the directory on their own disk GETs this. The most privileged read the
+    // control plane offers (every tenant, every hostname, every identity), so staff-only
+    // and audited by `exportDirectory` underneath.
+    app.get('/directory/backups/:capturedAt', async (c) => {
+        if (!options.directoryBackups)
+            return c.json({ error: 'no directory backup target configured' }, 501);
+        const capturedAt = c.req.param('capturedAt');
+        const dump = await options.directoryBackups.get({ capturedAt });
+        if (!dump)
+            return c.json({ error: `no directory backup at ${capturedAt}` }, 404);
+        return c.json(dump);
+    });
+    // Break-glass: REPLACE the directory with a stored copy.
+    //
+    // Guarded by an explicit `overwrite` rather than a confirmation string, because the
+    // dangerous case is not a slip of the fingers — it is a well-formed retry against a
+    // control plane that has already recovered, which would silently roll the platform
+    // back to the copy's moment and lose every tenant created since. So a directory that
+    // still holds tenants refuses (409) unless the caller says, in the body, that
+    // replacing them is the intent. An EMPTY directory — the actual disaster, a fresh DO
+    // with nothing in it — needs no such ceremony.
+    app.post('/directory/restore', async (c) => {
+        if (!options.directoryBackups)
+            return c.json({ error: 'no directory backup target configured' }, 501);
+        const body = restoreDirectoryBody.parse(await c.req.json().catch(() => ({})));
+        const dump = await options.directoryBackups.get({ capturedAt: body.capturedAt });
+        if (!dump)
+            return c.json({ error: `no directory backup at ${body.capturedAt}` }, 404);
+        const actor = c.get('actor');
+        const live = await admin.listTenants(actor, { limit: 1 });
+        if (live.length > 0 && !body.overwrite) {
+            return c.json({
+                error: 'the directory is not empty — a restore REPLACES it, so anything created ' +
+                    'since this copy was taken would be lost; pass overwrite=true to confirm',
+            }, 409);
+        }
+        await admin.restoreDirectory(actor, dump);
+        return c.json({ capturedAt: dump.capturedAt, tables: dump.tables.length });
+    });
     // Reap an ARCHIVED primary scope (control-plane.md §4.4): free its DO storage —
     // Cloudflare never garbage-collects a Durable Object, so a deleted app's bytes persist
     // forever otherwise — while keeping the directory row as a tombstone. A POST verb, not
@@ -1064,6 +1249,9 @@ export function createControlPlaneApi(options) {
         const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
         const scopeId = scopeIdSchema.parse(c.req.param('scopeId'));
         const actor = c.get('actor');
+        // Body is optional so the bare `POST …/reap` every existing caller sends still
+        // parses; `backup` tri-states on purpose (see the ordering comment below).
+        const { backup: wantsBackup } = reapScopeBody.parse(await c.req.json().catch(() => ({})));
         const scope = await admin.getScopeRecord(actor, tenantId, scopeId);
         if (!scope)
             return c.json({ error: `unknown scope for tenant: (${tenantId}, ${scopeId})` }, 404);
@@ -1082,12 +1270,41 @@ export function createControlPlaneApi(options) {
                     `unbind it before reaping (reap wipes storage and cannot be undone)`,
             }, 409);
         }
+        // The recoverable copy, BEFORE any byte is wiped (#493). Ordering is the whole
+        // guarantee: `backupScope` has to have resolved — durably stored — before
+        // `deleteScope` runs, so a store that throws (or is missing when one was asked for)
+        // aborts the reap with the scope intact. `backup: false` is the explicit "I accept an
+        // unrecoverable wipe"; omitting it backs up when a store is configured and proceeds
+        // without one when the platform has none.
+        //
+        // Its own try/catch, OUTSIDE the reap's: a backup failure and a reap failure are
+        // different facts to an operator, and collapsing a dead bucket into the generic 500
+        // would read as "the reap broke" when the scope is in fact untouched (#321's lesson).
+        let backup = null;
+        if (!(wantsBackup === false || (wantsBackup === undefined && !options.scopeBackups))) {
+            try {
+                backup = await backupScope(c, tenantId, scope);
+            }
+            catch (e) {
+                if (e instanceof ControlPlaneError) {
+                    return c.json({ error: e.message }, e.status);
+                }
+                return c.json({
+                    error: 'backup failed — the scope was NOT reaped and its data is intact; ' +
+                        'retry, or reap with backup=false to accept an unrecoverable wipe',
+                    detail: e instanceof Error ? e.message : String(e),
+                }, 502);
+            }
+        }
         try {
             const vertical = await verticalForScope(c, scope);
             if (vertical)
                 await vertical.deleteScope({ scopeId });
-            await admin.reapScope(actor, tenantId, scopeId);
-            return c.json(await admin.getScopeRecord(actor, tenantId, scopeId));
+            await admin.reapScope(actor, tenantId, scopeId, {
+                ...(backup ? { backupRef: backupRefOf(backup) } : {}),
+            });
+            const reaped = await admin.getScopeRecord(actor, tenantId, scopeId);
+            return c.json({ ...reaped, backup });
         }
         catch (e) {
             if (e instanceof ControlPlaneError) {
@@ -1136,6 +1353,117 @@ export function createControlPlaneApi(options) {
             throw e;
         }
     });
+    // -- tenant export (#36) ----------------------------------------------------
+    //
+    // GDPR Art. 20 portability, and the escrow handover: one tenant, whole, in one file.
+    //
+    // Composed ENTIRELY from the sanctioned reads above — `listScopes`, `listOrgs`,
+    // `listMembers`, `listRoles`, `listEntitlements`, `listIdentityLinks`,
+    // `listHostnames`, the store ledgers, `listConnections`, `exportScope`. That is the
+    // constraint the design puts on this route, not an implementation preference:
+    // control-plane.md §7 says the control plane must not acquire a back door into scope
+    // databases, and the only sanctioned path is the audited admin surface. An export
+    // that reached past it would BE the back door — and every read here is already
+    // K-24 access-logged, so the trail is a property of the parts.
+    //
+    // It is deliberately NOT the same shape as a directory dump (#40): that one is raw
+    // tables for recovery, this one is the platform's documented vocabulary for a reader
+    // who does not know the schema. Only the per-scope `data` is raw, because that is the
+    // half that has to be reloadable.
+    app.get('/tenants/:tenantId/export', async (c) => {
+        const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
+        const actor = c.get('actor');
+        const tenant = await admin.getTenant(actor, tenantId);
+        if (!tenant)
+            return c.json({ error: `unknown tenant: ${tenantId}` }, 404);
+        // Every scope, tombstones included: an archived or reaped scope is part of the
+        // tenant's history, and an export that quietly dropped them would misrepresent what
+        // the tenant was. Their DATA is a different question, handled below.
+        const scopes = await admin.listScopes(actor, { tenantId });
+        // Residency (K-7/K-32), checked across the WHOLE tenant before anything is read: an
+        // export lands on a machine outside the platform's control, so one pinned scope
+        // taints the file. Refused as a unit rather than silently exporting the global
+        // scopes and omitting the pinned ones — a partial export that does not say it is
+        // partial is the failure mode worth avoiding.
+        const pinned = scopes.filter((s) => s.jurisdiction !== 'global');
+        if (pinned.length > 0) {
+            return c.json({
+                error: `tenant ${tenantId} has ${pinned.length} scope(s) pinned to a jurisdiction ` +
+                    `(${[...new Set(pinned.map((s) => s.jurisdiction))].join(', ')}) — an export would ` +
+                    `move that data out of the region it was promised; refused (K-32). Export the ` +
+                    `global scopes individually, or wait for a per-jurisdiction path.`,
+            }, 403);
+        }
+        const full = c.req.query('full') === 'true';
+        try {
+            const orgs = await admin.listOrgs(actor, tenantId);
+            // Revoked memberships included: K-21 makes a removal a tombstone precisely because
+            // "was a member until March" is the fact an audit asks for.
+            const members = (await Promise.all(orgs.map((o) => admin.listMembers(actor, tenantId, o.id, { includeRevoked: true })))).flat();
+            const [roles, entitlements, identityLinks, hostnames, stores, blobStores, connections] = await Promise.all([
+                admin.listRoles(actor, { tenantId }),
+                admin.listEntitlements(actor, tenantId),
+                admin.listIdentityLinks(actor, tenantId),
+                admin.listHostnames(actor, { tenantId }),
+                admin.listTenantStores(actor, { tenantId }),
+                admin.listBlobStores(actor, { tenantId }),
+                admin.listConnections(actor, { tenantId }),
+            ]);
+            // Scope DATA, from the same delegation the per-scope export route uses: the
+            // canonical `exportScope` writes the audit entry and is the bytes when co-located;
+            // a vertical-held scope's real tables overlay it. A reaped scope has no storage
+            // left to read, so it is skipped here while its RECORD stays above — the tombstone
+            // is honest, an error would not be.
+            const live = scopes.filter((s) => s.status !== 'reaped');
+            const data = [];
+            for (const scope of live) {
+                const dump = await admin.exportScope(actor, tenantId, scope.id);
+                const vertical = await verticalForScope(c, scope);
+                const tables = vertical ? await vertical.exportScope(scope.id) : dump.tables;
+                data.push({ ...dump, tables: full ? tables : maskDump(tables) });
+            }
+            // The admin log is FULL-only (#36): it records what STAFF did, so it is not the
+            // customer's Art. 20 data, and it carries staff actor ids and internal action
+            // names. An escrow or a dispute needs it, which is why break-glass reaches it
+            // rather than nothing reaching it.
+            const adminLog = full ? await admin.auditLog(actor, { tenantId }) : null;
+            const body = {
+                tenantId,
+                capturedAt: new Date().toISOString(),
+                masked: !full,
+                tenant: full ? tenant : maskRecords([tenant])[0],
+                scopes: full ? scopes : maskRecords(scopes),
+                orgs: full ? orgs : maskRecords(orgs),
+                members: full ? members : maskRecords(members),
+                // Roles, entitlements and hostnames are configuration rather than personal data,
+                // so they read the same in both fidelities — but they still go through the sweep,
+                // because deciding per-collection what "cannot contain PII" means is exactly the
+                // assumption that ages badly. One rule, applied everywhere.
+                roles: full ? roles : maskRecords(roles),
+                entitlements: full ? entitlements : maskRecords(entitlements),
+                // Identity links are the sharpest item here: `externalId` is usually an email.
+                identityLinks: full ? identityLinks : maskRecords(identityLinks),
+                hostnames: full ? hostnames : maskRecords(hostnames),
+                stores: [...stores, ...blobStores].map((s) => ({
+                    kind: s.kind,
+                    vertical: s.vertical,
+                    binding: s.binding,
+                    ref: s.ref,
+                    createdAt: s.createdAt,
+                })),
+                connections: full ? connections : maskRecords(connections),
+                adminLog,
+                data,
+            };
+            return c.json(body);
+        }
+        catch (e) {
+            if (e instanceof ControlPlaneError) {
+                return c.json({ error: e.message }, e.status);
+            }
+            throw e;
+        }
+    });
     // The write half of the governed pull (§8) — load a dump INTO an existing scope:
     // restore a backup, back out to a snapshot, or land a locally-built world on a
     // hosted app. Staff-only like the export (not in BUILDER_ROUTES). No jurisdiction
@@ -1578,6 +1906,22 @@ export function createControlPlaneApi(options) {
         const registry = json ? (storedDeployManifest.parse(JSON.parse(json)).registry ?? null) : null;
         return c.json({ registry });
     });
+    // The static files (#340) one version ships: path, size, content type, content address —
+    // read straight out of the retained manifest, which is where they were persisted for the
+    // promote path anyway. Owner-narrowed exactly like the registry route above. `assets` is
+    // null for a version that retained no manifest (pushed pre-#286) or shipped no static
+    // files; the two are distinguishable by the caller only as "nothing to show", which is
+    // all the dashboard panel needs to render an empty state.
+    app.get('/verticals/:slug/versions/:id/assets', async (c) => {
+        const p = c.get('principal');
+        const slug = await resolveVerticalId(c, c.req.param('slug'));
+        if (p.kind === 'builder' && (await ownerOf(p.actor, slug)) !== p.tenantId) {
+            return c.json({ error: 'not found' }, 404);
+        }
+        const json = await admin.versionManifest(c.get('actor'), slug, c.req.param('id'));
+        const assets = json ? (storedDeployManifest.parse(JSON.parse(json)).assets ?? null) : null;
+        return c.json({ assets });
+    });
     app.post('/verticals/:slug/versions/:id/admit', async (c) => {
         const slug = c.req.param('slug');
         const id = c.req.param('id');
@@ -1725,6 +2069,13 @@ export function createControlPlaneApi(options) {
             modules,
             doClasses: manifest.doClasses,
             bindings: [...manifest.bindings, ...storeBindings],
+            // #340: the version's static files travel with it onto the serving script — from
+            // the RETAINED manifest, with no bytes. An asset upload session is driven by
+            // content addresses, and the runtime's asset store is namespace-wide and deduped,
+            // so re-declaring the same hashes re-attaches the same files. This is why the
+            // manifest is retained rather than the bytes: the archive script gives back the
+            // modules (#286), and the asset store gives back the assets.
+            ...(manifest.assets ? { assets: manifest.assets } : {}),
         }, serving
             ? { priorDoClasses: serving.doClasses, priorMigrationTag: serving.migrationTag }
             : undefined);
@@ -1915,21 +2266,72 @@ export function createControlPlaneApi(options) {
                 }, 409);
             }
         }
+        // Two kinds of part in one body (#340): worker MODULES (unprefixed) and static ASSETS
+        // (`asset:<served path>`). Splitting on the prefix — rather than on content type or on
+        // whether the manifest happens to name it — is what keeps an uploaded file from being
+        // able to enter the wrong pipeline.
         const modules = [];
+        const assetParts = new Map();
         for (const [name, value] of form.entries()) {
             if (name === 'manifest')
                 continue;
-            if (value instanceof File) {
-                modules.push({
-                    name,
-                    content: new Uint8Array(await value.arrayBuffer()),
-                    contentType: value.type || 'application/javascript+module',
-                });
+            if (!(value instanceof File))
+                continue;
+            if (name.startsWith(ASSET_PART_PREFIX)) {
+                assetParts.set(name.slice(ASSET_PART_PREFIX.length), value);
+                continue;
             }
+            modules.push({
+                name,
+                content: new Uint8Array(await value.arrayBuffer()),
+                contentType: value.type || 'application/javascript+module',
+            });
         }
         if (!modules.some((m) => m.name === manifest.entry)) {
             return c.json({ error: `entry module '${manifest.entry}' is not among the uploaded files` }, 400);
         }
+        // The static-asset half of the §4 sandbox contract (self-serve-deploy.md §4.1). The
+        // bytes are inert — no code, no authority — so they are ACCEPTED; the content-address
+        // is not, so it is VERIFIED. The runtime's asset store dedups by hash across the whole
+        // dispatch namespace, which means bytes stored under a hash they do not have would let
+        // one push decide what another vertical's identical-hash asset serves. Re-deriving the
+        // hash here (from the received bytes, with the same `assetHash` the CLI used) is what
+        // makes that structurally impossible — and it is the only inspection of the bytes we do.
+        const assets = [];
+        for (const file of manifest.assets?.files ?? []) {
+            const part = assetParts.get(file.path);
+            if (!part) {
+                return c.json({ error: `asset '${file.path}' is named in the manifest but was not uploaded` }, 400);
+            }
+            const content = new Uint8Array(await part.arrayBuffer());
+            if (content.byteLength !== file.size) {
+                return c.json({ error: `asset '${file.path}' declares ${file.size} bytes but ${content.byteLength} arrived` }, 400);
+            }
+            const actual = await assetHash(content, file.path);
+            if (actual !== file.hash) {
+                return c.json({
+                    error: `asset '${file.path}' does not match its declared content hash ` +
+                        `(declared ${file.hash}, computed ${actual}) — the hash is the runtime's shared dedup key, ` +
+                        `so a mismatch is refused rather than stored`,
+                }, 400);
+            }
+            assets.push({
+                path: file.path,
+                hash: file.hash,
+                size: file.size,
+                contentType: file.contentType,
+                content,
+            });
+            assetParts.delete(file.path);
+        }
+        if (assetParts.size > 0) {
+            // An unlisted asset part would be uploaded by nothing and served by nothing; saying so
+            // beats silently dropping it, because the builder's page would 404 with no explanation.
+            const extra = [...assetParts.keys()].slice(0, 5).join(', ');
+            return c.json({
+                error: `${assetParts.size} uploaded asset part(s) are absent from the manifest (${extra}${assetParts.size > 5 ? ', …' : ''})`,
+            }, 400);
+        }
         // Mint the version id first: the deploymentRef (the dispatch script name) is keyed
         // on it, so it is CF-valid and unique per version.
         const id = ulid();
@@ -1942,6 +2344,10 @@ export function createControlPlaneApi(options) {
                 modules,
                 doClasses: manifest.doClasses,
                 bindings: manifest.bindings,
+                // #340: the verified bytes go up with the bundle. The manifest's routing config
+                // rides along untouched — it decides what the RUNTIME does with paths, and carries
+                // no reach, so there is nothing in it for the sandbox contract to refuse.
+                ...(manifest.assets ? { assets: { ...manifest.assets, files: assets } } : {}),
             });
         }
         catch (e) {
@@ -2041,16 +2447,20 @@ export function createControlPlaneApi(options) {
         if (!options.observability) {
             return c.json({ error: 'observability is not configured on this control plane' }, 501);
         }
+        // `service` repeats: one deployed unit per param, so a caller can ask for a
+        // vertical's whole set (the dashboard's "all versions") in one query and get one
+        // merged stream back. Capped because each extra service is another backend query.
+        const services = (c.req.queries('service') ?? []).filter((s) => s.length > 0);
         const input = z
             .object({
-            service: z.string().min(1).max(200).optional(),
+            services: z.array(z.string().min(1).max(200)).max(20).optional(),
             level: z.enum(['log', 'info', 'warn', 'error', 'debug']).optional(),
             search: z.string().min(1).max(200).optional(),
             hours: z.coerce.number().int().min(1).max(72).default(1),
             limit: z.coerce.number().int().min(1).max(500).default(100),
         })
             .parse({
-            service: c.req.query('service') || undefined,
+            services: services.length ? services : undefined,
             level: c.req.query('level') || undefined,
             search: c.req.query('search') || undefined,
             hours: c.req.query('hours'),
@@ -2265,6 +2675,16 @@ export function createControlPlaneApi(options) {
      *  label is unique within a tenant, so dropping the prefix stays collision-free. Both
      *  create (provision + reuse-match) and delete (reap-match) run through here, so they agree. */
     const previewSlug = (slug, tag) => `${slug.split('/').at(-1)}--${tag}`;
+    /** Reap one preview: wipe the DO in its own deployment, then drop the directory row and
+     *  its hostnames. Storage-before-row, the same ordering the DELETE route uses — a crash
+     *  between the two converges on retry. Shared by that route and by the create path, which
+     *  reaps a HALF-BUILT leftover before re-forking (see `orchestratedPreview`). */
+    const reapPreview = async (c, preview) => {
+        const vertical = await verticalForScope(c, preview);
+        if (vertical)
+            await vertical.deleteScope({ scopeId: preview.id });
+        await options.host.deleteSnapshot(c.get('actor'), preview.tenantId, preview.id);
+    };
     /** Given a base hostname `<label>.<domain>`, mint (or find) the preview's `--<tag>`
      *  hostname `<label>--<tag>.<domain>` bound to `previewId`. Non-canonical, so it never
      *  demotes the prod surface. Shared by the fork and clean-room paths. */
@@ -2302,13 +2722,22 @@ export function createControlPlaneApi(options) {
             }
             return src.hostname;
         }
-        const base = platformBaseDomains[0];
-        if (!base) {
+        if (platformBaseDomains.length === 0) {
             throw new ControlPlaneError(409, `no platform base domain configured — a clean-room preview has no source URL to derive from`);
         }
+        // Mint under `<label>.<jurisdiction>.<baseDomain>`, exactly as provisioning does
+        // (provision.ts `bindDefaultHostname` → `egeryds.global.substrat.run`). A clean-room
+        // preview scope is provisioned `global` by construction (see the caller), and the
+        // wildcard DNS/cert lives on `*.global.substrat.run` — NOT the certless apex
+        // `*.substrat.run`. `platformBaseDomains` lists every platform suffix for custom-hostname
+        // detection (`substrat.run`, `global.substrat.run`, …); the registrable base is the
+        // shortest, the one all jurisdiction domains are subdomains of. Taking `[0]` grabbed the
+        // bare apex and stranded clean-room previews on a hostname that never resolves.
+        const baseDomain = [...platformBaseDomains].sort((a, b) => a.length - b.length)[0];
+        const jurisdiction = 'global';
         const tenant = await admin.getTenant(actor, tenantId);
         const handle = tenant?.slug ?? tenantId;
-        return `${slug.split('/').at(-1)}-${handle}.${base}`;
+        return `${slug.split('/').at(-1)}-${handle}.${jurisdiction}.${baseDomain}`;
     };
     const orchestratedPreview = async (c, tenantId, slug, 
     // A FORK copies this scope's data; `null` provisions an empty clean-room scope (#509 (b)).
@@ -2361,7 +2790,20 @@ export function createControlPlaneApi(options) {
         // onto the SAME preview — successive pushes roll their migrations forward on one copy
         // (§4's rehearsal case) — unless `refresh` asks for a fresh one.
         const existing = (await admin.listScopes(actor, { tenantId, vertical: slug })).find((s) => s.kind === 'preview' && s.slug === previewSlug(slug, opts.tag));
-        if (existing && !opts.refresh) {
+        // A preview only HAS data once its two-phase create finished: the directory row lands
+        // first as `provisioning` (K-31), the fork's export→restore runs, and `activateScope`
+        // is the last step. So a row still at `provisioning` is a create that DIED mid-fork —
+        // its DO is empty. Reuse must never adopt one: reuse only rebinds the version and the
+        // hostname, it never copies data, so adopting a half-built row hands back a
+        // permanently EMPTY preview and reports `reused: true` — success for a URL that shows
+        // a reviewer no data at all. That is exactly what a CI retry does (the generated
+        // workflow retries `preview create` on a transient), so the failure mode is the
+        // COMMON one, not a corner: attempt 1 forks and dies, attempt 2 adopts its corpse and
+        // goes green. Instead, reap the leftover and fall through to a fresh fork below —
+        // which is what the retry was asking for. Same for an explicit `refresh`, whose fresh
+        // scope would otherwise collide with the old row's still-bound `--<tag>` hostname.
+        const stale = existing !== undefined && (opts.refresh || existing.status !== 'active');
+        if (existing && !stale) {
             // Heal a preview provisioned before #527: clear any inherited serving_ref so routing
             // follows the bound version (its per-version script), not the prod serving script.
             if (existing.servingRef)
@@ -2373,6 +2815,10 @@ export function createControlPlaneApi(options) {
             await assertServesBoundVersion(existing.id);
             return { scopeId: existing.id, hostname, url: `https://${hostname}`, versionId: opts.versionId, reused: true };
         }
+        // Free the tag: the slug is unique per tenant and the `--<tag>` hostname is still bound
+        // to the old row, so the fresh fork below cannot be provisioned until this one is gone.
+        if (existing && stale)
+            await reapPreview(c, existing);
         const previewId = scopeIdSchema.parse(ulid());
         if (source) {
             // A fresh fork. Export from where the prod data lives TODAY. The canonical
@@ -2531,10 +2977,7 @@ export function createControlPlaneApi(options) {
             // Storage-before-row, the same ordering as the fork hard-delete: wipe the DO in the
             // PR version's deployment, then deleteSnapshot (fork-only re-check, hostnames + row,
             // audit). A crash between the two converges on retry.
-            const vertical = await verticalForScope(c, preview);
-            if (vertical)
-                await vertical.deleteScope({ scopeId: preview.id });
-            await options.host.deleteSnapshot(actor, tenantId, preview.id);
+            await reapPreview(c, preview);
             return c.json({ deleted: preview.id });
         }
         catch (e) {