@substrat-run/control-plane-api 0.53.0 → 0.55.0

package/dist/api.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"api.d.ts","sourceRoot":"","sources":["../src/api.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AA+B5B,OAAO,KAAK,EAGV,eAAe,EAMhB,MAAM,yBAAyB,CAAC;AACjC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AAGtD,OAAO,KAAK,EAAE,iBAAiB,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAC3E,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAe3D,OAAO,KAAK,EAAe,gBAAgB,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAC;AACzF,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAQtD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAC9D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EAAuB,KAAK,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACjF,OAAO,KAAK,EAEV,oBAAoB,EAEpB,gBAAgB,EACjB,MAAM,cAAc,CAAC;AAEtB,OAAO,EAGL,KAAK,yBAAyB,EAC/B,MAAM,uBAAuB,CAAC;AAE/B,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,SAAS,CAAC;IAChB;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IAC3C;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,eAAe,KAAK,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,CAAC;IAChG;;;;;;;;;OASG;IACH,sBAAsB,CAAC,EAAE,CACvB,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,eAAe,KACnB,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,CAAC;IACzC;;;;;;;OAOG;IACH,kBAAkB,CAAC,EAAE,CAAC,aAAa,EAAE,MAAM,KAAK,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,CAAC;IACpF;;;;;OAKG;IACH,cAAc,CAAC,EAAE,gBAAgB,CAAC;IAClC;;;;;OAKG;IACH,oBAAoB,CAAC,EAAE,sBAAsB,CAAC;IAC9C;;;;;;OAMG;IACH,mBAAmB,CAAC,EAAE,qBAAqB,CAAC;IAC5C;;;;;;;;OAQG;IACH,sBAAsB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC3C;;;;OAIG;IACH,YAAY,EAAE,iBAAiB,CAAC;IAChC;;;;;;OAMG;IACH,mBAAmB,CAAC,EAAE,WAAW,CAAC;IAClC;;;;;OAKG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;;;;OAQG;IACH,aAAa,CAAC,EAAE,mBAAmB,CAAC;IACpC;;;;;;;;;OASG;IACH,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC;;;;;OAKG;IACH,YAAY,CAAC,EAAE,iBAAiB,CAAC;IACjC;;;;;;;;;;;;;OAaG;IACH,YAAY,CAAC,EAAE,gBAAgB,CAAC;IAChC;;;;;;;;;OASG;IACH,gBAAgB,CAAC,EAAE,oBAAoB,CAAC;IACxC;;;;;;;OAOG;IACH,iBAAiB,CAAC,EAAE,yBAAyB,CAAC;IAC9C;;;;;;OAMG;IACH,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;CAChC;AAKD,KAAK,IAAI,GAAG;IAAE,KAAK,EAAE,eAAe,CAAC;IAAC,SAAS,EAAE,SAAS,CAAA;CAAE,CAAC;AA0O7D;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,sBAAsB,GAAG,IAAI,CAAC;IAAE,SAAS,EAAE,IAAI,CAAA;CAAE,CAAC,CAkhGhG"}
+{"version":3,"file":"api.d.ts","sourceRoot":"","sources":["../src/api.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAgC5B,OAAO,KAAK,EAGV,eAAe,EAMhB,MAAM,yBAAyB,CAAC;AACjC,OAAO,KAAK,EAAmB,SAAS,EAAE,MAAM,sBAAsB,CAAC;AAGvE,OAAO,KAAK,EAAE,iBAAiB,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAC3E,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAgB3D,OAAO,KAAK,EAAe,gBAAgB,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAC;AACzF,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAQtD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAC9D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EAAuB,KAAK,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACjF,OAAO,KAAK,EAEV,oBAAoB,EAEpB,gBAAgB,EACjB,MAAM,cAAc,CAAC;AAEtB,OAAO,EAGL,KAAK,yBAAyB,EAC/B,MAAM,uBAAuB,CAAC;AAE/B,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,SAAS,CAAC;IAChB;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IAC3C;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,eAAe,KAAK,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,CAAC;IAChG;;;;;;;;;OASG;IACH,sBAAsB,CAAC,EAAE,CACvB,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,eAAe,KACnB,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,CAAC;IACzC;;;;;;;OAOG;IACH,kBAAkB,CAAC,EAAE,CAAC,aAAa,EAAE,MAAM,KAAK,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,CAAC;IACpF;;;;;OAKG;IACH,cAAc,CAAC,EAAE,gBAAgB,CAAC;IAClC;;;;;OAKG;IACH,oBAAoB,CAAC,EAAE,sBAAsB,CAAC;IAC9C;;;;;;OAMG;IACH,mBAAmB,CAAC,EAAE,qBAAqB,CAAC;IAC5C;;;;;;;;OAQG;IACH,sBAAsB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC3C;;;;OAIG;IACH,YAAY,EAAE,iBAAiB,CAAC;IAChC;;;;;;OAMG;IACH,mBAAmB,CAAC,EAAE,WAAW,CAAC;IAClC;;;;;OAKG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;;;;OAQG;IACH,aAAa,CAAC,EAAE,mBAAmB,CAAC;IACpC;;;;;;;;;OASG;IACH,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC;;;;;OAKG;IACH,YAAY,CAAC,EAAE,iBAAiB,CAAC;IACjC;;;;;;;;;;;;;OAaG;IACH,YAAY,CAAC,EAAE,gBAAgB,CAAC;IAChC;;;;;;;;;OASG;IACH,gBAAgB,CAAC,EAAE,oBAAoB,CAAC;IACxC;;;;;;;OAOG;IACH,iBAAiB,CAAC,EAAE,yBAAyB,CAAC;IAC9C;;;;;;OAMG;IACH,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;CAChC;AAKD,KAAK,IAAI,GAAG;IAAE,KAAK,EAAE,eAAe,CAAC;IAAC,SAAS,EAAE,SAAS,CAAA;CAAE,CAAC;AA2P7D;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,sBAAsB,GAAG,IAAI,CAAC;IAAE,SAAS,EAAE,IAAI,CAAA;CAAE,CAAC,CAgtGhG"}

package/dist/api.js

@@ -1,11 +1,12 @@
 import { Hono } from 'hono';
-import { adminAction, ASSET_PART_PREFIX, assetHash, channelName, createTenantInput, entitlementGrantInput, hostname as hostnameSchema, hostnameRegion, hostnameStatus, identityLink, listPageQuery, pageOf, principalId as principalIdSchema, promotionAcknowledgement, provisionableJurisdiction, publishVersionInput, queryScopeInput, readScopeTableInput, registerVerticalInput, scopeDump, scopeId as scopeIdSchema, scopeStatus, storageShape, surfaceName, tenantId as tenantIdSchema, tenantStatus, z, } from '@substrat-run/contracts';
+import { adminAction, ASSET_PART_PREFIX, assetHash, channelName, createTenantInput, entitlementGrantInput, hostname as hostnameSchema, hostnameRegion, hostnameStatus, identityLink, listPageQuery, pageOf, principalId as principalIdSchema, promotionAcknowledgement, provisionableJurisdiction, publishVersionInput, queryScopeInput, readScopeTableInput, registerVerticalInput, scopeDump, dataSubjectId as dataSubjectIdSchema, scopeId as scopeIdSchema, scopeStatus, storageShape, surfaceName, tenantId as tenantIdSchema, tenantStatus, z, } from '@substrat-run/contracts';
 import { migrationProgress, ulid } from '@substrat-run/kernel';
 import { TENANT_HEADER } from './auth.js';
 import { ControlPlaneError } from './client.js';
 import { provisionSiblingScope } from './platform-drain.js';
 import { mapError } from './errors.js';
 import { maskDump, maskRecords } from './mask.js';
+import { openDump, sealDump } from './seal.js';
 import { assertSandboxContract, deployManifest, storedDeployManifest, deploymentRefFor, stableDeploymentRefFor, nextMigrationTag, upstreamStatusOf, } from './deploy.js';
 import { blobStoreBindings, collectBlobStoreHandles, collectTenantStoreHandles, tenantStoreBindings, } from './tenant-stores.js';
 import { mintPushToken, pushActorFor } from './push-token.js';
@@ -218,6 +219,21 @@ const auditLogQuery = z.object({
     // everything) — only the HTTP egress vectors are bounded by default.
     ...listPageQuery.shape,
 });
+const opsFailuresQuery = z.object({
+    tenantId: tenantIdSchema.optional(),
+    scopeId: scopeIdSchema.optional(),
+    vertical: z.string().optional(),
+    operation: z.string().optional(),
+    // Exact match — the lookup a CI log's `reference = <id>` line lands on.
+    reference: z.string().optional(),
+    since: z.string().optional(),
+    until: z.string().optional(),
+    // Bounded by default exactly as /admin-log, and for the same reason.
+    ...listPageQuery.shape,
+});
+/** The upstream provider's trace handle, when a failure message carries one —
+ *  Cloudflare's `internal error; reference = <id>` shape (#559). */
+const UPSTREAM_REFERENCE = /\breference\s*=\s*([a-z0-9]+)/i;
 /**
  * The audited HTTP surface over `HostAdmin` (control-plane.md §4.5).
  *
@@ -330,6 +346,10 @@ export function createControlPlaneApi(options) {
         { method: 'POST', re: /\/verticals\/[^/]+\/previews$/ },
         { method: 'GET', re: /\/verticals\/[^/]+\/previews$/ },
         { method: 'DELETE', re: /\/verticals\/[^/]+\/previews\/[^/]+$/ },
+        // The builder's slice of the ops-failure record (#559 step 5): why did MY deploy /
+        // preview / provision fail. Tenant-narrowed in the handler (the forced-filter
+        // pattern, like GET /scopes); the allowlist alone is not authz.
+        { method: 'GET', re: /\/ops-failures$/ },
     ];
     app.use('*', async (c, next) => {
         if (c.get('principal').kind === 'builder') {
@@ -339,6 +359,43 @@ export function createControlPlaneApi(options) {
         }
         await next();
     });
+    // Fire-and-forget ops-failure write (#559): a recorder that throws must never
+    // mask the failure it is recording, so every path through this swallows its own
+    // errors. The upstream reference is extracted here — one place — so a caller
+    // that only has the message still lands a searchable row.
+    const recordFailure = (entry) => {
+        void admin
+            .recordOpsFailure({
+            ...entry,
+            reference: entry.reference ?? UPSTREAM_REFERENCE.exec(entry.message)?.[1] ?? null,
+        })
+            .catch(() => undefined);
+    };
+    // Rides out a transient downstream window: the install path's binding-attach →
+    // script-settings propagation race (#424 case 2), and a one-shot DO storage blip
+    // during an export→restore or snapshot copy (#559 (2)). Retrying is cheap at THESE
+    // call sites specifically — the dump is already in memory and the far end is
+    // drop-then-replay idempotent — unlike CI's retry, which burns a pushed version per
+    // attempt. Honest refusals (4xx, and 501 = not implemented) surface immediately:
+    // retrying a refusal only delays the real message. ~3s worst case on the default
+    // delays, well inside a Worker request budget; a persistent fault still exhausts
+    // and surfaces (and lands an ops-failure row via the paths that record).
+    const PROVISION_RETRY_DELAYS_MS = [750, 2500];
+    const retryTransient = async (fn) => {
+        const delays = options.provisionRetryDelaysMs ?? PROVISION_RETRY_DELAYS_MS;
+        for (let attempt = 0;; attempt++) {
+            try {
+                return await fn();
+            }
+            catch (e) {
+                const transient = e instanceof ControlPlaneError && (e.status === 0 || (e.status >= 500 && e.status !== 501));
+                const delay = delays[attempt];
+                if (!transient || delay === undefined)
+                    throw e;
+                await new Promise((resolve) => setTimeout(resolve, delay));
+            }
+        }
+    };
     // One error boundary for every route: adapters throw plain Errors, and each
     // one is a fail-closed refusal that must reach the caller as a status, not a
     // stack trace.
@@ -347,6 +404,25 @@ export function createControlPlaneApi(options) {
             return c.json({ error: 'invalid request', issues: err.issues }, 400);
         }
         const { status, body } = mapError(err);
+        // A 5xx is the PLATFORM failing — an unmapped throw, or a downstream vertical's
+        // own 5xx passing through (a DO storage fault during a preview restore is the
+        // founding case, #559) — so it lands a durable ops-failure row the console can
+        // list and a `reference = <id>` search can find. 501 stays out: an honest
+        // not-implemented is a capability statement, not a failure. 4xx stay out too:
+        // they are refusals the caller can already read. Actor is unset only when the
+        // throw happened before authentication — nothing worth recording refuses there.
+        const actor = c.get('actor');
+        if (status >= 500 && status !== 501 && actor) {
+            recordFailure({
+                actor,
+                operation: `${c.req.method} ${c.req.routePath}`,
+                vertical: c.req.param('slug') ? decodeURIComponent(c.req.param('slug')) : null,
+                tenantId: c.req.param('tenantId') ?? null,
+                scopeId: c.req.param('scopeId') ?? null,
+                status,
+                message: err instanceof Error ? err.message : String(err),
+            });
+        }
         // A 500 is, by definition, a throw whose message `mapError` did not recognise — so the
         // client gets a GENERIC body that discloses nothing, and until now nothing recorded WHAT
         // threw either. That left every unmapped failure (e.g. a raw SQLite constraint from a
@@ -640,6 +716,19 @@ export function createControlPlaneApi(options) {
     // installed app lags prod. So we prefer bound-version resolution, then fall back to
     // prod-channel/static (a scope with no bound version), then to reading this host's own
     // scope DB directly (a co-located host, or the contract tests — data is right here).
+    /**
+     * The per-subject key operations bound to one scope and one actor (#37) — what
+     * `sealDump`/`openDump` need and nothing more.
+     *
+     * Narrowed on purpose: the seal path gets exactly two capabilities, so a future edit
+     * cannot quietly widen a backup routine into a general admin caller. Every call lands in
+     * the access log under the acting staff member, because reading or minting the keys that
+     * protect a subject's data is itself a thing an incident asks about.
+     */
+    const sealerFor = (c, tenantId, scopeId) => ({
+        seal: (items) => admin.sealSubjectPayloads(c.get('actor'), tenantId, scopeId, items),
+        open: (items) => admin.openSubjectPayloads(c.get('actor'), tenantId, scopeId, items),
+    });
     const verticalForScope = async (c, scope) => {
         if (!scope.vertical)
             return undefined;
@@ -744,7 +833,7 @@ export function createControlPlaneApi(options) {
             throw new ControlPlaneError(501, 'adopt-serving needs dispatch resolution for both ends');
         }
         const dump = await source.exportScope(scopeId);
-        const restored = await dest.restoreScope(tenantId, scopeId, dump);
+        const restored = await retryTransient(() => dest.restoreScope(tenantId, scopeId, dump));
         // Data landed — only now flip routing and move the version pointer.
         await admin.setScopeServingRef(actor, tenantId, scopeId, serving.ref);
         await admin.bindScopeVersion(actor, tenantId, scopeId, serving.versionId);
@@ -850,7 +939,7 @@ export function createControlPlaneApi(options) {
             throw new ControlPlaneError(501, 'rebind-vertical needs dispatch resolution for both ends');
         }
         const dump = await source.exportScope(scopeId);
-        const restored = await dest.restoreScope(tenantId, scopeId, dump);
+        const restored = await retryTransient(() => dest.restoreScope(tenantId, scopeId, dump));
         // Data landed on the target script — only now flip routing and cross the pointer.
         // `bindScopeVersion` rewrites `scopes.vertical` from the version row, audited. No
         // extra snapshot here (adopt-serving's precedent): the source script's copy is the
@@ -1050,7 +1139,7 @@ export function createControlPlaneApi(options) {
             forkedAt: new Date().toISOString(),
             expiresAt: opts.expiresAt,
         });
-        await vertical.snapshotScope({ sourceScopeId: scope.id, newScopeId: snapId });
+        await retryTransient(() => vertical.snapshotScope({ sourceScopeId: scope.id, newScopeId: snapId }));
         await admin.activateScope(actor, tenantId, snapId);
         // Bound to the SOURCE's current version: source and fork share a deployment, so
         // the fork resolves to the DO namespace its bytes actually live in.
@@ -1143,7 +1232,14 @@ export function createControlPlaneApi(options) {
         const dump = await admin.exportScope(c.get('actor'), tenantId, scope.id);
         const vertical = await verticalForScope(c, scope);
         const tables = vertical ? await vertical.exportScope(scope.id) : dump.tables;
-        return store.put({ vertical: scope.vertical, dump: { ...dump, tables } });
+        // Seal classified payloads per subject on the way in (#37). This copy is full-fidelity
+        // and the platform keeps it, which is precisely why an erasure could never reach it: a
+        // redaction fixes the live scope and does nothing to the object already in R2. Sealed
+        // here, destroying one subject's key reaches backwards into every copy already taken.
+        // Restores open it again (`restoreFromBackup`) — except for subjects shredded in the
+        // meantime, which is the mechanism working rather than the copy being damaged.
+        const sealed = await sealDump(tables, sealerFor(c, tenantId, scope.id));
+        return store.put({ vertical: scope.vertical, dump: { ...dump, tables: sealed } });
     };
     // The copies held for one scope — metadata only, so listing a reaped scope's backups
     // is cheap and hands out no bytes. Readable AFTER the reap (that is the point): the
@@ -1176,7 +1272,13 @@ export function createControlPlaneApi(options) {
         const dump = await options.scopeBackups.get({ tenantId, scopeId, capturedAt });
         if (!dump)
             return c.json({ error: `no backup for scope ${scopeId} at ${capturedAt}` }, 404);
-        return c.json(dump);
+        // Unseal on the way out (#37). The bytes in the store stay sealed — this is the
+        // authorized read opening them with the keys the platform holds. A subject shredded
+        // since the copy was taken opens to a null payload, which is exactly the erasure
+        // working: the ciphertext never left the object, and nothing turns it back into a
+        // person. A dump taken before sealing existed passes through untouched.
+        const opened = await openDump(dump.tables, sealerFor(c, tenantId, scopeId));
+        return c.json({ ...dump, tables: opened });
     });
     // Take a backup WITHOUT reaping — the standalone copy (a pre-migration checkpoint, an
     // export-to-keep). The reap route below takes its own; this is the same act made
@@ -1197,6 +1299,32 @@ export function createControlPlaneApi(options) {
             throw e;
         }
     });
+    // -- subject erasure (#37) --------------------------------------------------
+    // Erase one data subject from one scope: redact the spine payloads keyed to them, then
+    // destroy the key that seals every platform-retained copy of those payloads.
+    //
+    // Staff-only, and NOT in BUILDER_ROUTES. That placement is the shared-responsibility line
+    // drawn where hosting-and-certification.md §3 already draws it — "we provide extraction,
+    // they define scope". A builder forwards the request; the platform executes it and hands
+    // back a receipt they can answer their data subject with. Making it self-service would
+    // hand a vertical the ability to destroy evidence about a person on its own authority,
+    // which is a different decision than this one and belongs in its own issue.
+    //
+    // Idempotent by construction: a re-run redacts nothing (the payloads are already null),
+    // reports `keyDestroyed: false` (the key is already gone) and still returns
+    // `tombstoned: true`. Safe to retry, which matters because the audited half and the
+    // cryptographic half are two writes.
+    app.post('/tenants/:tenantId/scopes/:scopeId/subjects/:subjectId/shred', async (c) => {
+        const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
+        const scopeId = scopeIdSchema.parse(c.req.param('scopeId'));
+        // A ULID, like every other subject id the spine carries — parsed rather than trusted,
+        // so a wildcard or an injection attempt is a 400 and never reaches the UPDATE.
+        const subjectId = dataSubjectIdSchema.parse(c.req.param('subjectId'));
+        const scope = await admin.getScopeRecord(c.get('actor'), tenantId, scopeId);
+        if (!scope)
+            return c.json({ error: `unknown scope for tenant: (${tenantId}, ${scopeId})` }, 404);
+        return c.json(await admin.shredSubject(c.get('actor'), tenantId, scopeId, subjectId));
+    });
     // -- directory backups (#40) -----------------------------------------------
     // The platform's own disaster recovery, on the same store posture as the scope
     // backups above and deliberately on a different axis from them: a scope has ~30-day
@@ -1518,11 +1646,24 @@ export function createControlPlaneApi(options) {
             return c.json({ error: 'restore refused: the backup has no tables — not a scope dump' }, 422);
         }
         try {
-            await host.restoreScope(actor, tenantId, scopeId, dump);
+            // Belt and braces for #37: a caller who fetched the object bytes by some other route
+            // may POST a still-sealed dump, and a scope restored full of ciphertext is a silent
+            // data-loss bug. `openDump` skips cells that are not sealed, so a plaintext dump —
+            // the normal case, since the GET above already opened it — costs one pass and
+            // changes nothing.
+            //
+            // Opened against the dump's OWN provenance, not the destination. Subject keys are
+            // keyed by (scope, subject), so a copy of scope A landing in scope B — a backout onto
+            // a different scope, a world loaded sideways — must be opened with A's keys. Using
+            // the destination's would find no key, null every payload, and call it a restore.
+            const origin = { tenantId: tenantIdSchema.parse(dump.tenantId), scopeId: scopeIdSchema.parse(dump.scopeId) };
+            const tables = await openDump(dump.tables, sealerFor(c, origin.tenantId, origin.scopeId));
+            const landing = { ...dump, tables };
+            await host.restoreScope(actor, tenantId, scopeId, landing);
             const vertical = await verticalForScope(c, scope);
             if (vertical)
-                await vertical.restoreScope(tenantId, scopeId, dump.tables);
-            return c.json({ restored: scopeId, tables: dump.tables.length });
+                await retryTransient(() => vertical.restoreScope(tenantId, scopeId, tables));
+            return c.json({ restored: scopeId, tables: tables.length });
         }
         catch (e) {
             if (e instanceof ControlPlaneError) {
@@ -1714,10 +1855,6 @@ export function createControlPlaneApi(options) {
     // orphaned scope nobody can see, rather than a directory row promising a scope
     // that does not exist. `scopeStatus` has a `provisioning` state for expressing the
     // in-between properly, and it is still unused — see the PR.
-    // Rides out the binding-attach → script-settings propagation window (see the
-    // `provisionRetryDelaysMs` option); the same shape as the dashboard's #391
-    // configure retry. ~3s worst case, well inside a Worker request budget.
-    const PROVISION_RETRY_DELAYS_MS = [750, 2500];
     app.post('/verticals/:slug/instances', async (c) => {
         const slug = c.req.param('slug');
         // The install kill-switch: a blocked vertical takes no NEW instances, for anyone
@@ -1771,29 +1908,13 @@ export function createControlPlaneApi(options) {
             // #424 case 2: the binding attach above races Cloudflare script-settings
             // propagation, so the vertical's FIRST answer can be a transient 5xx that a
             // retry moments later heals. `provisionInstance` is idempotent at the far end
-            // (K-31), so ride the window out on a short backoff. Honest refusals (4xx, and
-            // 501 = not implemented) surface immediately — retrying a refusal only delays
-            // the real message.
-            const delays = options.provisionRetryDelaysMs ?? PROVISION_RETRY_DELAYS_MS;
-            let instance;
-            for (let attempt = 0;; attempt++) {
-                try {
-                    instance = await vertical.provisionInstance({
-                        ...input,
-                        entitlements,
-                        identityLinks,
-                        ...(tenantStores.length ? { tenantStores } : {}),
-                    });
-                    break;
-                }
-                catch (e) {
-                    const transient = e instanceof ControlPlaneError && (e.status === 0 || (e.status >= 500 && e.status !== 501));
-                    const delay = delays[attempt];
-                    if (!transient || delay === undefined)
-                        throw e;
-                    await new Promise((resolve) => setTimeout(resolve, delay));
-                }
-            }
+            // (K-31), so ride the window out on a short backoff.
+            const instance = await retryTransient(() => vertical.provisionInstance({
+                ...input,
+                entitlements,
+                identityLinks,
+                ...(tenantStores.length ? { tenantStores } : {}),
+            }));
             return c.json(instance, 201);
         }
         catch (e) {
@@ -1802,6 +1923,19 @@ export function createControlPlaneApi(options) {
             // must act on, and indistinguishable from "the vertical is broken" once it
             // has been flattened.
             if (e instanceof ControlPlaneError) {
+                // The retry above already rode out the transient window, so a 5xx here is a
+                // provision the platform could not complete — record it (#559). Refusals
+                // (4xx, and 501 = not implemented) stay unrecorded: the caller can read them.
+                if (e.status >= 500 && e.status !== 501) {
+                    recordFailure({
+                        actor: c.get('actor'),
+                        operation: 'install.provision',
+                        vertical: slug,
+                        tenantId: input.tenantId,
+                        status: e.status,
+                        message: e.message,
+                    });
+                }
                 return c.json({ error: e.message }, e.status);
             }
             throw e;
@@ -2396,7 +2530,20 @@ export function createControlPlaneApi(options) {
             const detail = e instanceof Error ? e.message : String(e);
             const upstream = upstreamStatusOf(e);
             console.error('deploy.upload.failed', { slug, deploymentRef, detail, upstream });
-            return upstream !== undefined && upstream >= 400 && upstream < 500
+            const rejected = upstream !== undefined && upstream >= 400 && upstream < 500;
+            // Both outcomes land an ops-failure row (#559): the 502 is the platform's to
+            // explain, and the 422 is what the builder-facing failure view (step 5) will
+            // list — a red push should be explainable from a durable record either way.
+            recordFailure({
+                actor: c.get('actor'),
+                operation: 'deploy.upload',
+                stage: 'wfp-upload',
+                vertical: slug,
+                tenantId: ownerTenant ?? null,
+                status: rejected ? 422 : 502,
+                message: detail,
+            });
+            return rejected
                 ? c.json({ error: 'deploy rejected', detail }, 422)
                 : c.json({ error: 'deploy upload failed', detail }, 502);
         }
@@ -2876,6 +3023,30 @@ export function createControlPlaneApi(options) {
         if (existing && stale)
             await reapPreview(c, existing);
         const previewId = scopeIdSchema.parse(ulid());
+        // The founding #559 case lands its durable row HERE, not in onError: the previews
+        // route answers a ControlPlaneError directly (its own catch, never the app-level
+        // recorder), and only this frame knows the preview's scopeId — the key that lets
+        // the console explain the stranded `provisioning` row this throw leaves behind.
+        const restoreOrRecord = async (fn) => {
+            try {
+                await retryTransient(fn);
+            }
+            catch (e) {
+                if (e instanceof ControlPlaneError && e.status >= 500 && e.status !== 501) {
+                    recordFailure({
+                        actor,
+                        operation: 'preview.create',
+                        stage: 'restore',
+                        tenantId,
+                        scopeId: previewId,
+                        vertical: slug,
+                        status: e.status,
+                        message: e.message,
+                    });
+                }
+                throw e;
+            }
+        };
         if (source) {
             // A fresh fork. Export from where the prod data lives TODAY. The canonical
             // `admin.exportScope` first — it writes the K-24 audit entry (and the co-located
@@ -2903,8 +3074,10 @@ export function createControlPlaneApi(options) {
                 expiresAt: expiresAt ?? undefined,
             });
             // Load the fork into the PR version's deployment (materializes the preview scope DO
-            // there; restore re-projects the vertical's roles from the dump's tuples).
-            await target.restoreScope(tenantId, previewId, tables);
+            // there; restore re-projects the vertical's roles from the dump's tuples). A one-shot
+            // DO storage blip heals on the in-request retry WITHOUT burning a CI attempt (which
+            // pushes a fresh version per try) — #559 (2).
+            await restoreOrRecord(() => target.restoreScope(tenantId, previewId, tables));
         }
         else {
             // A clean-room preview (#509 (b)): an EMPTY scope, no source to export. No `forkedFrom`
@@ -2923,7 +3096,7 @@ export function createControlPlaneApi(options) {
                 expiresAt: expiresAt ?? undefined,
             });
             if (target)
-                await target.restoreScope(tenantId, previewId, []);
+                await restoreOrRecord(() => target.restoreScope(tenantId, previewId, []));
         }
         await admin.activateScope(actor, tenantId, previewId);
         // Bind the PR version. A private vertical's push self-admitted, so this is accepted; a
@@ -3076,6 +3249,28 @@ export function createControlPlaneApi(options) {
         // page carries its own continuation and the console never assembles one.
         return c.json(pageOf(entries, filter.limit, (e) => e.id));
     });
+    // The recorded operational failures (#559) — the console's failures view, and the
+    // "what does this `reference = <id>` belong to" lookup. Newest first by default,
+    // unlike /admin-log: an operator asks "what broke lately". A builder reads only its
+    // OWN tenant's rows — the filter is forced, not trusted from the query (step 5: a
+    // red CI run is explainable from the dashboard without staff involvement).
+    app.get('/ops-failures', async (c) => {
+        const p = c.get('principal');
+        const filter = opsFailuresQuery.parse({
+            tenantId: p.kind === 'builder' ? p.tenantId : c.req.query('tenantId'),
+            scopeId: c.req.query('scopeId'),
+            vertical: c.req.query('vertical'),
+            operation: c.req.query('operation'),
+            reference: c.req.query('reference'),
+            since: c.req.query('since'),
+            until: c.req.query('until'),
+            limit: c.req.query('limit'),
+            cursor: c.req.query('cursor'),
+            order: c.req.query('order'),
+        });
+        const entries = await admin.listOpsFailures(c.get('actor'), filter);
+        return c.json(pageOf(entries, filter.limit, (e) => e.id));
+    });
     return app;
 }
 //# sourceMappingURL=api.js.map