@substrat-run/control-plane-api 0.158.0 → 0.160.0

package/dist/api.js

@@ -2,14 +2,15 @@ import { PREVIEW_CONNECTIONS_REFUSAL } from './connection-relay.js';
 import { connectionBindingOptions } from './connection-binding-options.js';
 import { resolveObservabilityWindow } from './observability-window.js';
 import { Hono } from 'hono';
-import { permissionKey, parseHostname, withLabel, RESERVED_LABEL_SEPARATOR, adminAction, lakeEventsInput, ownerTransferInput, memberInviteInput, memberRoleInput, ASSET_PART_PREFIX, assetHash, STORE_DRIVERS, STORE_RELEASE_MAX_BYTES, channelName, connectLinkId, connectionActivity, connectionActivitySource, connectionCredential, connectionCertificate, connectionFilter, connectionHealthState, connectionProvider, connectorDispatchKind, PROVISION_SIBLING_KIND, ARCHIVE_SCOPE_KIND, PROVISION_TENANT_KIND, SET_ENTITLEMENTS_KIND, MODEL_USAGE_KIND, SWEEP_RUNS_KIND, toConnectionHealthEntry, CONNECTION_EXPIRY_WARNING_DAYS, CONNECTION_STALE_DAYS, connectionProbe, createTenantInput, entitlementGrantInput, hostname as hostnameSchema, hostnameRegion, hostnameStatus, identityLink, errorCode, issueStatus, issueStatusInput, FINDINGS_LIMIT_DEFAULT, FINDINGS_LIMIT_MAX, findingCursorOf, findingKind, findingRuleInput, findingStatus, findingStatusInput, listPageQuery, matchesOutboundHost, pageOf, platformRequestFilter, principalId as principalIdSchema, impersonationStamp, promotionAcknowledgement, bindAcknowledgement, rebindAcknowledgement, provisionableJurisdiction, publishVersionInput, queryScopeInput, readScopeTableInput, entityHistoryInput, eventFacetInput, eventCauseInput, eventEffectsInput, invocationEventsInput, deadLettersInput, lifecycleFlowInput, operationSeriesCount, operationSeriesInput, delegatedReadParams, createOrgInput, roleKey as roleKeySchema, peerSwitch, verticalSlug, systemSwitch, DEFAULT_DENIAL_LIMIT, DENIAL_LIMIT_MAX, denialGroupBy, registerVerticalInput, platformScriptCollision, platformScriptRefusal, PLATFORM_SCRIPT_NAMES, serviceDimensions, sweepRunKind, sweepRunOutcome, scopeDump, dataSubjectId as dataSubjectIdSchema, scopeId as scopeIdSchema, scopeStatus, storageShape, surfaceName, tenantId as tenantIdSchema, tenantStatus, versionOrigin, z, PROBLEM_CONTENT_TYPE, toProblem, validationIssuesFrom, redrainEventsInput, importCursorMove, importCursorAcknowledgementMissing, REDRAIN_BATCH, migrationsOnTop, substratError, errorCodeOf, capabilityFilterQuery, capabilityId as capabilityIdSchema, PLATFORM_FEATURES_HEADER, PLATFORM_FEATURE_SCOPE_SWEEPER, SOCIAL_RELAY_VERTICALS, } from '@substrat-run/contracts';
+import { permissionKey, parseHostname, withLabel, RESERVED_LABEL_SEPARATOR, adminAction, lakeEventsInput, ownerTransferInput, memberInviteInput, memberRoleInput, ASSET_PART_PREFIX, assetHash, STORE_DRIVERS, STORE_RELEASE_MAX_BYTES, channelName, connectLinkId, connectionActivity, connectionActivitySource, connectionCredential, connectionCertificate, connectionFilter, connectionHealthState, connectionProvider, connectorDispatchKind, PROVISION_SIBLING_KIND, ARCHIVE_SCOPE_KIND, PROVISION_TENANT_KIND, SET_ENTITLEMENTS_KIND, MODEL_USAGE_KIND, SWEEP_RUNS_KIND, toConnectionHealthEntry, CONNECTION_EXPIRY_WARNING_DAYS, CONNECTION_STALE_DAYS, connectionProbe, createTenantInput, entitlementGrantInput, hostname as hostnameSchema, hostnameRegion, hostnameStatus, identityLink, errorCode, issueStatus, issueStatusInput, FINDINGS_LIMIT_DEFAULT, FINDINGS_LIMIT_MAX, findingCursorOf, findingKind, findingRuleInput, findingStatus, findingStatusInput, listPageQuery, matchesOutboundHost, pageOf, platformRequestFilter, principalId as principalIdSchema, impersonationStamp, promotionAcknowledgement, bindAcknowledgement, rebindAcknowledgement, provisionableJurisdiction, publishVersionInput, queryScopeInput, readScopeTableInput, entityHistoryInput, eventFacetInput, eventCauseInput, eventEffectsInput, invocationEventsInput, deadLettersInput, lifecycleFlowInput, operationSeriesCount, operationSeriesInput, delegatedReadParams, createOrgInput, roleKey as roleKeySchema, peerSwitch, verticalSlug, scopeText, systemSwitch, DEFAULT_DENIAL_LIMIT, DENIAL_LIMIT_MAX, denialGroupBy, registerVerticalInput, platformScriptCollision, platformScriptRefusal, PLATFORM_SCRIPT_NAMES, serviceDimensions, sweepRunKind, sweepRunOutcome, scopeDump, dataSubjectId as dataSubjectIdSchema, scopeId as scopeIdSchema, scopeStatus, requestRollupReading, meterDaysReading, verticalOwnersReading, utcDay, storageShape, surfaceName, tenantId as tenantIdSchema, tenantStatus, versionOrigin, z, PROBLEM_CONTENT_TYPE, toProblem, validationIssuesFrom, redrainEventsInput, importCursorMove, importCursorAcknowledgementMissing, REDRAIN_BATCH, migrationsOnTop, substratError, errorCodeOf, capabilityFilterQuery, capabilityId as capabilityIdSchema, PLATFORM_FEATURES_HEADER, PLATFORM_FEATURE_SCOPE_SWEEPER, SOCIAL_RELAY_VERTICALS, } from '@substrat-run/contracts';
+import { upstreamHandleIn, upstreamReferenceOf, withUpstreamReference } from '@substrat-run/contracts';
 import { SCOPE_COPY_LEASE_MS, copyRestoreFence, publishedMigrationDigest, replayStoreErasures } from '@substrat-run/kernel';
 import { attributeFailure } from './failure-attribution.js';
 import { auditedChange, withAuditedOutcomes } from './audited-change.js';
-import { BIND_EXPORT_BREAK_REFUSAL, CARRIED_AWAY_KEY, bindExportBreakRefusal, carriedAwayDump, dumpMetaValue, metaValueIn, crossVerticalHealth, exportBreakRefusal, isBindExportBreakRefusal, isExportBreakRefusal, isPrimaryScope, promoteReviewDigest, rebindReviewDigest, assertRebindReviewCurrent, migrationProgress, ulid, auditWarningOf, } from '@substrat-run/kernel';
+import { BIND_EXPORT_BREAK_REFUSAL, CARRIED_AWAY_KEY, bindExportBreakRefusal, carriedAwayDump, dumpMetaValue, metaValueIn, crossVerticalHealth, exportBreakRefusal, isBindExportBreakRefusal, isExportBreakRefusal, isExportBreakUnjudged, isPrimaryScope, promoteReviewDigest, rebindReviewDigest, assertRebindReviewCurrent, migrationProgress, ulid, auditWarningOf, REQUEST_ROLLUP_LIMIT_DEFAULT, REQUEST_ROLLUP_LIMIT_MAX, METER_DAYS_LIMIT_DEFAULT, METER_DAYS_LIMIT_MAX, releaseRolesAskScopes, releaseRoleSetRefusal, withheldOpsFailure, } from '@substrat-run/kernel';
 import { TENANT_HEADER, confinedTenant } from './auth.js';
 import { mintTenantToken } from './tenant-token.js';
-import { beginTransfer, connectionGrantsForScope, transferScope } from './vertical-client.js';
+import { beginTransfer, connectionGrantsForScope, transferScope, isScopeNotHeld, notHeldIsBenign } from './vertical-client.js';
 import { COPY_JOB_MAX_MS, copyJobAt, copyJobHeld, copyJobId, copyJobLive, copyTagId, versionIdOfOrder, versionOrderOf, } from './copy-job.js';
 import { runningVersionOf, runningVersionStoresOf, scopeStoreReachOf, scopeStoresOf, unrouteScope } from './platform-scope-stores.js';
 import { STORE_REBUILD_REQUEST_SUBREQUESTS, runStoreFleetRebuild, storeRebuildBudget, storeRebuildStatus } from './store-rebuild.js';
@@ -61,11 +62,12 @@ function refuseStatus(c, status, detail, carried) {
 }
 import { backfillDeclaredStores, blobStoreBindings, collectBlobStoreHandles, collectTenantStoreHandles, missingStoresForTenant, tenantStoreBindings, } from './tenant-stores.js';
 import { mintPushToken, pushActorFor } from './push-token.js';
-import { TENANT_SERIES_SCOPE_CAP, REQUEST_FACET_KEYS } from './observability.js';
+import { TENANT_SERIES_SCOPE_CAP, REQUEST_FACET_KEYS, LOGS_INCOMPLETE_HEADER } from './observability.js';
 import { parseTenantRequestQuery, parseTenantWindowQuery, refineTenantWindow, scriptFamiliesOfScopes } from './tenant-request-query.js';
 import { namespacesForScript } from './do-namespaces.js';
 import { backupDirectoryIfDue } from './directory-backup.js';
 import { deployedDoClasses, platformSweeperDecision } from './platform-entry.js';
+import { wireUploadFor } from './wire-upload.js';
 import { STORAGE_PAGE_DEFAULT, STORAGE_PAGE_MAX, readStoragePage } from './storage-meter.js';
 import { queryLakeEvents } from './lake-gateway.js';
 import { LakeQueryError } from './r2-sql-engine.js';
@@ -166,6 +168,46 @@ function withholdIfPredatesRule(input, answer) {
         truncated: false,
     };
 }
+/** #2446: the most scopes one promote reads the role state of (`roleStateScopeCap`). */
+export const ROLE_STATE_SCOPE_CAP = 500;
+/** #2446: how many of those reads are in flight at once. */
+const ROLE_STATE_CONCURRENCY = 8;
+/** #2446: what a promote answers about the scopes it could not ask for their roles. */
+export const ROLE_STATE_UNREAD_NOTE = "their deployment predates the role-state read, so this version's roles were not judged against them before the " +
+    'promote; their reconcile still refuses a role set it cannot define';
+/**
+ * #2446: the promote's refusal for a role-state judgement, or null when it passes, as `pin` may
+ * read it. A confined caller sees only its own tenant's scopes by id and every other tenant as a
+ * count, and never what a deployment answered (that is logged where it was read); staff see all of
+ * it. A refusal found (409) is answered ahead of a read that failed (502), since it stands either way.
+ */
+function roleStateAnswer(roles, pin) {
+    if (roles.refused.length > 0) {
+        const { visible, otherTenants } = narrowToCaller(roles.refused, pin);
+        const why = visible[0] ? `: ${visible[0].refusal}` : ', none of them yours';
+        return {
+            status: 409,
+            message: `promotion refused: this version's roles cannot be defined in ${roles.refused.length} of the scopes it would move, ` +
+                `so their reconcile would be refused there${why}. Nothing was promoted`,
+            extra: { roleRefusals: { affected: visible, ...(otherTenants ? { otherTenants } : {}) } },
+        };
+    }
+    if (roles.failed.length > 0) {
+        const { visible, otherTenants } = narrowToCaller(roles.failed, pin);
+        return {
+            status: 502,
+            message: `the roles of ${roles.failed.length} of the scopes this version would move could not be read, and its roles are ` +
+                `judged against them before it moves them. Nothing was promoted; retry the promote`,
+            extra: {
+                roleStateUnreadable: {
+                    affected: pin === null ? visible : visible.map(({ tenantId, scopeId }) => ({ tenantId, scopeId })),
+                    ...(otherTenants ? { otherTenants } : {}),
+                },
+            },
+        };
+    }
+    return null;
+}
 /**
  * A fleet-wide report, narrowed to what a caller may see: a confined caller gets its own
  * tenant's rows and a COUNT of the other tenants, never their ids (K-3's forced filter, held on
@@ -190,9 +232,11 @@ const provisionScopeBody = z.object({
     tenantId: tenantIdSchema,
     scopeId: scopeIdSchema,
     slug: z.string().optional(),
-    kind: z.string().optional(),
-    name: z.string().optional(),
-    vertical: z.string().nullable().optional(),
+    kind: scopeText.optional(),
+    name: scopeText.optional(),
+    // The registry's grammar, as `resolveScopeRecord` holds it too: null or absent is "no vertical",
+    // and an empty string is refused here rather than stored.
+    vertical: verticalSlug.nullable().optional(),
     storageShape: storageShape.optional(),
     // The gate: only `global` is accepted until `eu`/`us` enforcement exists (K-32).
     // A request naming a jurisdiction we cannot yet honour is refused at the Zod
@@ -614,9 +658,6 @@ const modelUsageSummaryQuery = z.object({
     until: z.string().optional(),
 });
 export const DEFAULT_MODEL_MARGIN_PERCENT = 20;
-/** The upstream provider's trace handle, when a failure message carries one —
- *  Cloudflare's `internal error; reference = <id>` shape (#559). */
-const UPSTREAM_REFERENCE = /\breference\s*=\s*([a-z0-9]+)/i;
 const TENANT_ROUTES = [
     { method: 'GET', re: /\/verticals\/[^/]+\/email-sender$/, pin: 'owner' },
     { method: 'POST', re: /\/verticals\/[^/]+\/email-sender$/, pin: 'owner' },
@@ -1037,16 +1078,34 @@ export function createControlPlaneApi(options) {
             const cause = c.get('failure');
             const marked = typeof cause === 'object' && cause !== null && recordedCauses.has(cause);
             if (!marked) {
-                const { vertical, tenantId } = await failureSubject(c);
-                recordFailure({
-                    actor,
-                    operation: `${c.req.method} ${c.req.routePath}`,
-                    vertical,
-                    tenantId,
-                    scopeId: c.req.param('scopeId') ?? null,
-                    status,
-                    message: cause !== undefined ? messageOfThrow(cause) : await answeredMessage(c.res),
-                }, cause === undefined ? NO_CAUSE : cause);
+                const { vertical, tenantId } = c.get('failureSubjectOf') ?? (await failureSubject(c));
+                const scopeId = c.req.param('scopeId') ?? null;
+                const base = { actor, operation: `${c.req.method} ${c.req.routePath}`, vertical, scopeId, status };
+                const answered = await answeredMessage(c.res);
+                const raw = cause !== undefined ? messageOfThrow(cause) : answered;
+                if (tenantId !== null && raw !== answered) {
+                    // #2540: a row attributed to a tenant is read back by that tenant, so it says what this
+                    // request was answered, whoever made it. The throw's own text is unreviewed: it goes on a
+                    // row with no tenant, which only staff read, and to the log line, under the same reference.
+                    let reference = c.get('failureReference');
+                    if (!reference) {
+                        reference = ulid();
+                        console.error('control-plane.unhandled', { method: c.req.method, path: c.req.path, reference, tenantId, scopeId, detail: raw });
+                    }
+                    recordWithheld({ ...base, tenantId, reference }, raw, answered, cause);
+                }
+                else {
+                    recordFailure({
+                        ...base,
+                        tenantId,
+                        message: raw,
+                        // The one the log line carries. `recordFailure` keeps an upstream handle in the text
+                        // off a tenant's row, on a staff row beside it, and takes it on a row with no tenant.
+                        reference: tenantId
+                            ? (c.get('failureReference') ?? null)
+                            : (upstreamHandleIn(raw) ?? c.get('failureReference') ?? null),
+                    }, cause === undefined ? NO_CAUSE : cause);
+                }
             }
         }
         if (inflightRecords.size) {
@@ -1282,6 +1341,13 @@ export function createControlPlaneApi(options) {
     // `undefined` (`throw undefined`, a bare rejection), and that throw deserves an
     // `unknown` attribution, not a skipped one. Only true omission skips.
     const NO_CAUSE = Symbol('no-cause');
+    /**
+     * #2540: the one kind of tenant row whose reference is an upstream trace handle. A failed upload
+     * of a builder's own bundle answers the builder the upload's text verbatim, handle included, and
+     * the CLI prints that handle for its lookup (#559), so the row keeps it as before. Every other
+     * row filed under a tenant carries the platform's own reference. Named, never inferred from text.
+     */
+    const UPSTREAM_REFERENCE_KEPT = new Set(['deploy.upload']);
     // The throws a step already recorded, so the 5xx backstop below does not record them twice.
     const recordedCauses = new WeakSet();
     // The writes still in flight. A Worker may end once its response is sent, and a write nobody
@@ -1295,17 +1361,80 @@ export function createControlPlaneApi(options) {
         // regex a message. A caller with no throw in hand leaves the columns null,
         // which reads as "nobody classified this" rather than a guess nobody made.
         const attributed = cause === NO_CAUSE ? undefined : attributeFailure(cause);
+        // #2540: a row filed under a tenant is read back by it, so its reference is always the
+        // platform's own. When its text carries an upstream trace handle (#559), that handle is kept
+        // on a staff row beside it (`detail`, under the same reference), where staff's search finds it.
+        // A row with no tenant is staff's: its text is written as a staff row's always is
+        // (`withUpstreamReference`: any marker spelling neutralised, a genuine handle's marker first),
+        // and it takes that handle as its reference, as before.
+        const handle = upstreamHandleIn(entry.message);
+        const staffMessage = entry.tenantId ? entry.message : withUpstreamReference(entry.message);
+        const shaped = !entry.tenantId
+            ? { ...entry, message: staffMessage, reference: entry.reference ?? upstreamReferenceOf(staffMessage) }
+            : UPSTREAM_REFERENCE_KEPT.has(entry.operation)
+                ? { ...entry, reference: entry.reference ?? handle }
+                : entry.detail == null && handle
+                    ? {
+                        ...entry,
+                        reference: entry.reference && entry.reference !== handle ? entry.reference : ulid(),
+                        detail: withUpstreamReference(entry.message),
+                    }
+                    : { ...entry, reference: entry.reference ?? null };
         const write = host.admin
             .recordOpsFailure({
-            ...entry,
+            ...shaped,
             origin: entry.origin ?? attributed?.origin ?? null,
             code: entry.code ?? attributed?.code ?? null,
-            reference: entry.reference ?? UPSTREAM_REFERENCE.exec(entry.message)?.[1] ?? null,
+            reference: shaped.reference ?? null,
         })
             .catch(() => undefined)
             .finally(() => inflightRecords.delete(write));
         inflightRecords.add(write);
     };
+    /** #2540: whether this request's principal is staff. Anything else, or no principal yet, is confined. */
+    const staffCaller = (c) => c.get('principal')?.kind === 'staff';
+    /**
+     * #2540: one failure whose text is not ours to hand a tenant. Filed under a tenant, it is one
+     * write with a `detail`: the host puts what was answered on the tenant's row, which the tenant
+     * reads back, and the text on a row with no tenant under the same reference, which only staff
+     * read, and counts the failure once. With no tenant, the text is the row's message.
+     */
+    const recordWithheld = (entry, raw, answered, cause) => {
+        // The pair is the one the host-level writers file too (`withheldOpsFailure`, #2554): an upstream
+        // trace handle the text carries is named on staff's row only, first, so either id finds the pair.
+        // With no tenant, `recordFailure` writes the text as a staff row's.
+        const { tenantId } = entry;
+        recordFailure(tenantId ? withheldOpsFailure({ ...entry, tenantId }, raw, () => answered, entry.reference) : { ...entry, tenantId: null, message: raw }, cause);
+    };
+    /**
+     * #2540: a failure whose text is not ours to hand a confined caller: what an upstream or a store
+     * answered, often while the request reached scopes that are not the caller's (a promote's serve
+     * and store backfill run across every install). The text goes to the `control-plane.unhandled`
+     * log line and to an ops-failure row with no tenant on it, which only staff read, both under one
+     * reference (`recordWithheld`). A `tenantId` the failure belongs to gets a row too, saying only
+     * `answered(reference)`. The answer carries the reference, and the text too when the caller is
+     * staff. Who the caller is is decided by the request's principal alone (`staffCaller`), never by a
+     * scope or tenant the request names. The cause is marked recorded, so the 5xx backstop writes no
+     * second row for it; a route that answers the failure as a 5xx of its own also sets `failureRecorded`.
+     */
+    const withheldFailure = (c, entry, cause, answered) => {
+        const reference = ulid();
+        console.error('control-plane.unhandled', {
+            method: c.req.method,
+            path: c.req.path,
+            reference,
+            operation: entry.operation,
+            stage: entry.stage ?? null,
+            vertical: entry.vertical ?? null,
+            version: entry.version ?? null,
+            tenantId: entry.tenantId ?? null,
+            scopeId: entry.scopeId ?? null,
+            detail: entry.message,
+        });
+        const { message, ...rest } = entry;
+        recordWithheld({ ...rest, actor: c.get('actor'), tenantId: answered ? (entry.tenantId ?? null) : null, reference }, message, answered?.(reference) ?? '', cause);
+        return staffCaller(c) ? { reference, detail: message } : { reference };
+    };
     /**
      * Whose failure the backstop is recording, so the dashboard's tenant-pinned failure view lists
      * it. The path's tenant when it names one; otherwise the credential's own (a CI push token, a
@@ -1369,7 +1498,7 @@ export function createControlPlaneApi(options) {
     // One error boundary for every route: adapters throw plain Errors, and each
     // one is a fail-closed refusal that must reach the caller as a status, not a
     // stack trace.
-    app.onError((err, c) => {
+    app.onError(async (err, c) => {
         // A parse failure goes through the same builder as everything else (#113 phase 4):
         // `toProblem` maps zod's `issues` onto the declared `errors: [{ path, message }]`
         // extension, which is the shape the model documents and a client can act on. The raw
@@ -1382,6 +1511,31 @@ export function createControlPlaneApi(options) {
         if (err instanceof CarrySourceChanged) {
             return refuse(c, 'conflict', err.message, { carrySourceChanged: { scopeId: err.scopeId, from: err.from, to: err.to, attempts: err.attempts } });
         }
+        // Whose failure this is, as the backstop files it, so a log line names the same tenant as its row.
+        // Read only for an answer that is logged or filed: a 4xx refusal pays nothing.
+        const subjectOf = async () => {
+            const subject = await failureSubject(c).catch(() => ({ vertical: null, tenantId: null }));
+            c.set('failureSubjectOf', subject);
+            return subject;
+        };
+        // #2540: a refusal to judge what a promote or bind breaks names what could not be read, which may
+        // be another tenant's app, and that read's own error. Every door that reaches it answers here.
+        if (isExportBreakUnjudged(err)) {
+            const subject = await subjectOf();
+            const sentence = (reference) => `cannot say whom this change breaks: what an installed app imports could not be read, so nothing was changed. Retry (reference ${reference})`;
+            const withheld = withheldFailure(c, {
+                operation: `${c.req.method} ${c.req.routePath}`,
+                stage: 'export-breaks',
+                vertical: subject.vertical,
+                tenantId: subject.tenantId,
+                scopeId: c.req.param('scopeId') ?? null,
+                status: 503,
+                code: 'unavailable',
+                message: messageOfThrow(err),
+            }, err, sentence);
+            c.set('failureRecorded', true);
+            return refuse(c, 'unavailable', sentence(withheld.reference), withheld);
+        }
         const { status, body } = mapError(err);
         // The 5xx backstop (registered first, above) records it, with this throw as the cause.
         c.set('failure', err);
@@ -1394,9 +1548,16 @@ export function createControlPlaneApi(options) {
         // (the 503 for a plane with no seal key, #603) passes through here too: its body already
         // says what is wrong, and the tail line is what ties it to the request that hit it.
         if (status >= 500) {
+            const subject = await subjectOf();
+            const reference = ulid();
+            c.set('failureReference', reference);
             console.error('control-plane.unhandled', {
                 method: c.req.method,
                 path: c.req.path,
+                reference,
+                tenantId: subject.tenantId,
+                vertical: subject.vertical,
+                scopeId: c.req.param('scopeId') ?? null,
                 detail: err instanceof Error ? err.message : String(err),
                 stack: err instanceof Error ? err.stack : undefined,
             });
@@ -1531,11 +1692,13 @@ export function createControlPlaneApi(options) {
         return c.json(await c.var.admin.listEntitlements(c.get('actor'), tenantId));
     });
     // -- the meters (§5, #38) --------------------------------------------------
-    // Meters 1 and 2, read fleet-wide or narrowed with `?tenantId=`. Meters 3 and 4 have
-    // no route because they have no number: the outbox is per-scope-database with no
-    // cross-tenant fan-in and reads emit nothing, so anything served here would be
-    // invented. Staff-only (absent from BUILDER_ROUTES) — a fleet-wide revenue aggregate
-    // is the platform's own book, not a builder's view of their installs.
+    // Meters 1 and 2, read fleet-wide or narrowed with `?tenantId=`. Meter 3's parts are
+    // read at their own routes, not here (`/meters/storage`, `/model-usage`, and the estimated
+    // daily request rollup at `/meters/requests`); lake bytes have no reader yet, and exact API counts are not computable, since
+    // reads emit nothing on the spine. Meter 4 has no route because it has no number
+    // (it needs the cross-tenant order flow). See control-plane.md §5. Staff-only (absent
+    // from BUILDER_ROUTES) — a fleet-wide revenue aggregate is the platform's own book, not
+    // a builder's view of their installs.
     //
     // GET, and nothing is stored: D-30 is meter, do not bill, so a reading is recomputed
     // per call and stamped with the instant it was taken.
@@ -1580,14 +1743,123 @@ export function createControlPlaneApi(options) {
                 if (!scope.vertical)
                     return c.var.admin.scopeDatabaseSize(actor, q.tenantId, scope.id);
                 const vertical = await verticalForScope(c, scope);
-                if (vertical)
-                    return vertical.databaseSize(scope.id);
+                if (vertical) {
+                    try {
+                        return await vertical.databaseSize(scope.id);
+                    }
+                    catch (err) {
+                        // #2541: a deployment that holds nothing for a scope the directory has reaped or archived
+                        // since the listing (or does not have) reads 0; any other status holds a store and stays a
+                        // failure here, as does a directory read that fails.
+                        if (!isScopeNotHeld(err, scope.id))
+                            throw err;
+                        if (!(await notHeldIsBenign(() => c.var.admin.getScopeRecord(actor, q.tenantId, scope.id))))
+                            throw err;
+                        return 0;
+                    }
+                }
                 if (delegatesToVerticals)
                     throw new Error(`no deployment resolves for vertical ${scope.vertical}`);
                 return c.var.admin.scopeDatabaseSize(actor, q.tenantId, scope.id);
             },
         }));
     });
+    // The daily request rollup (#2501): the stored per-(day, tenant, vertical, scope) request
+    // counts the platform sweep reads from the router's Analytics Engine datapoints before they
+    // age out, and the rollup's record of each day in the window. Staff-only like `/meters`. Every
+    // count is an estimate (Analytics Engine samples), and the answer says so twice: `estimated`
+    // on the reading and on every row. A directory read; nothing is queried live. `days` is what
+    // tells a day without traffic from a day not yet rolled up.
+    //
+    // A page is at most half the directory's own read bound, so the one row past the page that
+    // says the window holds more always fits under it.
+    const REQUEST_ROLLUP_PAGE_MAX = REQUEST_ROLLUP_LIMIT_MAX / 2;
+    // Unless a window is named, the 31 days ending at `until`, both ends included.
+    const windowSince = (until) => new Date(Date.parse(`${until}T00:00:00Z`) - 30 * 86_400_000).toISOString().slice(0, 10);
+    app.get('/meters/requests', async (c) => {
+        const admin = c.var.admin;
+        if (!admin.listRequestDays || !admin.listRequestRollupDays) {
+            return refuseStatus(c, 501, 'this control plane keeps no request rollup');
+        }
+        const today = new Date().toISOString().slice(0, 10);
+        const q = z
+            .object({
+            tenantId: tenantIdSchema.optional(),
+            scopeId: scopeIdSchema.optional(),
+            vertical: z.string().min(1).optional(),
+            since: utcDay.optional(),
+            until: utcDay.default(today),
+            limit: z.coerce.number().int().min(1).max(REQUEST_ROLLUP_PAGE_MAX).default(REQUEST_ROLLUP_LIMIT_DEFAULT),
+        })
+            .parse({
+            tenantId: c.req.query('tenantId') || undefined,
+            scopeId: c.req.query('scopeId') || undefined,
+            vertical: c.req.query('vertical') || undefined,
+            since: c.req.query('since') || undefined,
+            until: c.req.query('until') || undefined,
+            limit: c.req.query('limit') || undefined,
+        });
+        const since = q.since ?? windowSince(q.until);
+        if (since > q.until)
+            return refuse(c, 'validation_failed', '`since` is after `until`');
+        const actor = c.get('actor');
+        // One row past the page says whether the window holds more than it returned.
+        const rows = await admin.listRequestDays(actor, {
+            ...(q.tenantId ? { tenantId: q.tenantId } : {}),
+            ...(q.scopeId ? { scopeId: q.scopeId } : {}),
+            ...(q.vertical ? { vertical: q.vertical } : {}),
+            since,
+            until: q.until,
+            limit: q.limit + 1,
+        });
+        const days = await admin.listRequestRollupDays(actor, { since, until: q.until });
+        return c.json(requestRollupReading.parse({
+            estimated: true,
+            since,
+            until: q.until,
+            rows: rows.slice(0, q.limit),
+            truncated: rows.length > q.limit,
+            days,
+        }));
+    });
+    // The daily meter snapshot (#2501): meters 1 and 2 as the first scheduled pass of each UTC day
+    // found them, one row per (day, tenant), and the record of each day in the window. Staff-only
+    // like `/meters`. A directory read; nothing is recomputed, so a past day reads as it was then.
+    // `days` is what tells a day no pass took from a day whose snapshot met no tenant. The page bound
+    // leaves room under the directory's own for the one row past it.
+    const METER_DAYS_PAGE_MAX = METER_DAYS_LIMIT_MAX / 2;
+    app.get('/meters/days', async (c) => {
+        const admin = c.var.admin;
+        if (!admin.listMeterDays || !admin.listMeterDayTakes) {
+            return refuseStatus(c, 501, 'this control plane keeps no meter snapshots');
+        }
+        const today = new Date().toISOString().slice(0, 10);
+        const q = z
+            .object({
+            tenantId: tenantIdSchema.optional(),
+            since: utcDay.optional(),
+            until: utcDay.default(today),
+            limit: z.coerce.number().int().min(1).max(METER_DAYS_PAGE_MAX).default(METER_DAYS_LIMIT_DEFAULT),
+        })
+            .parse({
+            tenantId: c.req.query('tenantId') || undefined,
+            since: c.req.query('since') || undefined,
+            until: c.req.query('until') || undefined,
+            limit: c.req.query('limit') || undefined,
+        });
+        const since = q.since ?? windowSince(q.until);
+        if (since > q.until)
+            return refuse(c, 'validation_failed', '`since` is after `until`');
+        const actor = c.get('actor');
+        const rows = await admin.listMeterDays(actor, {
+            ...(q.tenantId ? { tenantId: q.tenantId } : {}),
+            since,
+            until: q.until,
+            limit: q.limit + 1,
+        });
+        const days = await admin.listMeterDayTakes(actor, { since, until: q.until });
+        return c.json(meterDaysReading.parse({ since, until: q.until, rows: rows.slice(0, q.limit), truncated: rows.length > q.limit, days }));
+    });
     // -- per-tenant stores (#301, #473) ----------------------------------------
     // The two ledgers as INVENTORY — what `listTenantStores`/`listBlobStores` were always
     // meant to answer for a staff surface: which database and which bucket hold this
@@ -3707,15 +3979,18 @@ export function createControlPlaneApi(options) {
         // ready-gate and a fresh install without it is simply broken, so a failed mint must
         // fail the provision rather than hand back a half-built scope.
         const storeErrors = [];
+        // #2540: the same failures as the scope's tenant may read them: the sentences, never the text.
+        const storeSentences = [];
         const mintBestEffort = async (stage, mint) => {
             try {
                 return await mint();
             }
             catch (e) {
-                const message = e instanceof Error ? e.message : String(e);
-                console.error('stores.provision.failed', { tenantId, scopeId, vertical: scope.vertical, stage, detail: message });
-                recordFailure({
-                    actor,
+                // #2540: what the store answered is staff's. A confined caller (a builder may repair any
+                // install of its own vertical, whoever's) reads that the mint failed and the reference, and
+                // the scope's tenant reads the same on its failure row, whoever made the request.
+                const sentence = (reference) => `a declared store could not be minted (${stage}, reference ${reference})`;
+                const withheld = withheldFailure(c, {
                     operation: 'scope.provision.stores',
                     stage,
                     tenantId,
@@ -3723,9 +3998,10 @@ export function createControlPlaneApi(options) {
                     vertical: scope.vertical,
                     version: scope.verticalVersionId ?? null,
                     status: e instanceof ControlPlaneError ? e.status : null,
-                    message,
-                }, e);
-                storeErrors.push(message);
+                    message: messageOfThrow(e),
+                }, e, sentence);
+                storeErrors.push(withheld.detail ? `${withheld.detail} (reference ${withheld.reference})` : sentence(withheld.reference));
+                storeSentences.push(sentence(withheld.reference));
                 return undefined;
             }
         };
@@ -3803,8 +4079,60 @@ export function createControlPlaneApi(options) {
                 // Both halves failed. The store fault is usually the CAUSE (an unmigrated or
                 // unbound store is exactly what makes a vertical refuse its own reconcile), so
                 // it must not be the half that gets dropped.
-                const detail = storeErrors.length ? `${e.message} (stores: ${storeErrors.join('; ')})` : e.message;
-                return refuseStatus(c, failedWith(c, e), detail);
+                const stores = storeErrors.length ? ` (stores: ${storeErrors.join('; ')})` : '';
+                // #2540: the vertical's answer about a scope is its tenant's own text. Staff, and a confined
+                // caller repairing an install of its own tenant, read it as before.
+                if (staffCaller(c) || confinedTenant(c.get('principal')) === tenantId) {
+                    const answer = `${e.message}${stores}`;
+                    // Staff are answered the store text too, which the tenant's row must not say: a 5xx the
+                    // backstop would file is filed here as a pair, the tenant's row with the sentences only.
+                    const tenantStores = storeSentences.length ? ` (stores: ${storeSentences.join('; ')})` : '';
+                    if (e.status >= 500 && e.status !== 501 && tenantStores !== stores) {
+                        const reference = ulid();
+                        recordWithheld({
+                            actor,
+                            operation: `${c.req.method} ${c.req.routePath}`,
+                            tenantId,
+                            scopeId,
+                            vertical: scope.vertical,
+                            version: scope.verticalVersionId ?? null,
+                            status: e.status,
+                            reference,
+                        }, answer, `${e.message}${tenantStores}`, e);
+                        c.set('failureRecorded', true);
+                        return refuseStatus(c, failedWith(c, e), answer, { reference });
+                    }
+                    return refuseStatus(c, failedWith(c, e), answer);
+                }
+                // A builder repairing another tenant's install of its vertical reads the status, the store
+                // sentences and a reference; what that deployment answered is staff's.
+                const sentence = (reference) => `the reconcile of this scope failed (${e.status})${stores} (reference ${reference})`;
+                if (e.status < 500) {
+                    // A refusal, which the 5xx backstop leaves unrecorded for staff too: logged, no row.
+                    const reference = ulid();
+                    console.warn('control-plane.withheld', {
+                        method: c.req.method,
+                        path: c.req.path,
+                        reference,
+                        operation: 'scope.provision.reconcile',
+                        tenantId,
+                        scopeId,
+                        status: e.status,
+                        detail: e.message,
+                    });
+                    return refuseStatus(c, e.status, sentence(reference), { reference });
+                }
+                const withheld = withheldFailure(c, {
+                    operation: 'scope.provision.reconcile',
+                    tenantId,
+                    scopeId,
+                    vertical: scope.vertical,
+                    version: scope.verticalVersionId ?? null,
+                    status: e.status,
+                    message: e.message,
+                }, e, sentence);
+                c.set('failureRecorded', true);
+                return refuseStatus(c, e.status, sentence(withheld.reference), withheld);
             }
             throw e;
         }
@@ -4507,6 +4835,7 @@ export function createControlPlaneApi(options) {
             scopeId: snapId,
             kind: opts.kind ?? 'archive',
             vertical: scope.vertical,
+            verticalSource: 'copy',
             jurisdiction: scope.jurisdiction,
             forkedFrom: scope.id,
             forkedAt: new Date().toISOString(),
@@ -6901,6 +7230,20 @@ export function createControlPlaneApi(options) {
         await c.var.admin.deleteVertical(c.get('actor'), slug);
         return c.json({ slug, deleted: true });
     });
+    // Who owned a vertical slug when (#2501): every period, opened when the slug was registered and
+    // closed when it was deleted, so a past period bills the builder who owned the vertical then.
+    // Staff-only, like `/meters`: absent from BUILDER_ROUTES and from the tenant credential's pins.
+    // Served for a deleted slug too, which is the point; a slug never registered has no period.
+    // The FULL slug, read as given: unlike the routes behind `resolveVerticalId`, the tenant pin is
+    // ignored, since it resolves a bare name through the registry row a deleted slug no longer has.
+    // A bare name is read as a slug of its own; one never registered answers with no periods.
+    app.get('/verticals/:slug/owners', async (c) => {
+        const admin = c.var.admin;
+        if (!admin.listVerticalOwners)
+            return refuseStatus(c, 501, 'this control plane keeps no ownership history');
+        const slug = verticalSlug.parse(c.req.param('slug'));
+        return c.json(verticalOwnersReading.parse({ slug, owners: await admin.listVerticalOwners(c.get('actor'), slug) }));
+    });
     app.get('/verticals/:slug/channels', async (c) => {
         const p = c.get('principal');
         const slug = await resolveVerticalId(c, c.req.param('slug'));
@@ -6987,6 +7330,40 @@ export function createControlPlaneApi(options) {
                 await host.admin.releaseStoreDriverHold(actor, { driver, holder }).catch(() => undefined);
         }
     };
+    /**
+     * Upload `bundle` to the script `ref`, a deployment of `slug`, with what the control plane's
+     * settings give it for the envelope's verifier (RFC wire-signing step 2, follow-on 3:
+     * `wireUploadFor`) added to its bindings. Both uploads of a vertical's script go through here, so
+     * no upload path can leave the lists out. A refused setting or refused lists give nothing, and are
+     * logged and recorded as an operational failure by reason, never with a key. Answers what the
+     * upload gave, for its record (`recordWireUpload`).
+     */
+    const deployVerticalScript = async (deploy, actor, slug, ref, bundle, inPlace) => {
+        const wire = await wireUploadFor(options.wireUpload, { vertical: slug, deploymentRef: ref, bundle });
+        if (wire.refusal !== null) {
+            console.error(JSON.stringify({ substrat: 'wire-upload', vertical: slug, deploymentRef: ref, withheld: wire.record.withheld, reason: wire.refusal }));
+            recordFailure({ actor, operation: 'deploy.wire-upload', stage: wire.record.withheld, vertical: slug, message: `no key lists given to ${ref}: ${wire.refusal}` });
+        }
+        await deploy(ref, { ...bundle, bindings: [...bundle.bindings, ...wire.bindings] }, inPlace);
+        return wire;
+    };
+    /**
+     * The capability record of an upload that succeeded (RFC wire-signing §7, Q8). A write that fails
+     * never fails the upload, which is done: it is logged and recorded as an operational failure, and
+     * the readiness report reads a deployment with no record as unknown, never as ready.
+     */
+    const recordWireUpload = async (admin, actor, versionId, wire) => {
+        if (!admin.recordWireAuth)
+            return;
+        const { vertical, deploymentRef, record } = wire;
+        try {
+            await admin.recordWireAuth(actor, { deploymentRef, verticalSlug: vertical, versionId, record, recordedAt: new Date().toISOString() });
+        }
+        catch (error) {
+            console.error(JSON.stringify({ substrat: 'wire-upload-record-failed', vertical, deploymentRef, error: error instanceof Error ? error.name : 'error' }));
+            recordFailure({ actor, operation: 'deploy.wire-record', vertical, version: versionId, message: `the capability record of ${deploymentRef} was not written` }, error);
+        }
+    };
     /**
      * Serve a version IN PLACE (#286): upload its bundle onto the vertical's ONE stable
      * serving script, where every scope's data DO lives — this is the step that makes a
@@ -7041,7 +7418,7 @@ export function createControlPlaneApi(options) {
             : { type: 'r2_bucket', name: b.name, bucket_name: b.bucketName });
         // #2210: the store scripts first, so the vertical is never served before what it reaches.
         await ensureScopeStoresOf(actor, 'serve', slug, manifest.scopeStores, async () => (await ownerOf(actor, slug)) ?? null);
-        await options.deployVertical(ref, {
+        const wire = await deployVerticalScript(options.deployVertical, actor, slug, ref, {
             entry: manifest.entry,
             compatibilityDate: manifest.compatibilityDate,
             compatibilityFlags: manifest.compatibilityFlags,
@@ -7095,6 +7472,7 @@ export function createControlPlaneApi(options) {
         }, serving
             ? { priorDoClasses: serving.doClasses, priorMigrationTag: serving.migrationTag }
             : undefined);
+        await recordWireUpload(admin, actor, versionId, wire);
         const addedClasses = serving ? doClasses.some((cls) => !serving.doClasses.includes(cls)) : false;
         await admin.setVerticalServing(actor, slug, {
             ref,
@@ -7154,12 +7532,98 @@ export function createControlPlaneApi(options) {
             return { minted };
         }
         catch (e) {
-            const message = e instanceof Error ? e.message : String(e);
-            console.error('stores.backfill.failed', { slug, detail: message });
-            recordFailure({ actor, operation: 'promote.store-backfill', vertical: slug, message }, e);
-            return { minted: [], error: message };
+            // The sweep mints across every installed tenant, so what failed is withheld from a confined
+            // caller (#2540): it reads that the backfill failed, and the reference.
+            const withheld = withheldFailure(c, { operation: 'promote.store-backfill', vertical: slug, message: messageOfThrow(e) }, e);
+            return {
+                minted: [],
+                error: `${withheld.detail ?? 'the stores this version declares could not be minted for every install'} (reference ${withheld.reference})`,
+            };
         }
     };
+    /**
+     * #2446: the scopes a prod promote of `versionId` would hold at their reconcile, judged before
+     * anything moves. A pushed vertical's reconcile defines the incoming version's role set in each
+     * scope it runs in (`refuseReservedRoleSet`), and refuses it where the scope's stored roles and
+     * seats forbid it (#2525: a seated person's role made `holder: 'system'`, among others), which
+     * holds that scope's whole reconcile while it serves the new code. So each scope the promote moves
+     * onto the incoming version is asked for that state (`/internal/role-state`) and judged with the
+     * reconcile's own rule and inputs (`releaseRoleSetRefusal`: the incoming registry's roles and
+     * marks, which the push holds to the module declarations it carries).
+     *
+     * The scopes moved are the ones the in-place serve and the adopt after it move: every live primary
+     * scope of the vertical already on its serving script, and, for a private vertical, every one the
+     * owning tenant runs. Only a dispatch-backed version served in place moves them here; an embedded
+     * vertical's roles are its host's, defined by no reconcile of the registry, so it is asked nothing.
+     * Nor is a release no stored state can refuse (`releaseRolesAskScopes`), so a vertical with no
+     * system-held role pays no read.
+     *
+     * At most `roleStateScopeCap` scopes are read, `ROLE_STATE_CONCURRENCY` at a time in (tenant,
+     * scope) order; more is a 503 naming the two ways through (raise the cap, or staff's
+     * `acknowledge.roleStateUnjudged`, which skips the read and is audited), never a partial
+     * judgement. A deployment that cannot answer the read (one that predates the route, a 404, or one
+     * whose scope host predates the method, its 501) is counted instead; its reconcile still refuses.
+     * Any other failed read is collected in `failed`, with what the deployment said, and refuses the
+     * promote with nothing moved. Nothing here is phrased for a caller: the route narrows every answer
+     * to the scopes the caller may see, and what a deployment said reaches only the log and staff.
+     */
+    const releaseRoleRefusalsOf = async (c, slug, versionId) => {
+        const none = { refused: [], failed: [], unread: 0 };
+        if (!options.deployVertical || !options.fetchVerticalModules)
+            return none; // no in-place serve moves anything
+        const actor = c.get('actor');
+        const incoming = await host.admin.getVersion(actor, versionId, slug);
+        if (!incoming?.deploymentRef)
+            return none;
+        const manifestJson = await host.admin.versionManifest(actor, slug, versionId);
+        const registry = manifestJson ? storedDeployManifest.parse(JSON.parse(manifestJson)).registry : undefined;
+        if (!registry || !releaseRolesAskScopes(registry))
+            return none;
+        const v = await verticalOf(actor, slug);
+        const owner = v && v.ownerTenant !== null && !v.listed ? v.ownerTenant : null;
+        const moved = (await c.var.admin.listScopes(actor, { vertical: slug, status: ['active', 'suspended'] }))
+            .filter((s) => !s.forkedFrom && s.kind !== 'preview' && (!!s.servingRef || (s.tenantId === owner && s.status === 'active')))
+            .sort((a, b) => (a.tenantId === b.tenantId ? a.id.localeCompare(b.id) : a.tenantId.localeCompare(b.tenantId)));
+        const cap = options.roleStateScopeCap ?? ROLE_STATE_SCOPE_CAP;
+        if (moved.length > cap) {
+            throw new ControlPlaneError(503, `this version declares a system-held role, so a promote judges the roles of every scope it moves before moving ` +
+                `any, and it would move ${moved.length}, more than the ${cap} one promote reads. Nothing was promoted. Raise the control ` +
+                'plane\'s `roleStateScopeCap`, or, as staff, promote with `acknowledge.roleStateUnjudged`, which skips the read and ' +
+                'is recorded in the promote\'s audit row');
+        }
+        const refused = [];
+        const failed = [];
+        let unread = 0;
+        for (let i = 0; i < moved.length; i += ROLE_STATE_CONCURRENCY) {
+            const batch = moved.slice(i, i + ROLE_STATE_CONCURRENCY);
+            const states = await Promise.all(batch.map(async (s) => {
+                try {
+                    const client = await verticalForScope(c, s);
+                    if (!client)
+                        return { failed: 'no deployment serves it' };
+                    return await client.roleState({ tenantId: s.tenantId, scopeId: s.id });
+                }
+                catch (e) {
+                    return { failed: e instanceof Error ? e.message : String(e) };
+                }
+            }));
+            batch.forEach((s, j) => {
+                const state = states[j];
+                if (state === 'predates')
+                    return void unread++;
+                if ('failed' in state) {
+                    console.warn('promote.role-state.unreadable', { slug, versionId, tenantId: s.tenantId, scopeId: s.id, detail: state.failed });
+                    return void failed.push({ tenantId: s.tenantId, scopeId: s.id, detail: state.failed });
+                }
+                const refusal = releaseRoleSetRefusal(registry, state);
+                if (refusal)
+                    refused.push({ tenantId: s.tenantId, scopeId: s.id, refusal });
+            });
+        }
+        if (unread > 0)
+            console.warn('promote.role-state.predates', { slug, versionId, scopes: unread });
+        return { refused, failed, unread };
+    };
     /**
      * Which installed apps promoting `versionId` to `channel` would break: the promote gate's own
      * answer (`promotionImpact`, judged from the channel's previous version), and — for a PRIVATE,
@@ -7265,6 +7729,23 @@ export function createControlPlaneApi(options) {
             return { affected: visible, ...(otherTenants ? { otherTenants } : {}) };
         };
         const exportBreaks = acknowledge?.exportBreak ? await breaksOf() : null;
+        // #2446: the scopes whose reconcile this version's roles would hold, before anything moves. The
+        // digest and export-break acknowledgements do not pass it: the way through is taking the seats
+        // back, or a release that keeps the role as each scope stores it. It runs ahead of the host's
+        // admission and digest gates rather than being skipped when they look set to refuse: those are
+        // judged inside `promoteVersion` against the channel as it stands then, so a read skipped on a
+        // prediction would let a promote through unjudged if an admit or a channel move landed between.
+        // The read is paid only by a release declaring a system-held role, at most the cap.
+        // `roleStateUnjudged` skips the read: staff only, and recorded with the promote's acknowledgements.
+        if (acknowledge?.roleStateUnjudged && confinedTenant(p) !== null) {
+            return refuse(c, 'forbidden', 'acknowledge.roleStateUnjudged is for staff: it skips the role-state read a promote judges its roles with');
+        }
+        const roles = acknowledge?.roleStateUnjudged
+            ? { refused: [], failed: [], unread: 0 }
+            : await releaseRoleRefusalsOf(c, slug, versionId);
+        const answer = roleStateAnswer(roles, confinedTenant(p));
+        if (answer)
+            return refuseStatus(c, answer.status, answer.message, answer.extra);
         // The blast-radius moment: refuses a changed digest without the acknowledgement,
         // and refuses a non-admitted version. Both are enforced below the seam and
         // surface as a 4xx through mapError, not a 500.
@@ -7276,10 +7757,16 @@ export function createControlPlaneApi(options) {
                 throw err;
             // The refusal stands whether or not the listing can be read. A listing that fails turns
             // into a note, never into a 500 that would hide that the promote was refused.
-            const listing = await breaksOf().then((exportBreaks) => ({ exportBreaks }), (e) => ({
-                exportBreaks: null,
-                exportBreaksUnavailable: `which apps it breaks could not be read: ${e instanceof Error ? e.message : String(e)}`,
-            }));
+            // The listing is read across every tenant that installed the vertical, so what failed is
+            // withheld from a confined caller (#2540).
+            const listing = await breaksOf().then((exportBreaks) => ({ exportBreaks }), (e) => {
+                const withheld = withheldFailure(c, { operation: 'promote.export-breaks', vertical: slug, version: versionId, message: messageOfThrow(e) }, e);
+                return {
+                    exportBreaks: null,
+                    exportBreaksUnavailable: `which apps it breaks could not be read: ${withheld.detail ?? 'the platform recorded why'} ` +
+                        `(reference ${withheld.reference})`,
+                };
+            });
             return refuse(c, 'conflict', listing.exportBreaks ? exportBreakRefusal(counted) : err.message, listing);
         }
         // The in-place serve (#286), prod only, AFTER every promote gate has passed —
@@ -7304,9 +7791,21 @@ export function createControlPlaneApi(options) {
                 backfill = await backfillFleetStores(c, slug);
             }
             catch (e) {
-                const detail = e instanceof Error ? e.message : String(e);
-                console.error('serve.inplace.failed', { slug, versionId, detail });
-                return refuseStatus(c, 502, 'promoted, but the in-place serve failed — scopes still run the previous code; promote again to retry', { detail });
+                // The serve, the adopt and the backfill reach every install of the vertical, whoever's, so
+                // what failed is withheld from a confined caller (#2540), and the tenant the backstop would
+                // have filed it under reads only the sentence.
+                const sentence = (reference) => `promoted, but the in-place serve failed — scopes still run the previous code; promote again to retry (reference ${reference})`;
+                const withheld = withheldFailure(c, {
+                    operation: 'promote.serve',
+                    stage: 'serve-in-place',
+                    vertical: slug,
+                    version: versionId,
+                    tenantId: (await failureSubject(c)).tenantId,
+                    status: 502,
+                    message: messageOfThrow(e),
+                }, e, sentence);
+                c.set('failureRecorded', true);
+                return refuseStatus(c, 502, sentence(withheld.reference), withheld);
             }
         }
         // #1756: served, and every install that could move has moved and been backfilled, but one
@@ -7350,6 +7849,8 @@ export function createControlPlaneApi(options) {
             ...promoted,
             // What the acknowledged break reached, so a promoter who passed `exportBreak` reads it.
             ...(exportBreaks ? { exportBreaks } : {}),
+            // #2446: scopes whose deployment predates the role-state read were moved unjudged, in counts.
+            ...(roles.unread > 0 ? { roleStateUnread: { scopes: roles.unread, note: ROLE_STATE_UNREAD_NOTE } } : {}),
             ...(backfill.minted.length || backfill.error
                 ? {
                     storeBackfill: {
@@ -7584,8 +8085,9 @@ export function createControlPlaneApi(options) {
         // #2210: the store scripts first, outside the vertical upload's try, so a store failure
         // answers as itself and the vertical is not uploaded at all.
         await ensureScopeStoresOf(c.get('actor'), 'deploy', slug, manifest.scopeStores, async () => ownerTenant ?? null);
+        let wire;
         try {
-            await options.deployVertical(deploymentRef, {
+            wire = await deployVerticalScript(options.deployVertical, c.get('actor'), slug, deploymentRef, {
                 entry: manifest.entry,
                 compatibilityDate: manifest.compatibilityDate,
                 compatibilityFlags: manifest.compatibilityFlags,
@@ -7689,6 +8191,7 @@ export function createControlPlaneApi(options) {
             // #1902: with the sweeper decision the push made, which no later upload re-makes.
             manifestJson: JSON.stringify({ ...manifest, platformSweeper: sweeper.supply }),
         });
+        await recordWireUpload(c.var.admin, c.get('actor'), id, wire);
         // Same spirit as the permission-surface gate, advisory tier: when the push DECLARES
         // surfaces, name any surface that hostnames are still bound to but the declaration
         // dropped — the URL keeps resolving (routing never keys on the declaration), it just
@@ -7791,7 +8294,9 @@ export function createControlPlaneApi(options) {
         }
         // `service` repeats: one deployed unit per param, so a caller can ask for a
         // vertical's whole set (the dashboard's "all versions") in one query and get one
-        // merged stream back. Capped because each extra service is another backend query.
+        // merged stream back. Capped because each extra service is another backend query, and
+        // the Cloudflare reader may add a few more to fill the merged page (at most
+        // `MAX_FILL_QUERIES` per read, whatever the number of services).
         const asked = (c.req.queries('service') ?? []).filter((s) => s.length > 0);
         const ownedLogServices = await ownedServices(c.get('principal'), c.get('actor'));
         // A confined principal reads only its own services' lines. An ask that names none
@@ -7815,7 +8320,12 @@ export function createControlPlaneApi(options) {
             hours: c.req.query('hours'),
             limit: c.req.query('limit'),
         });
-        return c.json(await options.observability.recentLogs(input));
+        const { events, incomplete } = await options.observability.recentLogs(input);
+        // The body stays the list every caller reads; a read that stopped short of vouching for the
+        // whole window says so beside it, so a short page is never silently "nothing more".
+        if (incomplete)
+            c.header(LOGS_INCOMPLETE_HEADER, '1');
+        return c.json(events);
     });
     // -- the TENANT grain (observability.md §3 view 4, §4.2) -----------------------
     //
@@ -9742,7 +10252,12 @@ export function createControlPlaneApi(options) {
             cursor: c.req.query('cursor'),
             order: c.req.query('order'),
         });
-        const entries = await c.var.admin.listOpsFailures(c.get('actor'), filter);
+        const entries = await c.var.admin.listOpsFailures(c.get('actor'), {
+            ...filter,
+            // #2540: staff's `reference` also finds a pair by the upstream handle its staff row names.
+            // The text that matches is staff's, so a confined caller's search stays exact.
+            ...(staffCaller(c) ? { referenceInText: true } : {}),
+        });
         return c.json(pageOf(entries, filter.limit, (e) => e.id));
     });
     // -- sweep runs (#1232): the durable record of what a sweep pass touched -------