@substrat-run/control-plane-api 0.116.0 → 0.118.0

package/dist/api.js

@@ -1,9 +1,11 @@
 import { Hono } from 'hono';
-import { parseHostname, withLabel, RESERVED_LABEL_SEPARATOR, adminAction, ASSET_PART_PREFIX, assetHash, channelName, connectionActivity, connectionActivitySource, connectionCredential, connectionFilter, connectionProbe, createTenantInput, entitlementGrantInput, hostname as hostnameSchema, hostnameRegion, hostnameStatus, identityLink, errorCode, issueStatus, issueStatusInput, listPageQuery, matchesOutboundHost, pageOf, platformRequestFilter, principalId as principalIdSchema, promotionAcknowledgement, provisionableJurisdiction, publishVersionInput, queryScopeInput, readScopeTableInput, entityHistoryInput, eventFacetInput, eventCauseInput, eventEffectsInput, invocationEventsInput, deadLettersInput, delegatedReadParams, createOrgInput, roleKey as roleKeySchema, DEFAULT_DENIAL_LIMIT, DENIAL_LIMIT_MAX, denialGroupBy, registerVerticalInput, serviceDimensions, sweepRunKind, sweepRunOutcome, scopeDump, dataSubjectId as dataSubjectIdSchema, scopeId as scopeIdSchema, scopeStatus, storageShape, surfaceName, tenantId as tenantIdSchema, tenantStatus, versionOrigin, z, PROBLEM_CONTENT_TYPE, toProblem, redrainEventsInput, REDRAIN_BATCH, } from '@substrat-run/contracts';
+import { parseHostname, withLabel, RESERVED_LABEL_SEPARATOR, adminAction, ASSET_PART_PREFIX, assetHash, channelName, connectionActivity, connectionActivitySource, connectionCredential, connectionFilter, connectionProbe, createTenantInput, entitlementGrantInput, hostname as hostnameSchema, hostnameRegion, hostnameStatus, identityLink, errorCode, issueStatus, issueStatusInput, listPageQuery, matchesOutboundHost, pageOf, platformRequestFilter, principalId as principalIdSchema, promotionAcknowledgement, provisionableJurisdiction, publishVersionInput, queryScopeInput, readScopeTableInput, entityHistoryInput, eventFacetInput, eventCauseInput, eventEffectsInput, invocationEventsInput, deadLettersInput, delegatedReadParams, createOrgInput, roleKey as roleKeySchema, systemSwitch, DEFAULT_DENIAL_LIMIT, DENIAL_LIMIT_MAX, denialGroupBy, registerVerticalInput, serviceDimensions, sweepRunKind, sweepRunOutcome, scopeDump, dataSubjectId as dataSubjectIdSchema, scopeId as scopeIdSchema, scopeStatus, storageShape, surfaceName, tenantId as tenantIdSchema, tenantStatus, versionOrigin, z, PROBLEM_CONTENT_TYPE, toProblem, redrainEventsInput, REDRAIN_BATCH, } from '@substrat-run/contracts';
 import { attributeFailure } from './failure-attribution.js';
 import { migrationProgress, ulid } from '@substrat-run/kernel';
-import { TENANT_HEADER } from './auth.js';
+import { TENANT_HEADER, confinedTenant } from './auth.js';
+import { mintTenantToken } from './tenant-token.js';
 import { connectionGrantsForScope } from './vertical-client.js';
+import { versionReachedAt } from './scope-deployment.js';
 import { reconcileConnectionGrants } from './connection-grants.js';
 import { ConnectionRelayError, relayConnectionUpsert } from './connection-relay.js';
 import { ControlPlaneError } from './client.js';
@@ -31,6 +33,7 @@ import { mintPushToken, pushActorFor } from './push-token.js';
 import { TENANT_SERIES_SCOPE_CAP } from './observability.js';
 import { namespacesForScript } from './do-namespaces.js';
 import { backupDirectoryIfDue } from './directory-backup.js';
+import { STORAGE_PAGE_DEFAULT, STORAGE_PAGE_MAX, readStoragePage } from './storage-meter.js';
 import { isCustomHostname, validateBindableHostname, } from './custom-hostnames.js';
 /**
  * Response headers carrying the CLI version advisory (`ControlPlaneApiOptions.cliAdvisory`).
@@ -39,6 +42,18 @@ import { isCustomHostname, validateBindableHostname, } from './custom-hostnames.
  */
 export const CLI_MIN_VERSION_HEADER = 'x-substrat-cli-min-version';
 export const CLI_LATEST_VERSION_HEADER = 'x-substrat-cli-latest-version';
+/**
+ * "This principal is confined to a tenant, and this is not that tenant."
+ *
+ * The question every `/tenants/:tenantId/…` route asks before it answers. Spelled in
+ * terms of CONFINEMENT rather than of a principal kind, because a route that asks
+ * `kind === 'builder'` answers fleet-wide for every other confined credential — which
+ * is the shape #977 found the dashboard's in.
+ */
+function outsideTenant(p, tenantId) {
+    const pin = confinedTenant(p);
+    return pin !== null && pin !== tenantId;
+}
 // -- request schemas ---------------------------------------------------------
 // Parse, don't trust: every input crosses Zod at the boundary. The ids stay
 // CALLER-SUPPLIED rather than minted here, exactly as the contract has them —
@@ -322,6 +337,12 @@ const tenantRoleAssignmentBody = z
     roleKey: roleKeySchema,
 })
     .strict();
+/**
+ * The schedule kill switch's body (#1666) — the module and the reason. The node comes
+ * from the PATH, after the K-3 cross-check, for the reason `tenantRoleAssignmentBody`
+ * spells out; `.strict()` so a body that tried to name one is refused, not stripped.
+ */
+const systemSwitchBody = systemSwitch.pick({ moduleId: true, reason: true }).strict();
 /** A new org under an addressed tenant (#1343) — `tenantId` comes from the path. */
 const tenantOrgBody = createOrgInput.omit({ tenantId: true });
 const sweepRunsQuery = z.object({
@@ -356,6 +377,105 @@ export const DEFAULT_MODEL_MARGIN_PERCENT = 20;
 /** The upstream provider's trace handle, when a failure message carries one —
  *  Cloudflare's `internal error; reference = <id>` shape (#559). */
 const UPSTREAM_REFERENCE = /\breference\s*=\s*([a-z0-9]+)/i;
+const TENANT_ROUTES = [
+    // Everything under one tenant's own path. One entry rather than sixty, because
+    // the confinement is carried by the URL shape and not by a list anybody has to
+    // keep current: `/tenants/<pin>/…` cannot address another tenant's row whatever
+    // the sub-route does. `/tenants` itself (the fleet list) does not match.
+    { method: 'GET', re: /\/tenants\/[^/]+(\/.*)?$/, pin: 'path' },
+    { method: 'POST', re: /\/tenants\/[^/]+(\/.*)?$/, pin: 'path' },
+    { method: 'PUT', re: /\/tenants\/[^/]+(\/.*)?$/, pin: 'path' },
+    { method: 'PATCH', re: /\/tenants\/[^/]+(\/.*)?$/, pin: 'path' },
+    { method: 'DELETE', re: /\/tenants\/[^/]+(\/.*)?$/, pin: 'path' },
+    // Create the tenant's own directory row (idempotent) — `id` is the pin.
+    { method: 'POST', re: /\/tenants$/, pin: 'body' },
+    // Provision an app + materialize its instance.
+    { method: 'POST', re: /\/scopes$/, pin: 'body' },
+    { method: 'POST', re: /\/verticals\/[^/]+\/instances$/, pin: 'body' },
+    // Mint the tenant's CI push token. Note what is NOT here: `/tenant-tokens`. A
+    // tenant credential cannot mint another, for itself or for anyone — the mint
+    // stays staff-only, so this credential can never widen its own reach.
+    { method: 'POST', re: /\/push-tokens$/, pin: 'body' },
+    // The forced-filter reads: the route refuses without a tenantId and answers only
+    // that tenant's rows.
+    //
+    // `GET /scopes` is the one the dashboard leans on hardest — the Data tab's scope
+    // switcher and the whole Move/Retire flow (#1593) read it — and it is the one a
+    // path-shaped reading of this list would miss, because it names its tenant in a
+    // query while every other scope route names it in the path.
+    { method: 'GET', re: /\/scopes$/, pin: 'query' },
+    { method: 'GET', re: /\/admin-log$/, pin: 'query' },
+    { method: 'GET', re: /\/ops-failures$/, pin: 'query' },
+    { method: 'GET', re: /\/sweep-runs$/, pin: 'query' },
+    { method: 'GET', re: /\/service-refs$/, pin: 'query' },
+    { method: 'GET', re: /\/observability\/tenant-metrics$/, pin: 'query' },
+    { method: 'GET', re: /\/observability\/tenant-metrics-series$/, pin: 'query' },
+    { method: 'GET', re: /\/observability\/tenant-logs$/, pin: 'query' },
+    { method: 'GET', re: /\/hostnames$/, pin: 'query' },
+    { method: 'POST', re: /\/hostnames$/, pin: 'body' },
+    // The registry + install catalog.
+    { method: 'GET', re: /\/verticals$/, pin: 'catalog' },
+    // Ownership-narrowed: the handler resolves who owns the slug / the hostname.
+    // Not listed, because the dashboard does not call them and an unused route on an
+    // allowlist is reach nobody asked for: POST /verticals (register), POST
+    // /verticals/:slug/versions (publish), .../deploy, .../publish-request. A push is
+    // the CLI's act, over a push token, and stays one.
+    { method: 'DELETE', re: /\/verticals\/[^/]+$/, pin: 'owner' },
+    { method: 'GET', re: /\/verticals\/[^/]+\/versions$/, pin: 'owner' },
+    { method: 'GET', re: /\/verticals\/[^/]+\/versions\/[^/]+\/registry$/, pin: 'owner' },
+    { method: 'GET', re: /\/verticals\/[^/]+\/versions\/[^/]+\/model$/, pin: 'owner' },
+    { method: 'GET', re: /\/verticals\/[^/]+\/versions\/[^/]+\/flow$/, pin: 'owner' },
+    { method: 'GET', re: /\/verticals\/[^/]+\/versions\/[^/]+\/assets$/, pin: 'owner' },
+    { method: 'GET', re: /\/verticals\/[^/]+\/versions\/[^/]+\/schedules$/, pin: 'owner' },
+    { method: 'GET', re: /\/verticals\/[^/]+\/channels$/, pin: 'owner' },
+    { method: 'GET', re: /\/verticals\/[^/]+\/channels\/[^/]+\/history$/, pin: 'owner' },
+    { method: 'POST', re: /\/verticals\/[^/]+\/channels\/[^/]+\/promote$/, pin: 'owner' },
+    { method: 'GET', re: /\/verticals\/[^/]+\/previews$/, pin: 'owner' },
+    { method: 'POST', re: /\/verticals\/[^/]+\/previews$/, pin: 'owner' },
+    { method: 'DELETE', re: /\/verticals\/[^/]+\/previews\/[^/]+$/, pin: 'owner' },
+    { method: 'POST', re: /\/hostnames\/[^/]+\/verify$/, pin: 'owner' },
+    { method: 'PATCH', re: /\/hostnames\/[^/]+\/status$/, pin: 'owner' },
+    { method: 'DELETE', re: /\/hostnames\/[^/]+$/, pin: 'owner' },
+    // Script-grain observability, narrowed in the handler to the services this
+    // tenant's own verticals deploy (see `ownedServices`). Without that narrowing
+    // these are fleet reads, which is how the dashboard used them over the staff
+    // token — filtering the fleet's answer in the browser worker's memory.
+    { method: 'GET', re: /\/observability\/metrics$/, pin: 'owner' },
+    { method: 'GET', re: /\/observability\/metrics-series$/, pin: 'owner' },
+    { method: 'GET', re: /\/observability\/logs$/, pin: 'owner' },
+];
+/**
+ * How a tenant-scoped credential's pin is checked on this route, or `undefined` when
+ * the route is off its allowlist entirely (default-deny).
+ *
+ * Exported so the DASHBOARD's own suite can assert the other direction: that every
+ * route its authority seam can issue is one this credential reaches. Nothing else can
+ * check that — the allowlist lives here and the call sites live there — and the gap
+ * between the two is a 403 in production and a green test on both sides. The first
+ * pass of this list was written by reading `authority.ts`, and reading it missed
+ * `GET /scopes`, which is the route the Data tab and Move/Retire lean on hardest.
+ */
+export function tenantCredentialPin(method, path) {
+    return TENANT_ROUTES.find((r) => r.method === method && r.re.test(path))?.pin;
+}
+/** The `/tenants/<id>/…` segment a path names, if it names one. */
+const pathTenant = (path) => /\/tenants\/([^/]+)(?:\/|$)/.exec(path)?.[1];
+/**
+ * Every tenant this request names where the middleware can see it — the path
+ * segment, the tenant-ish query params, and the `x-substrat-tenant` pin header.
+ * The BODY is not swept here: it is read only for a `'body'` route, because parsing
+ * it costs a read on every request and only those routes are decided by it.
+ */
+const namedTenants = (c) => {
+    const named = [
+        pathTenant(c.req.path),
+        c.req.query('tenantId'),
+        c.req.query('ownerTenant'),
+        c.req.query('visibleTo'),
+        c.req.header(TENANT_HEADER)?.trim() || undefined,
+    ];
+    return named.filter((v) => typeof v === 'string' && v.length > 0);
+};
 /**
  * The audited HTTP surface over `HostAdmin` (control-plane.md §4.5).
  *
@@ -384,7 +504,7 @@ const UPSTREAM_REFERENCE = /\breference\s*=\s*([a-z0-9]+)/i;
  *    adapter's read path, not an admin surface.
  */
 export function createControlPlaneApi(options) {
-    const { host, authenticate, authenticateBuilder } = options;
+    const { host, authenticate, authenticateBuilder, authenticateTenantService } = options;
     const admin = host.admin;
     const app = new Hono();
     // The CLI version advisory (#971), stamped on EVERY response — including the 401 the
@@ -401,11 +521,28 @@ export function createControlPlaneApi(options) {
                 c.header(CLI_LATEST_VERSION_HEADER, advisory.latestVersion);
         });
     }
-    // Fail closed, before any route runs: no principal, no reach. Staff/service first
-    // (unchanged, a superset); a builder session only when staff declines.
+    // Fail closed, before any route runs: no principal, no reach.
+    //
+    // The TENANT-SCOPED credential is read FIRST (#977), ahead of staff. Not for
+    // precedence between two things that could both match — an `stt1.` value is not a
+    // platform service token and never will be — but because order is the only thing
+    // that makes "a tenant token is never staff" hold no matter how `authenticate` is
+    // configured. A deployment that pasted a tenant token into `SERVICE_TOKEN` would,
+    // with the readers the other way round, silently hand it the whole fleet.
+    //
+    // Staff/service next (unchanged, a superset); a builder session only when staff
+    // declines.
     app.use('*', async (c, next) => {
-        const staff = await authenticate(c.req.raw);
-        let principal = staff ? { kind: 'staff', actor: staff } : null;
+        let principal = null;
+        if (authenticateTenantService) {
+            const tenant = await authenticateTenantService(c.req.raw);
+            if (tenant) {
+                principal = { kind: 'tenant', actor: tenant.actor, tenantId: tenant.tenantId };
+            }
+        }
+        const staff = principal ? null : await authenticate(c.req.raw);
+        if (!principal && staff)
+            principal = { kind: 'staff', actor: staff };
         if (!principal && authenticateBuilder) {
             const builder = await authenticateBuilder(c.req.raw);
             if (builder) {
@@ -518,6 +655,45 @@ export function createControlPlaneApi(options) {
         }
         await next();
     });
+    app.use('*', async (c, next) => {
+        const p = c.get('principal');
+        if (p.kind !== 'tenant')
+            return next();
+        const pin = tenantCredentialPin(c.req.method, c.req.path);
+        if (!pin)
+            return c.json({ error: 'forbidden' }, 403);
+        // The sweep: anything naming a tenant must name THIS one. `x-substrat-tenant` is
+        // included, which is what stops the pre-#977 shape — a fleet credential plus a
+        // caller-asserted tenant header — from surviving as a way to ask for another.
+        if (namedTenants(c).some((t) => t !== p.tenantId)) {
+            return c.json({ error: 'forbidden: outside this credential’s tenant' }, 403);
+        }
+        // The route's own pin must be PRESENT, not merely consistent. A missing filter is
+        // what makes these routes answer fleet-wide, so absence is the refusal.
+        if (pin === 'path' && pathTenant(c.req.path) !== p.tenantId) {
+            return c.json({ error: 'forbidden: outside this credential’s tenant' }, 403);
+        }
+        if (pin === 'query' && c.req.query('tenantId') !== p.tenantId) {
+            return c.json({ error: 'forbidden: this credential must name its own tenant' }, 403);
+        }
+        if (pin === 'catalog') {
+            const named = c.req.query('ownerTenant') ?? c.req.query('visibleTo');
+            if (named !== p.tenantId) {
+                return c.json({ error: 'forbidden: this credential must name its own tenant' }, 403);
+            }
+        }
+        if (pin === 'body') {
+            // Hono caches a parsed body, so the handler's own `c.req.json()` still works.
+            // An unparseable body names no tenant, which is a refusal here rather than the
+            // handler's 400 — a request this middleware cannot read is one it cannot clear.
+            const body = (await c.req.json().catch(() => null));
+            const named = body && typeof body === 'object' ? (body['tenantId'] ?? body['id']) : undefined;
+            if (named !== p.tenantId) {
+                return c.json({ error: 'forbidden: this credential must name its own tenant' }, 403);
+            }
+        }
+        await next();
+    });
     // Fire-and-forget ops-failure write (#559): a recorder that throws must never
     // mask the failure it is recording, so every path through this swallows its own
     // errors. The upstream reference is extracted here — one place — so a caller
@@ -742,6 +918,50 @@ export function createControlPlaneApi(options) {
         const tenantId = raw ? tenantIdSchema.parse(raw) : undefined;
         return c.json(await admin.readMeters(c.get('actor'), tenantId ? { tenantId } : undefined));
     });
+    // Storage, read on demand (#1524): one tenant's scope-database sizes and their sum.
+    // Staff-only like `/meters` (on neither allowlist, so a builder or a tenant token is
+    // refused). `tenantId` is REQUIRED. A fleet-wide form would wake every DO in the fleet,
+    // the cost the decision rejected. Paged over the tenant's scopes, because every scope
+    // read is a DO wake. `readStoragePage` holds the page and concurrency bounds.
+    //
+    // A scope that names a vertical is read through that vertical's deployment, which
+    // holds its DO. When this plane delegates to verticals at all and none resolves, the
+    // read FAILS for that scope rather than falling back to the co-located host. On the
+    // hosted plane that host's namespace is a module-less placeholder, and waking it would
+    // create an empty database and report its size as the scope's.
+    const delegatesToVerticals = Boolean(options.verticals || options.resolveVertical || options.resolveVerticalVersion || options.resolveVerticalRef);
+    app.get('/meters/storage', async (c) => {
+        const q = z
+            .object({
+            tenantId: tenantIdSchema,
+            cursor: scopeIdSchema.optional(),
+            limit: z.coerce.number().int().min(1).max(STORAGE_PAGE_MAX).default(STORAGE_PAGE_DEFAULT),
+        })
+            .parse({
+            tenantId: c.req.query('tenantId'),
+            cursor: c.req.query('cursor') || undefined,
+            limit: c.req.query('limit') || undefined,
+        });
+        const actor = c.get('actor');
+        const scopes = await admin.listScopes(actor, { tenantId: q.tenantId });
+        return c.json(await readStoragePage({
+            tenantId: q.tenantId,
+            readAt: new Date().toISOString(),
+            scopes: scopes.map((s) => ({ ...s, scopeId: s.id })),
+            cursor: q.cursor,
+            limit: q.limit,
+            read: async (scope) => {
+                if (!scope.vertical)
+                    return admin.scopeDatabaseSize(actor, q.tenantId, scope.id);
+                const vertical = await verticalForScope(c, scope);
+                if (vertical)
+                    return vertical.databaseSize(scope.id);
+                if (delegatesToVerticals)
+                    throw new Error(`no deployment resolves for vertical ${scope.vertical}`);
+                return admin.scopeDatabaseSize(actor, q.tenantId, scope.id);
+            },
+        }));
+    });
     // -- per-tenant stores (#301, #473) ----------------------------------------
     // The two ledgers as INVENTORY — what `listTenantStores`/`listBlobStores` were always
     // meant to answer for a staff surface: which database and which bucket hold this
@@ -945,7 +1165,7 @@ export function createControlPlaneApi(options) {
         const filter = listScopesQuery.parse({
             // A builder reads only its OWN tenant's directory rows — the filter is forced,
             // not trusted from the query (#424 CLI parity for `substrat installs`).
-            tenantId: p.kind === 'builder' ? p.tenantId : c.req.query('tenantId'),
+            tenantId: confinedTenant(p) ?? c.req.query('tenantId'),
             status: c.req.queries('status'),
             vertical: c.req.query('vertical'),
         });
@@ -993,7 +1213,7 @@ export function createControlPlaneApi(options) {
         const p = c.get('principal');
         // A builder acts only within its own tenant; a foreign tenant is hidden as 404 (K-3),
         // never distinguished from "no such scope".
-        if (p.kind === 'builder' && p.tenantId !== tenantId) {
+        if (outsideTenant(p, tenantId)) {
             return c.json({ error: `unknown scope for tenant: (${tenantId}, ${input.parentScopeId})` }, 404);
         }
         // The provisioning sequence lives in one place — `provisionSiblingScope` — shared with the
@@ -1022,7 +1242,7 @@ export function createControlPlaneApi(options) {
         const scopeId = scopeIdSchema.parse(c.req.param('scopeId'));
         // A builder reads only its own tenant; a foreign tenant is hidden as 404 (K-3).
         const p = c.get('principal');
-        if (p.kind === 'builder' && p.tenantId !== tenantId) {
+        if (outsideTenant(p, tenantId)) {
             return c.json({ error: `unknown scope for tenant: (${tenantId}, ${scopeId})` }, 404);
         }
         const record = await admin.getScopeRecord(c.get('actor'), tenantId, scopeId);
@@ -1058,7 +1278,18 @@ export function createControlPlaneApi(options) {
         seal: (items) => admin.sealSubjectPayloads(c.get('actor'), tenantId, scopeId, items),
         open: (items) => admin.openSubjectPayloads(c.get('actor'), tenantId, scopeId, items),
     });
-    const verticalForScope = async (c, scope) => {
+    const verticalForScope = async (c, scope) => (await deploymentForScope(c, scope))?.client;
+    /**
+     * `verticalForScope`, saying WHICH rung of the ladder chose the client (#1653).
+     *
+     * A reconcile's receipt must name the version whose hook actually ran, and that is a
+     * fact about the client chosen here — not about the scope's serving pointer read on the
+     * side. When the serving ref does not resolve, the ladder falls back to the bound
+     * version's deployment (or a slug's), and a receipt derived separately would claim the
+     * served version's hook ran when the bound one's did. `versionReachedAt(via, …)` turns
+     * this answer into the version, so the choice and the receipt have one source.
+     */
+    const deploymentForScope = async (c, scope) => {
         if (!scope.vertical)
             return undefined;
         const actor = c.get('actor');
@@ -1067,15 +1298,16 @@ export function createControlPlaneApi(options) {
         if (scope.servingRef && options.resolveVerticalRef) {
             const serving = await options.resolveVerticalRef(scope.servingRef);
             if (serving)
-                return serving;
+                return { client: serving, via: 'serving-script' };
         }
         const bySlug = async (slug) => {
             if (scope.verticalVersionId && options.resolveVerticalVersion) {
                 const bound = await options.resolveVerticalVersion(slug, scope.verticalVersionId, actor);
                 if (bound)
-                    return bound;
+                    return { client: bound, via: 'bound-version' };
             }
-            return options.verticals?.[slug] ?? (await options.resolveVertical?.(slug, actor));
+            const bySlugClient = options.verticals?.[slug] ?? (await options.resolveVertical?.(slug, actor));
+            return bySlugClient ? { client: bySlugClient, via: 'slug' } : undefined;
         };
         const direct = await bySlug(scope.vertical);
         if (direct)
@@ -1311,7 +1543,7 @@ export function createControlPlaneApi(options) {
         const scopeId = scopeIdSchema.parse(c.req.param('scopeId'));
         // A builder reads only its own tenant; a foreign tenant is hidden as 404 (K-3).
         const principal = c.get('principal');
-        if (principal.kind === 'builder' && principal.tenantId !== tenantId) {
+        if (outsideTenant(principal, tenantId)) {
             return c.json({ error: `unknown scope for tenant: (${tenantId}, ${scopeId})` }, 404);
         }
         const scope = await admin.getScopeRecord(c.get('actor'), tenantId, scopeId);
@@ -1363,7 +1595,7 @@ export function createControlPlaneApi(options) {
         const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
         const scopeId = scopeIdSchema.parse(c.req.param('scopeId'));
         const principal = c.get('principal');
-        if (principal.kind === 'builder' && principal.tenantId !== tenantId) {
+        if (outsideTenant(principal, tenantId)) {
             return c.json({ error: `unknown scope for tenant: (${tenantId}, ${scopeId})` }, 404);
         }
         const filter = platformRequestFilter.parse({
@@ -1396,14 +1628,16 @@ export function createControlPlaneApi(options) {
         if (!scope)
             return c.json({ error: `unknown scope for tenant: (${tenantId}, ${scopeId})` }, 404);
         const p = c.get('principal');
-        if (p.kind === 'builder') {
+        const scopePin = confinedTenant(p);
+        if (scopePin) {
             const v = scope.vertical ? await verticalOf(actor, scope.vertical) : undefined;
-            if (!v || v.ownerTenant !== p.tenantId)
+            if (!v || v.ownerTenant !== scopePin)
                 return c.json({ error: 'forbidden' }, 403);
         }
-        const vertical = await verticalForScope(c, scope);
-        if (!vertical)
+        const reached = await deploymentForScope(c, scope);
+        if (!reached)
             return c.json({ error: await diagnoseUnboundScope(c.get('actor'), scope) }, 501);
+        const vertical = reached.client;
         const entitlements = await admin.listEntitlements(actor, tenantId);
         // #406: re-gathered and re-delivered like entitlements, so a reconcile also repairs a
         // dropped identity-link delivery — and is the channel a link/unlink after provision rides.
@@ -1502,6 +1736,17 @@ export function createControlPlaneApi(options) {
                 patchBindings: options.patchScriptBindings,
             }));
         }
+        // #1653: WHICH version the hook is about to run as — the version of the deployment
+        // `reached` IS, never one read on the side: a serving ref that did not resolve fell back
+        // to the bound version's deployment, and it is that version's hook that runs. For the
+        // serving script, the pointer is read BEFORE the call, as the sweep reads it, so a
+        // promote landing mid-call cannot have its version recorded by a hook that ran as the
+        // one before. Best-effort: without it the receipt falls back to the bound version, the
+        // pre-#1653 answer, which costs at most one more reconcile.
+        const serving = reached.via === 'serving-script' && scope.vertical
+            ? await admin.verticalServing(actor, scope.vertical).catch(() => null)
+            : null;
+        const ranAs = versionReachedAt(reached.via, scope, serving);
         try {
             const result = await vertical.reconcileInstance({
                 tenantId,
@@ -1515,15 +1760,16 @@ export function createControlPlaneApi(options) {
                 // but unmigrated database fails as loudly as an absent one.
                 ...(tenantStores.length ? { tenantStores } : {}),
             });
-            // The reconcile succeeded; a store that did not mint rides back as a diagnosis
-            // (`substrat scope provision` prints it, and `/health` keeps reporting the gap
             // #1172: the reconcile succeeded, so record WHICH version it ran against. Without
             // this the sweep would come back and do it again on the next pass — the console
             // button and the automatic phase have to write the same receipt, or pressing the
-            // button means nothing to the thing that watches.
-            if (scope.verticalVersionId) {
-                await admin.markScopeProvisioned(actor, tenantId, scopeId, scope.verticalVersionId);
+            // button means nothing to the thing that watches. Both name it with
+            // `versionReachedAt`, from the deployment each actually reached.
+            if (ranAs) {
+                await admin.markScopeProvisioned(actor, tenantId, scopeId, ranAs);
             }
+            // The reconcile succeeded; a store that did not mint rides back as a diagnosis
+            // (`substrat scope provision` prints it, and `/health` keeps reporting the gap
             // until it closes). Absent when nothing failed, so the response is unchanged for
             // every scope that has no declared store or already has them all.
             return c.json(storeErrors.length ? { ...result, storeError: storeErrors.join('; ') } : result);
@@ -1749,7 +1995,7 @@ export function createControlPlaneApi(options) {
         const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
         const scopeId = scopeIdSchema.parse(c.req.param('scopeId'));
         const principal = c.get('principal');
-        if (principal.kind === 'builder' && principal.tenantId !== tenantId) {
+        if (outsideTenant(principal, tenantId)) {
             return c.json({ error: `unknown scope for tenant: (${tenantId}, ${scopeId})` }, 404);
         }
         const filter = denialLogQuery.parse(c.req.query());
@@ -1765,7 +2011,7 @@ export function createControlPlaneApi(options) {
         const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
         const scopeId = scopeIdSchema.parse(c.req.param('scopeId'));
         const principal = c.get('principal');
-        if (principal.kind === 'builder' && principal.tenantId !== tenantId) {
+        if (outsideTenant(principal, tenantId)) {
             return c.json({ error: `unknown scope for tenant: (${tenantId}, ${scopeId})` }, 404);
         }
         const filter = denialLogQuery.parse(c.req.query());
@@ -1788,7 +2034,7 @@ export function createControlPlaneApi(options) {
         const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
         const scopeId = scopeIdSchema.parse(c.req.param('scopeId'));
         const principal = c.get('principal');
-        if (principal.kind === 'builder' && principal.tenantId !== tenantId) {
+        if (outsideTenant(principal, tenantId)) {
             return c.json({ error: `unknown scope for tenant: (${tenantId}, ${scopeId})` }, 404);
         }
         const scope = await admin.getScopeRecord(c.get('actor'), tenantId, scopeId);
@@ -1810,7 +2056,7 @@ export function createControlPlaneApi(options) {
         const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
         const scopeId = scopeIdSchema.parse(c.req.param('scopeId'));
         const principal = c.get('principal');
-        if (principal.kind === 'builder' && principal.tenantId !== tenantId) {
+        if (outsideTenant(principal, tenantId)) {
             return c.json({ error: `unknown scope for tenant: (${tenantId}, ${scopeId})` }, 404);
         }
         const actor = c.get('actor');
@@ -2666,9 +2912,10 @@ export function createControlPlaneApi(options) {
         const p = c.get('principal');
         const slug = await resolveVerticalId(c, c.req.param('slug'));
         const actor = c.get('actor');
-        if (p.kind === 'builder') {
+        const adoptPin = confinedTenant(p);
+        if (adoptPin) {
             const v = await verticalOf(actor, slug);
-            if (!v || v.ownerTenant !== p.tenantId)
+            if (!v || v.ownerTenant !== adoptPin)
                 return c.json({ error: 'forbidden' }, 403);
         }
         const v = await verticalOf(actor, slug);
@@ -2902,6 +3149,19 @@ export function createControlPlaneApi(options) {
     // vertical is PRIVATE (owned + not listed), which is what scopes a builder's prod
     // self-serve below.
     const verticalOf = async (actor, slug) => (await admin.listVerticals(actor)).find((x) => x.slug === slug);
+    /**
+     * "This principal is confined, and this vertical is not its tenant's." The one
+     * predicate every slug-addressed route asks.
+     *
+     * It is `confinedTenant`, not `kind === 'builder'`, because the question is about
+     * confinement and never about which KIND of credential is confined — the dashboard's
+     * tenant token (#977) is confined too, and a route asking the narrower question
+     * would hand it the fleet. Staff are unconfined and always pass.
+     */
+    const notOwned = async (p, slug) => {
+        const pin = confinedTenant(p);
+        return pin !== null && (await ownerOf(p.actor, slug)) !== pin;
+    };
     // The vertical id a request actually addresses. For a BUILDER it is `<tenantSlug>/<name>`
     // (builder-plane.md §5): they send a bare `--slug`, the control plane forms the prefix
     // from their authenticated tenant — so two builders can each own a `helpdesk` with no
@@ -2970,33 +3230,22 @@ export function createControlPlaneApi(options) {
     // release-health and issues views) whether or not a telemetry backend is
     // configured. The dashboard's ownedServiceRefs() fetches this instead of
     // re-deriving it, so every consumer means the same thing by `version`.
-    app.get('/service-refs', async (c) => {
-        const p = c.get('principal');
-        // The /ops-failures forced-filter pattern: a builder's tenant comes from the
-        // principal, never the query. A staff/service caller must SAY whose view it
-        // wants — answering fleet-wide on a forgotten param would leak, so refuse.
-        const tenantId = p.kind === 'builder' ? p.tenantId : c.req.query('tenantId');
-        if (!tenantId)
-            throw new ControlPlaneError(400, 'tenantId is required');
-        const query = z
-            .object({ tenantId: tenantIdSchema, vertical: z.string().optional() })
-            .parse({ tenantId, vertical: c.req.query('vertical') });
-        const actor = c.get('actor');
-        const all = (await admin.listVerticals(actor)).filter((v) => v.ownerTenant === query.tenantId);
-        let considered = all;
-        if (query.vertical !== undefined) {
-            const slug = await resolveVerticalId(c, query.vertical);
-            considered = all.filter((v) => v.slug === slug);
-            // Non-ownership reads as absence (K-3), exactly like the per-slug registry reads.
-            if (considered.length === 0)
-                throw new ControlPlaneError(404, 'not found');
-        }
-        // Cap the per-vertical fan-out and REPORT the cut — truncating silently is the
-        // defect the egress route names (#859), and the same posture holds here.
-        const MAX_VERTICALS = 50;
-        const verticalsTruncated = considered.length > MAX_VERTICALS;
+    // Cap the per-vertical fan-out and REPORT the cut — truncating silently is the
+    // defect the egress route names (#859), and the same posture holds here.
+    const MAX_SERVICE_REF_VERTICALS = 50;
+    /**
+     * Every Cloudflare service ref a tenant's own verticals deploy, mapped to the
+     * signals dimensions it stamps. The ONE implementation of "what a service ref
+     * means", and now also of "which service refs are this tenant's" — the two are the
+     * same enumeration, and #977's observability narrowing must not be a second copy of
+     * it that can drift when a third ref scheme appears.
+     */
+    const tenantServiceRefs = async (actor, tenantId, slug) => {
+        const all = (await admin.listVerticals(actor)).filter((v) => v.ownerTenant === tenantId);
+        const considered = slug === undefined ? all : all.filter((v) => v.slug === slug);
+        const verticalsTruncated = considered.length > MAX_SERVICE_REF_VERTICALS;
         const entries = [];
-        for (const v of considered.slice(0, MAX_VERTICALS)) {
+        for (const v of considered.slice(0, MAX_SERVICE_REF_VERTICALS)) {
             const versions = await admin.listVersions(actor, v.slug);
             const labelOf = new Map(versions.map((ver) => [ver.id, ver.version]));
             // The stable serving script — where real traffic lands. Its version is the
@@ -3022,6 +3271,44 @@ export function createControlPlaneApi(options) {
                 }));
             }
         }
+        return { entries, verticalsTruncated, matched: considered.length };
+    };
+    /**
+     * The service names a CONFINED principal may see telemetry for — its tenant's own,
+     * and nothing else (#977).
+     *
+     * `null` means unconfined (staff): no narrowing, the fleet view the console reads.
+     * A truncated enumeration throws rather than narrowing to a partial set: a chart
+     * missing a vertical reads as "that vertical had no traffic", and inventing silence
+     * is the one answer an observability route must never give.
+     */
+    const ownedServices = async (p, actor) => {
+        const pin = confinedTenant(p);
+        if (pin === null)
+            return null;
+        const refs = await tenantServiceRefs(actor, pin);
+        if (refs.verticalsTruncated) {
+            throw new ControlPlaneError(503, 'service map truncated: too many verticals to resolve in one answer');
+        }
+        return new Set(refs.entries.map((e) => e.service));
+    };
+    app.get('/service-refs', async (c) => {
+        const p = c.get('principal');
+        // The /ops-failures forced-filter pattern: a confined principal's tenant comes from
+        // the principal, never the query. A staff/service caller must SAY whose view it
+        // wants — answering fleet-wide on a forgotten param would leak, so refuse.
+        const tenantId = confinedTenant(p) ?? c.req.query('tenantId');
+        if (!tenantId)
+            throw new ControlPlaneError(400, 'tenantId is required');
+        const query = z
+            .object({ tenantId: tenantIdSchema, vertical: z.string().optional() })
+            .parse({ tenantId, vertical: c.req.query('vertical') });
+        const actor = c.get('actor');
+        const slug = query.vertical === undefined ? undefined : await resolveVerticalId(c, query.vertical);
+        const { entries, verticalsTruncated, matched } = await tenantServiceRefs(actor, query.tenantId, slug);
+        // Non-ownership reads as absence (K-3), exactly like the per-slug registry reads.
+        if (slug !== undefined && matched === 0)
+            throw new ControlPlaneError(404, 'not found');
         return c.json({ entries, verticalsTruncated });
     });
     app.post('/verticals', async (c) => {
@@ -3049,7 +3336,7 @@ export function createControlPlaneApi(options) {
         const slug = await resolveVerticalId(c, c.req.param('slug'));
         // A builder reading a vertical it does not own gets 404 — indistinguishable from
         // absent, the same fail-closed reflex K-3 uses for a cross-tenant scope.
-        if (p.kind === 'builder' && (await ownerOf(p.actor, slug)) !== p.tenantId) {
+        if (await notOwned(p, slug)) {
             return c.json({ error: 'not found' }, 404);
         }
         const page = pageParams(c);
@@ -3066,7 +3353,7 @@ export function createControlPlaneApi(options) {
         }
         const p = c.get('principal');
         const slug = await resolveVerticalId(c, input.verticalSlug);
-        if (p.kind === 'builder' && (await ownerOf(p.actor, slug)) !== p.tenantId) {
+        if (await notOwned(p, slug)) {
             return c.json({ error: 'forbidden' }, 403);
         }
         input.verticalSlug = slug;
@@ -3083,7 +3370,7 @@ export function createControlPlaneApi(options) {
     app.get('/verticals/:slug/versions/:id/registry', async (c) => {
         const p = c.get('principal');
         const slug = await resolveVerticalId(c, c.req.param('slug'));
-        if (p.kind === 'builder' && (await ownerOf(p.actor, slug)) !== p.tenantId) {
+        if (await notOwned(p, slug)) {
             return c.json({ error: 'not found' }, 404);
         }
         const json = await admin.versionManifest(c.get('actor'), slug, c.req.param('id'));
@@ -3100,7 +3387,7 @@ export function createControlPlaneApi(options) {
     app.get('/verticals/:slug/versions/:id/schedules', async (c) => {
         const p = c.get('principal');
         const slug = await resolveVerticalId(c, c.req.param('slug'));
-        if (p.kind === 'builder' && (await ownerOf(p.actor, slug)) !== p.tenantId) {
+        if (await notOwned(p, slug)) {
             return c.json({ error: 'not found' }, 404);
         }
         const json = await admin.versionManifest(c.get('actor'), slug, c.req.param('id'));
@@ -3123,7 +3410,7 @@ export function createControlPlaneApi(options) {
     app.get('/verticals/:slug/versions/:id/flow', async (c) => {
         const p = c.get('principal');
         const slug = await resolveVerticalId(c, c.req.param('slug'));
-        if (p.kind === 'builder' && (await ownerOf(p.actor, slug)) !== p.tenantId) {
+        if (await notOwned(p, slug)) {
             return c.json({ error: 'not found' }, 404);
         }
         const json = await admin.versionManifest(c.get('actor'), slug, c.req.param('id'));
@@ -3145,7 +3432,7 @@ export function createControlPlaneApi(options) {
     app.get('/verticals/:slug/versions/:id/model', async (c) => {
         const p = c.get('principal');
         const slug = await resolveVerticalId(c, c.req.param('slug'));
-        if (p.kind === 'builder' && (await ownerOf(p.actor, slug)) !== p.tenantId) {
+        if (await notOwned(p, slug)) {
             return c.json({ error: 'not found' }, 404);
         }
         const json = await admin.versionManifest(c.get('actor'), slug, c.req.param('id'));
@@ -3164,7 +3451,7 @@ export function createControlPlaneApi(options) {
     app.get('/verticals/:slug/versions/:id/assets', async (c) => {
         const p = c.get('principal');
         const slug = await resolveVerticalId(c, c.req.param('slug'));
-        if (p.kind === 'builder' && (await ownerOf(p.actor, slug)) !== p.tenantId) {
+        if (await notOwned(p, slug)) {
             return c.json({ error: 'not found' }, 404);
         }
         const json = await admin.versionManifest(c.get('actor'), slug, c.req.param('id'));
@@ -3189,7 +3476,7 @@ export function createControlPlaneApi(options) {
     app.post('/verticals/:slug/publish-request', async (c) => {
         const p = c.get('principal');
         const slug = await resolveVerticalId(c, c.req.param('slug'));
-        if (p.kind === 'builder' && (await ownerOf(p.actor, slug)) !== p.tenantId) {
+        if (await notOwned(p, slug)) {
             return c.json({ error: 'not found' }, 404);
         }
         await admin.requestPublish(c.get('actor'), slug);
@@ -3233,18 +3520,27 @@ export function createControlPlaneApi(options) {
         await admin.setVerticalEmailSender(c.get('actor'), slug, granted);
         return c.json({ slug, emailSender: granted });
     });
-    // Delete a vertical + its versions and channels (staff-only, same confinement).
-    // Refused below the seam while any scope is still bound — surfaces as a 4xx via
-    // mapError, naming the count. Dispatch scripts become orphans for cleanup (#248).
+    // Delete a vertical + its versions and channels. Refused below the seam while any
+    // scope is still bound — surfaces as a 4xx via mapError, naming the count. Dispatch
+    // scripts become orphans for cleanup (#248).
+    //
+    // A CONFINED principal deletes only what its tenant owns (#977). This route had no
+    // ownership check at all, because it was staff-only and the dashboard — which calls
+    // it — checked ownership in its own worker before asking. That check was the whole
+    // guard on a destructive registry write, and it lived in the caller.
     app.delete('/verticals/:slug', async (c) => {
         const slug = c.req.param('slug');
+        // 404, not 403: a slug this tenant does not own reads as absent, the same
+        // existence-hiding reflex the registry's other per-slug reads use (K-3).
+        if (await notOwned(c.get('principal'), slug))
+            return c.json({ error: 'not found' }, 404);
         await admin.deleteVertical(c.get('actor'), slug);
         return c.json({ slug, deleted: true });
     });
     app.get('/verticals/:slug/channels', async (c) => {
         const p = c.get('principal');
         const slug = await resolveVerticalId(c, c.req.param('slug'));
-        if (p.kind === 'builder' && (await ownerOf(p.actor, slug)) !== p.tenantId) {
+        if (await notOwned(p, slug)) {
             return c.json({ error: 'not found' }, 404);
         }
         const page = pageParams(c);
@@ -3262,7 +3558,7 @@ export function createControlPlaneApi(options) {
             return c.json({ error: `channel '${rawChannel}' is retired — only 'prod' has history` }, 400);
         }
         const channel = channelName.parse(rawChannel);
-        if (p.kind === 'builder' && (await ownerOf(p.actor, slug)) !== p.tenantId) {
+        if (await notOwned(p, slug)) {
             return c.json({ error: 'not found' }, 404);
         }
         const page = pageParams(c);
@@ -3432,17 +3728,25 @@ export function createControlPlaneApi(options) {
             }, 400);
         }
         const channel = channelName.parse(rawChannel);
-        if (p.kind === 'builder') {
-            // A builder promotes only verticals it owns — and prod only while the vertical
-            // is PRIVATE (not listed). A private vertical's blast radius is the owning
-            // tenant itself, and dev/staging already run the same bundle in the same
-            // sandbox, so a staff prod gate there protected nothing; it returns the moment
-            // the audience widens (publish flips `listed`, and prod becomes staff-only
-            // again — the trust boundary marketplace-publish.md §2 draws).
+        const promotePin = confinedTenant(p);
+        if (promotePin) {
+            // A confined principal promotes only verticals its tenant owns.
             const v = await verticalOf(p.actor, slug);
-            if (!v || v.ownerTenant !== p.tenantId)
+            if (!v || v.ownerTenant !== promotePin)
                 return c.json({ error: 'forbidden' }, 403);
-            if (channel === 'prod' && v.listed) {
+            // …and a BUILDER promotes prod only while the vertical is PRIVATE (not listed).
+            // A private vertical's blast radius is the owning tenant itself, and dev/staging
+            // already run the same bundle in the same sandbox, so a staff prod gate there
+            // protected nothing; it returns the moment the audience widens (publish flips
+            // `listed`, and prod becomes staff-only again — the trust boundary
+            // marketplace-publish.md §2 draws).
+            //
+            // Deliberately NOT widened to #977's tenant credential: promoting a vertical the
+            // tenant has published is something the dashboard could already do over the
+            // service token, and this issue narrows WHOSE data a credential reaches, not what
+            // a tenant may do inside its own. Taking the gate away from them here would be a
+            // capability change hidden in a security fix.
+            if (p.kind === 'builder' && channel === 'prod' && v.listed) {
                 return c.json({ error: 'promotion to prod is staff-only for a listed vertical' }, 403);
             }
         }
@@ -3491,7 +3795,8 @@ export function createControlPlaneApi(options) {
         // installed their vertical. The rest is a count, which tells them the fleet was covered
         // without naming who is in it.
         const p2 = c.get('principal');
-        const visible = p2.kind === 'builder' ? backfill.minted.filter((m) => m.tenantId === p2.tenantId) : backfill.minted;
+        const reportPin = confinedTenant(p2);
+        const visible = reportPin ? backfill.minted.filter((m) => m.tenantId === reportPin) : backfill.minted;
         const otherTenants = new Set(backfill.minted.filter((m) => !visible.includes(m)).map((m) => m.tenantId)).size;
         return c.json({
             ...promoted,
@@ -3837,10 +4142,18 @@ export function createControlPlaneApi(options) {
         return c.json(namespacesForScript(await options.doNamespaces.list(), script));
     });
     // -- observability (design/observability.md §4.1) --------------------------
-    // Proxied Cloudflare-native reads: the console's fleet view and (later, owner-
-    // narrowed) the dashboard's builder view. STAFF-ONLY — not in BUILDER_ROUTES; see
-    // the option's doc for why. Tier-3 numbers (master-plan §5.3): sampled, approximate,
+    // Proxied Cloudflare-native reads at SCRIPT grain: one deployed unit, which serves
+    // every tenant that installed the vertical. Not in BUILDER_ROUTES — a builder reads
+    // the tenant grain below. Tier-3 numbers (master-plan §5.3): sampled, approximate,
     // never money.
+    //
+    // For a CONFINED principal (#977) the answer is narrowed to the services its own
+    // tenant's verticals deploy, HERE. These three were the sharpest edge of this issue:
+    // they are fleet reads by construction, `/observability/metrics` takes no filter at
+    // all, and the dashboard read them over the staff token and dropped other tenants'
+    // rows in its own worker. So every one of the fleet's numbers crossed the seam and
+    // the narrowing was a `.filter()` in the caller — the exact shape the issue is about,
+    // on the route where a mistake is least visible.
     app.get('/observability/metrics', async (c) => {
         if (!options.observability) {
             return c.json({ error: 'observability is not configured on this control plane' }, 501);
@@ -3848,7 +4161,12 @@ export function createControlPlaneApi(options) {
         const { hours } = z
             .object({ hours: z.coerce.number().int().min(1).max(72).default(24) })
             .parse({ hours: c.req.query('hours') });
-        return c.json(await options.observability.serviceMetrics({ hours }));
+        const owned = await ownedServices(c.get('principal'), c.get('actor'));
+        const rows = await options.observability.serviceMetrics({ hours });
+        // The route takes no service filter, so the narrowing is on the way OUT. It is the
+        // last step before the response either way: a row this tenant does not own must not
+        // leave the plane, whatever the backend was asked.
+        return c.json(owned ? rows.filter((r) => owned.has(r.service)) : rows);
     });
     // The bucketed twin (#1236): the series a chart plots, with the deploy markers
     // drawn on top by the caller (push instants are registry facts, not telemetry).
@@ -3863,7 +4181,19 @@ export function createControlPlaneApi(options) {
         // Repeats like /observability/logs' `service`: a caller narrows to the handful
         // it will plot, which is what keeps scripts × buckets bounded.
         const services = c.req.queries('service') ?? undefined;
-        return c.json(await options.observability.serviceMetricsSeries({ hours, services }));
+        const owned = await ownedServices(c.get('principal'), c.get('actor'));
+        if (!owned)
+            return c.json(await options.observability.serviceMetricsSeries({ hours, services }));
+        // A confined principal plots its own services: what it asked for, intersected —
+        // and its whole set when it asked for none, since an unfiltered ask here means
+        // "everything I may see", never "everything there is". Asking for nothing but
+        // foreign refs is an empty series, indistinguishable from services with no
+        // traffic (existence hiding, K-3).
+        const narrowed = (services ?? [...owned]).filter((s) => owned.has(s));
+        if (narrowed.length === 0)
+            return c.json([]);
+        const rows = await options.observability.serviceMetricsSeries({ hours, services: narrowed });
+        return c.json(rows.filter((r) => owned.has(r.service)));
     });
     app.get('/observability/logs', async (c) => {
         if (!options.observability) {
@@ -3872,7 +4202,14 @@ export function createControlPlaneApi(options) {
         // `service` repeats: one deployed unit per param, so a caller can ask for a
         // vertical's whole set (the dashboard's "all versions") in one query and get one
         // merged stream back. Capped because each extra service is another backend query.
-        const services = (c.req.queries('service') ?? []).filter((s) => s.length > 0);
+        const asked = (c.req.queries('service') ?? []).filter((s) => s.length > 0);
+        const ownedLogServices = await ownedServices(c.get('principal'), c.get('actor'));
+        // A confined principal reads only its own services' lines. An ask that names none
+        // of them answers `[]` rather than the fleet's stream — and an ask naming NOTHING
+        // is one such ask, because for this caller "no filter" cannot mean "everything".
+        const services = ownedLogServices ? asked.filter((s) => ownedLogServices.has(s)) : asked;
+        if (ownedLogServices && services.length === 0)
+            return c.json([]);
         const input = z
             .object({
             services: z.array(z.string().min(1).max(200)).max(20).optional(),
@@ -3909,7 +4246,7 @@ export function createControlPlaneApi(options) {
             return c.json({ error: 'tenant-grain observability is not configured on this control plane' }, 501);
         }
         const p = c.get('principal');
-        const tenantId = p.kind === 'builder' ? p.tenantId : c.req.query('tenantId');
+        const tenantId = confinedTenant(p) ?? c.req.query('tenantId');
         if (!tenantId)
             throw new ControlPlaneError(400, 'tenantId is required');
         const input = z
@@ -3940,7 +4277,7 @@ export function createControlPlaneApi(options) {
             return c.json({ error: 'bucketed tenant-grain observability is not configured on this control plane' }, 501);
         }
         const p = c.get('principal');
-        const tenantId = p.kind === 'builder' ? p.tenantId : c.req.query('tenantId');
+        const tenantId = confinedTenant(p) ?? c.req.query('tenantId');
         if (!tenantId)
             throw new ControlPlaneError(400, 'tenantId is required');
         const scopeIds = (c.req.queries('scopeId') ?? []).filter((s) => s.length > 0);
@@ -3960,7 +4297,7 @@ export function createControlPlaneApi(options) {
             return c.json({ error: 'tenant-grain observability is not configured on this control plane' }, 501);
         }
         const p = c.get('principal');
-        const tenantId = p.kind === 'builder' ? p.tenantId : c.req.query('tenantId');
+        const tenantId = confinedTenant(p) ?? c.req.query('tenantId');
         if (!tenantId)
             throw new ControlPlaneError(400, 'tenantId is required');
         const input = z
@@ -3970,6 +4307,15 @@ export function createControlPlaneApi(options) {
             vertical: z.string().min(1).max(200).optional(),
             level: z.enum(['log', 'info', 'warn', 'error', 'debug']).optional(),
             search: z.string().min(1).max(200).optional(),
+            // One call's lines (#1525): the id `invocationLog` stamps on its line and the
+            // events carry. The shape is `ulid()`'s — the only thing that mints it — so
+            // anything else is refused as a 400 rather than sent to the backend as a filter
+            // that can only match nothing, which reads as "no logs for that call".
+            // Tighter than the platform's id schemas (`ids.ts` accepts any Crockford
+            // character first): a 48-bit timestamp encodes to 10 characters whose first is
+            // 0–7, so `Z000…` is not a ULID however it looks. The contracts package exports
+            // no generic ULID schema to reuse — each id brands its own private copy.
+            invocationId: z.string().regex(/^[0-7][0-9A-HJKMNP-TV-Z]{25}$/, 'invocationId must be a ULID').optional(),
             hours: z.coerce.number().int().min(1).max(72).default(24),
             // The chart's time cursor (#1447): a window that ENDS in the past, which `hours`
             // cannot spell — it always ends at now.
@@ -4007,6 +4353,9 @@ export function createControlPlaneApi(options) {
             vertical: c.req.query('vertical') || undefined,
             level: c.req.query('level') || undefined,
             search: c.req.query('search') || undefined,
+            // Raw, not `|| undefined`: an EMPTY id is a caller bug, and folding it into
+            // "absent" would answer it with the tenant's whole log.
+            invocationId: c.req.query('invocationId'),
             hours: c.req.query('hours'),
             since: c.req.query('since') || undefined,
             until: c.req.query('until') || undefined,
@@ -4126,6 +4475,25 @@ export function createControlPlaneApi(options) {
         });
         return c.json({ token, tenantSlug: tenant.slug }, 201);
     });
+    // -- tenant tokens (tenant-token.ts, #977) ---------------------------------
+    // Mint the dashboard's tenant-scoped service credential. STAFF-ONLY, and this is
+    // the load-bearing part of the whole design: it is on NEITHER allowlist, so a
+    // tenant token can never mint another — not for a different tenant, not even for
+    // its own. The power to mint one for any tenant stays with the platform service
+    // credential, which is what a multi-tenant dashboard must hold and which is now the
+    // ONLY fleet-wide thing it does. Everything else it does is spent per tenant.
+    app.post('/tenant-tokens', async (c) => {
+        if (!options.tenantTokenSecret) {
+            return c.json({ error: 'tenant tokens are not configured on this control plane' }, 501);
+        }
+        const { tenantId } = z.object({ tenantId: tenantIdSchema }).parse(await c.req.json());
+        // Unlike the push-token mint, this does NOT require the tenant to exist: a token
+        // for a tenant with no directory row reaches nothing except creating that row
+        // (`POST /tenants`, body-pinned to the same id), which is the sign-up bootstrap.
+        // See the mint's docblock in tenant-token.ts.
+        const token = await mintTenantToken(options.tenantTokenSecret, { tenantId });
+        return c.json({ token }, 201);
+    });
     // -- the hostname map (§4.7, K-26) -----------------------------------------
     // Staff actions, PLUS a tenant-narrowed builder view (multi-surface exposure —
     // binding an EKA-style second surface is self-serve for the scope's own tenant).
@@ -4139,7 +4507,8 @@ export function createControlPlaneApi(options) {
     // (which would confirm the name is taken by someone).
     const tenantHostname = async (c, name) => {
         const p = c.get('principal');
-        const filter = p.kind === 'builder' ? { tenantId: p.tenantId } : {};
+        const pin = confinedTenant(p);
+        const filter = pin ? { tenantId: pin } : {};
         return (await admin.listHostnames(c.get('actor'), filter)).find((h) => h.hostname === name.toLowerCase());
     };
     app.get('/hostnames', async (c) => {
@@ -4148,10 +4517,14 @@ export function createControlPlaneApi(options) {
             tenantId: c.req.query('tenantId'),
             scopeId: c.req.query('scopeId'),
         });
-        // A builder's list is ALWAYS its own tenant's — the query may narrow further
-        // (scopeId) but never widen; a foreign tenantId in the query loses silently.
-        if (p.kind === 'builder')
-            filter.tenantId = p.tenantId;
+        // A CONFINED principal's list is ALWAYS its own tenant's — the query may narrow
+        // further (scopeId) but never widen; a foreign tenantId in the query loses
+        // silently. `confinedTenant`, not a kind check: the dashboard's tenant token is
+        // confined too (#977), and reading this as "builders only" is what left it
+        // paging the whole fleet's bindings.
+        const hostnamePin = confinedTenant(p);
+        if (hostnamePin)
+            filter.tenantId = hostnamePin;
         const page = pageParams(c);
         const entries = await admin.listHostnames(c.get('actor'), { ...filter, ...page });
         return c.json(pageOf(entries, page.limit, (h) => h.hostname));
@@ -4223,12 +4596,19 @@ export function createControlPlaneApi(options) {
     app.post('/hostnames', async (c) => {
         const p = c.get('principal');
         const input = bindHostnameBody.parse(await c.req.json());
-        if (p.kind === 'builder') {
-            // The body names the tenant the binding lands under; a builder may only name
-            // its own (the adapter then verifies the scope belongs to it, K-3).
-            if (input.tenantId !== p.tenantId)
+        const bindPin = confinedTenant(p);
+        if (bindPin) {
+            // The body names the tenant the binding lands under; a confined principal may
+            // only name its own (the adapter then verifies the scope belongs to it, K-3).
+            if (input.tenantId !== bindPin)
                 return c.json({ error: 'forbidden' }, 403);
+        }
+        if (p.kind === 'builder') {
             // The region column is an EU-residency claim (K-30) — never builder-suppliable.
+            // Left as a BUILDER rule rather than a confinement one: the dashboard binds the
+            // platform's own hostnames for a tenant and has always been able to say where
+            // the binding sits, so widening this to #977's credential would be a behaviour
+            // change wearing a security fix's clothes.
             if (input.region !== null) {
                 return c.json({ error: 'region is derived from the scope, not chosen on a binding' }, 403);
             }
@@ -4255,7 +4635,7 @@ export function createControlPlaneApi(options) {
     // re-affirm. Idempotent and safe to hammer — it just reflects Cloudflare's current state.
     app.post('/hostnames/:hostname/verify', async (c) => {
         const name = c.req.param('hostname');
-        if (c.get('principal').kind === 'builder' && !(await tenantHostname(c, name))) {
+        if (confinedTenant(c.get('principal')) !== null && !(await tenantHostname(c, name))) {
             return c.json({ error: `unknown hostname: ${name.toLowerCase()}` }, 404);
         }
         const row = await hostnameRow(c, name);
@@ -4269,7 +4649,7 @@ export function createControlPlaneApi(options) {
         // Not path-parsed through the schema: a hostname is the path segment here, and
         // `setHostnameStatus` normalizes and 404s an unknown one below the seam.
         const name = c.req.param('hostname');
-        if (c.get('principal').kind === 'builder' && !(await tenantHostname(c, name))) {
+        if (confinedTenant(c.get('principal')) !== null && !(await tenantHostname(c, name))) {
             return c.json({ error: `unknown hostname: ${name.toLowerCase()}` }, 404);
         }
         await admin.setHostnameStatus(c.get('actor'), name, status, note);
@@ -4283,7 +4663,7 @@ export function createControlPlaneApi(options) {
     // idempotency yields to existence hiding at the tenant boundary.
     app.delete('/hostnames/:hostname', async (c) => {
         const name = c.req.param('hostname');
-        if (c.get('principal').kind === 'builder' && !(await tenantHostname(c, name))) {
+        if (confinedTenant(c.get('principal')) !== null && !(await tenantHostname(c, name))) {
             return c.json({ error: `unknown hostname: ${name.toLowerCase()}` }, 404);
         }
         // Release the Cloudflare custom hostname before dropping the row, or the CF object
@@ -4556,8 +4936,9 @@ export function createControlPlaneApi(options) {
         const v = await verticalOf(c.get('actor'), slug);
         if (!v)
             return { error: `unknown vertical '${slug}'`, status: 404 };
-        if (p.kind === 'builder' && v.ownerTenant !== p.tenantId)
+        if (confinedTenant(p) !== null && v.ownerTenant !== confinedTenant(p)) {
             return { error: 'forbidden', status: 403 };
+        }
         if (v.ownerTenant === null)
             return { error: `vertical '${slug}' has no owner tenant`, status: 409 };
         return { tenantId: v.ownerTenant, slug };
@@ -4708,6 +5089,47 @@ export function createControlPlaneApi(options) {
         });
         return c.body(null, 204);
     });
+    // -- the schedule kill switch (#1666) --------------------------------------
+    //
+    // One module's scheduled work on ONE scope, off and back on — `revokeFromSystem` and
+    // `restoreToSystem`. Module-wide on the scope by design (the body schema says why a
+    // per-permission switch is either inexpressible or noisy), `reason` required, and both
+    // halves audited on the admin log — intent first, then outcome, on every attempt,
+    // paired by the `operationId` the route answers with. Idempotent: a repeat answers
+    // `changed: false`, and is audited all the same.
+    //
+    // **Restore is the lever; a grant is not.** While the switch is off, a `grantToSystem`
+    // for the module is refused (409) and a reconcile seats none of its system grants, not
+    // even a newly declared one — only the POST below turns it back on, and it gives back
+    // exactly what the DELETE took. A switch that the next deploy, or a stray grant,
+    // silently undid would not be a kill switch.
+    //
+    // For a HOSTED scope the grants live in the vertical's deployment, and the host's
+    // `systemSwitchDelegation` moves the switch there; a deployment built before the far
+    // end existed answers 501 "redeploy", nothing is switched, and the log shows the intent
+    // and its failure.
+    //
+    // **Staff and the platform service token ONLY.** Builders are refused by BUILDER_ROUTES
+    // (default-deny). A tenant-scoped credential (#977) would pass this path's `/tenants/<pin>`
+    // confinement, so the handler refuses it by `confinedTenant`: the switched party must
+    // not be able to switch itself back on, and the dashboard has no flow that pulls it.
+    const switchScheduleRoute = (to) => async (c) => {
+        if (confinedTenant(c.get('principal')) !== null) {
+            return c.json({ error: 'forbidden: the schedule switch is staff-only' }, 403);
+        }
+        const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
+        const scopeId = scopeIdSchema.parse(c.req.param('scopeId'));
+        const body = systemSwitchBody.parse(await c.req.json());
+        const actor = c.get('actor');
+        // K-3 first, so a scope of another tenant reads as absent before anything is reached.
+        if (!(await admin.getScopeRecord(actor, tenantId, scopeId))) {
+            return c.json({ error: `unknown scope for tenant: (${tenantId}, ${scopeId})` }, 404);
+        }
+        const input = { moduleId: body.moduleId, node: { tenantId, scopeId }, reason: body.reason };
+        return c.json(to === 'off' ? await admin.revokeFromSystem(actor, input) : await admin.restoreToSystem(actor, input));
+    };
+    app.delete('/tenants/:tenantId/scopes/:scopeId/system-grants', switchScheduleRoute('off'));
+    app.post('/tenants/:tenantId/scopes/:scopeId/system-grants', switchScheduleRoute('on'));
     // Orgs: the portal-customer grouping (§4.1). Creating one mints no permission —
     // members reach what the org was GRANTED, and granting stays off this surface.
     app.post('/tenants/:tenantId/orgs', async (c) => {
@@ -4746,7 +5168,7 @@ export function createControlPlaneApi(options) {
     app.get('/ops-failures', async (c) => {
         const p = c.get('principal');
         const filter = opsFailuresQuery.parse({
-            tenantId: p.kind === 'builder' ? p.tenantId : c.req.query('tenantId'),
+            tenantId: confinedTenant(p) ?? c.req.query('tenantId'),
             scopeId: c.req.query('scopeId'),
             vertical: c.req.query('vertical'),
             version: c.req.query('version'),
@@ -4771,7 +5193,7 @@ export function createControlPlaneApi(options) {
             kind: c.req.query('kind'),
             unit: c.req.query('unit'),
             outcome: c.req.query('outcome'),
-            tenantId: p.kind === 'builder' ? p.tenantId : c.req.query('tenantId'),
+            tenantId: confinedTenant(p) ?? c.req.query('tenantId'),
             scopeId: c.req.query('scopeId'),
             vertical: c.req.query('vertical'),
             connectionId: c.req.query('connectionId'),