@substrat-run/control-plane-api 0.118.0 → 0.120.0

package/dist/api.js

@@ -1,10 +1,12 @@
+import { resolveObservabilityWindow } from './observability-window.js';
 import { Hono } from 'hono';
-import { parseHostname, withLabel, RESERVED_LABEL_SEPARATOR, adminAction, ASSET_PART_PREFIX, assetHash, channelName, connectionActivity, connectionActivitySource, connectionCredential, connectionFilter, connectionProbe, createTenantInput, entitlementGrantInput, hostname as hostnameSchema, hostnameRegion, hostnameStatus, identityLink, errorCode, issueStatus, issueStatusInput, listPageQuery, matchesOutboundHost, pageOf, platformRequestFilter, principalId as principalIdSchema, promotionAcknowledgement, provisionableJurisdiction, publishVersionInput, queryScopeInput, readScopeTableInput, entityHistoryInput, eventFacetInput, eventCauseInput, eventEffectsInput, invocationEventsInput, deadLettersInput, delegatedReadParams, createOrgInput, roleKey as roleKeySchema, systemSwitch, DEFAULT_DENIAL_LIMIT, DENIAL_LIMIT_MAX, denialGroupBy, registerVerticalInput, serviceDimensions, sweepRunKind, sweepRunOutcome, scopeDump, dataSubjectId as dataSubjectIdSchema, scopeId as scopeIdSchema, scopeStatus, storageShape, surfaceName, tenantId as tenantIdSchema, tenantStatus, versionOrigin, z, PROBLEM_CONTENT_TYPE, toProblem, redrainEventsInput, REDRAIN_BATCH, } from '@substrat-run/contracts';
+import { parseHostname, withLabel, RESERVED_LABEL_SEPARATOR, adminAction, ASSET_PART_PREFIX, assetHash, channelName, connectionActivity, connectionActivitySource, connectionCredential, connectionFilter, connectionHealthState, connectionProvider, connectorDispatchKind, PROVISION_SIBLING_KIND, ARCHIVE_SCOPE_KIND, PROVISION_TENANT_KIND, SET_ENTITLEMENTS_KIND, MODEL_USAGE_KIND, SWEEP_RUNS_KIND, toConnectionHealthEntry, CONNECTION_EXPIRY_WARNING_DAYS, CONNECTION_STALE_DAYS, connectionProbe, createTenantInput, entitlementGrantInput, hostname as hostnameSchema, hostnameRegion, hostnameStatus, identityLink, errorCode, issueStatus, issueStatusInput, listPageQuery, matchesOutboundHost, pageOf, platformRequestFilter, principalId as principalIdSchema, promotionAcknowledgement, provisionableJurisdiction, publishVersionInput, queryScopeInput, readScopeTableInput, entityHistoryInput, eventFacetInput, eventCauseInput, eventEffectsInput, invocationEventsInput, deadLettersInput, delegatedReadParams, createOrgInput, roleKey as roleKeySchema, peerSwitch, systemSwitch, DEFAULT_DENIAL_LIMIT, DENIAL_LIMIT_MAX, denialGroupBy, registerVerticalInput, serviceDimensions, sweepRunKind, sweepRunOutcome, scopeDump, dataSubjectId as dataSubjectIdSchema, scopeId as scopeIdSchema, scopeStatus, storageShape, surfaceName, tenantId as tenantIdSchema, tenantStatus, versionOrigin, z, PROBLEM_CONTENT_TYPE, toProblem, redrainEventsInput, REDRAIN_BATCH, } from '@substrat-run/contracts';
 import { attributeFailure } from './failure-attribution.js';
 import { migrationProgress, ulid } from '@substrat-run/kernel';
 import { TENANT_HEADER, confinedTenant } from './auth.js';
 import { mintTenantToken } from './tenant-token.js';
 import { connectionGrantsForScope } from './vertical-client.js';
+import { oidcCallbackUrl, retireClientsOfReapedScope, wirePreviewAuth } from './preview-auth.js';
 import { versionReachedAt } from './scope-deployment.js';
 import { reconcileConnectionGrants } from './connection-grants.js';
 import { ConnectionRelayError, relayConnectionUpsert } from './connection-relay.js';
@@ -298,6 +300,32 @@ const opsFailuresQuery = z.object({
     // Bounded by default exactly as /admin-log, and for the same reason.
     ...listPageQuery.shape,
 });
+// The fleet connection-health read (#1690). `status` is the DERIVED health state, not the
+// stored `Connection.status`. Paged like every HTTP list read; the cursor is a connection id.
+const connectionHealthQuery = z.object({
+    status: connectionHealthState.optional(),
+    provider: connectionProvider.optional(),
+    tenantId: tenantIdSchema.optional(),
+    // Free text, matched case-insensitively against the row's own fields BEFORE paging, so
+    // a search finds a row on any page rather than only the ones a console has loaded.
+    q: z.string().trim().min(1).max(200).optional(),
+    limit: listPageQuery.shape.limit,
+    cursor: listPageQuery.shape.cursor,
+    // The walk is ascending by connection id, only. A `desc` is refused rather than
+    // silently ignored: a caller would otherwise read ascending pages as descending ones.
+    order: z.literal('asc').optional(),
+});
+/** The dead-letter window (#1690) — a week, the same horizon as the stale window. */
+const CONNECTOR_DEAD_LETTER_WINDOW_DAYS = 7;
+/** The per-provider read bound: past it the count is a floor, and says so (`capped`). */
+export const CONNECTOR_DEAD_LETTER_CAP = 200;
+/**
+ * `/platform-requests/backlog` (#1690 §2) rides the SAME window and per-kind bound as the
+ * connector dead-letter count above — one incident horizon, one honestly-capped read, for
+ * every fleet-wide "how much has given up lately" number this API answers.
+ */
+const PLATFORM_REQUEST_BACKLOG_WINDOW_DAYS = CONNECTOR_DEAD_LETTER_WINDOW_DAYS;
+const PLATFORM_REQUEST_BACKLOG_CAP = CONNECTOR_DEAD_LETTER_CAP;
 // The issues read (#1233). No cursor by design: grouping IS the compression —
 // cardinality is the number of distinct failure shapes — and `limit` bounds it.
 const issuesQuery = z.object({
@@ -343,6 +371,8 @@ const tenantRoleAssignmentBody = z
  * spells out; `.strict()` so a body that tried to name one is refused, not stripped.
  */
 const systemSwitchBody = systemSwitch.pick({ moduleId: true, reason: true }).strict();
+/** The peer kill switch's body (#1706) — the tenant and scope come from the path. */
+const peerSwitchBody = peerSwitch.pick({ vertical: true, reason: true }).strict();
 /** A new org under an addressed tenant (#1343) — `tenantId` comes from the path. */
 const tenantOrgBody = createOrgInput.omit({ tenantId: true });
 const sweepRunsQuery = z.object({
@@ -1031,6 +1061,180 @@ export function createControlPlaneApi(options) {
         });
         return c.json(await admin.listConnections(c.get('actor'), filter));
     });
+    // -- connection health, fleet-wide (#1690) -----------------------------------
+    //
+    // The staff console's Health → Connections read: every tenant's connections, each with
+    // its derived health (`deriveConnectionHealth`, contracts) and refresh-expiry warning,
+    // plus connector dead letters counted per provider. Read-only, and staff/service only
+    // by construction: it is neither in BUILDER_ROUTES nor in the tenant credential's
+    // TENANT_ROUTES, so both default-deny it — and it lives outside `/tenants/:id` so no
+    // future tenant-path pin can reach it by pattern.
+    //
+    // Secrets: the read goes through the AUDITED `listConnections` (the access log records
+    // the filter and a count, never rows — §3.4's rule for the log), whose `Connection`
+    // cannot carry a credential; and each row is then projected through
+    // `toConnectionHealthEntry`'s allow-list rather than spread, so a widened row upstream
+    // does not ride out to a browser.
+    //
+    // Cost: `listConnections` has no page of its own, so this loads the (filtered) fleet's
+    // connection rows on every request and pages in memory. Unfiltered that is a full scan
+    // of `_substrat_connections`, a table in the low hundreds today. Past ~2,000 live rows
+    // (a directory read and a response-time cost paid on every console load) it wants a
+    // keyset page pushed into the adapter, which the derived-status filter makes
+    // non-trivial, because health is not a column. No cache: a stale health view is the
+    // one thing this page must not show. Filter-then-page is deliberate: paging first would hand back a short or
+    // empty page while a cursor still existed.
+    app.get('/connections/health', async (c) => {
+        const q = connectionHealthQuery.parse({
+            status: c.req.query('status'),
+            provider: c.req.query('provider'),
+            tenantId: c.req.query('tenantId'),
+            q: c.req.query('q'),
+            limit: c.req.query('limit'),
+            cursor: c.req.query('cursor'),
+            order: c.req.query('order'),
+        });
+        const now = new Date();
+        const rows = await admin.listConnections(c.get('actor'), {
+            ...(q.tenantId ? { tenantId: q.tenantId } : {}),
+            ...(q.provider ? { provider: q.provider } : {}),
+        });
+        const all = rows.map((r) => toConnectionHealthEntry(r, now)).sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
+        const summary = {
+            total: all.length,
+            healthy: 0,
+            erroring: 0,
+            stale: 0,
+            'never-used': 0,
+            expiring: 0,
+            expired: 0,
+        };
+        for (const e of all) {
+            summary[e.health] += 1;
+            if (e.expiryWarning === 'soon')
+                summary.expiring += 1;
+            if (e.expiryWarning === 'expired')
+                summary.expired += 1;
+        }
+        // #1716: ids are Crockford-base32 ULIDs, drawn from a small alphabet that a short
+        // human needle collides with by chance (`a2` inside a random 16-char tail, ~1.5%
+        // of the time) — worse, two ids minted in the same millisecond share a monotonic
+        // prefix, so one hit there matches every row born that millisecond. The console's
+        // box promises "provider, vertical, account, or error" (ConnectionsHealth.tsx) —
+        // never ids — so only THOSE fields get substring matching. A tenant already has the
+        // exact `?tenantId=` filter, so `e.tenantId` gets no needle match at all. Pasting a
+        // connection id still finds its row: `q` equal to the WHOLE id, case-insensitively,
+        // is an exact match a fragment can never collide with.
+        const needle = q.q?.toLowerCase();
+        const matching = all.filter((e) => (!q.status || e.health === q.status) &&
+            (!needle ||
+                e.id.toLowerCase() === needle ||
+                [e.provider, e.vertical, e.label, e.externalAccountRef, e.lastError].some((f) => f?.toLowerCase().includes(needle))));
+        const page = pageSlice(matching, { limit: q.limit, cursor: q.cursor }, (e) => e.id);
+        // Connector dead letters. A hosted vertical is CP-less, so its connector deliveries
+        // run as `connector:<provider>` platform intents, and a terminal or given-up intent
+        // lands an ops-failure row (platform-drain.ts) — directory-side, so this is one
+        // bounded query per provider rather than a walk over every scope. What it cannot see:
+        // an IN-PROCESS executor drain's dead letters (a host that runs connectors itself —
+        // dev, self-host), which journal only in the scope.
+        const since = new Date(now.getTime() - CONNECTOR_DEAD_LETTER_WINDOW_DAYS * 86_400_000).toISOString();
+        const providers = q.provider ? [q.provider] : [...new Set(rows.map((r) => r.provider))].sort();
+        const deadLetters = await Promise.all(providers.map(async (provider) => {
+            const found = await admin.listOpsFailures(c.get('actor'), {
+                operation: `intent.${connectorDispatchKind(provider)}`,
+                ...(q.tenantId ? { tenantId: q.tenantId } : {}),
+                since,
+                limit: CONNECTOR_DEAD_LETTER_CAP + 1,
+            });
+            return {
+                provider,
+                count: Math.min(found.length, CONNECTOR_DEAD_LETTER_CAP),
+                capped: found.length > CONNECTOR_DEAD_LETTER_CAP,
+            };
+        }));
+        const body = {
+            ...page,
+            summary,
+            deadLetters,
+            deadLettersSince: since,
+            staleAfterDays: CONNECTION_STALE_DAYS,
+            expiryWarningDays: CONNECTION_EXPIRY_WARNING_DAYS,
+            asOf: now.toISOString(),
+        };
+        return c.json(body);
+    });
+    // -- connector calls, fleet-wide, over time (#1691) ---------------------------
+    //
+    // The trend behind the health line above: calls and errors per provider, bucketed,
+    // from the connector-call dataset the host writes on every settled call (kernel
+    // `connector-calls.ts`). Staff/service only by the same construction as
+    // `/connections/health` — in neither BUILDER_ROUTES nor TENANT_ROUTES, so both
+    // credentials default-deny it. 501 when the reader has no dataset named: an empty
+    // series would draw as "no calls". Up to a week, because "is this provider getting
+    // worse this week" is the question it exists for.
+    app.get('/connections/calls', async (c) => {
+        if (!options.observability?.connectorCallsSeries) {
+            return c.json({ error: 'connector-call analytics are not configured on this control plane' }, 501);
+        }
+        const input = z
+            .object({
+            hours: z.coerce.number().int().min(1).max(168).default(24),
+            // A provider slug — and the character set the AE literal accepts, so a bad one is
+            // this route's 400 rather than the reader's 500.
+            provider: z.string().regex(/^[A-Za-z0-9_\-.]{1,64}$/).optional(),
+        })
+            .parse({ hours: c.req.query('hours'), provider: c.req.query('provider') || undefined });
+        const buckets = await options.observability.connectorCallsSeries(input);
+        return c.json({ hours: input.hours, buckets });
+    });
+    // -- platform-request backlog, fleet-wide (#1690 §2) -----------------------
+    //
+    // "Is the platform's own intent drain keeping up" — composed from the SAME ops-failure
+    // record the dead-letter count above reads, generalized from one provider family to
+    // every known intent kind. Staff/service only by the same construction as
+    // `/connections/health` — absent from BUILDER_ROUTES and TENANT_ROUTES, so both
+    // credentials default-deny it.
+    //
+    // This is a count of terminal FAILURES, never a queue depth (see `platformRequestBacklog`'s
+    // doc): a still-`pending` intent lives in the vertical's own scope DO and nothing here
+    // walks the fleet to find it. A `0` means "nothing has given up lately", not "nothing is
+    // waiting" — the console tile is worded to keep that distinction, not just this comment.
+    app.get('/platform-requests/backlog', async (c) => {
+        const now = new Date();
+        const since = new Date(now.getTime() - PLATFORM_REQUEST_BACKLOG_WINDOW_DAYS * 86_400_000).toISOString();
+        // The fixed kinds the platform itself registers, plus one `connector:<provider>` kind
+        // per provider that has (or had) a live connection anywhere in the fleet — the same
+        // derivation the dead-letter count above uses, since that kind only exists for a
+        // provider some connection actually names.
+        const providers = [...new Set((await admin.listConnections(c.get('actor'), { includeRevoked: true })).map((r) => r.provider))];
+        const kinds = [
+            PROVISION_SIBLING_KIND,
+            ARCHIVE_SCOPE_KIND,
+            PROVISION_TENANT_KIND,
+            SET_ENTITLEMENTS_KIND,
+            MODEL_USAGE_KIND,
+            SWEEP_RUNS_KIND,
+            ...providers.map(connectorDispatchKind),
+        ];
+        const counts = await Promise.all(kinds.map(async (kind) => {
+            const found = await admin.listOpsFailures(c.get('actor'), {
+                operation: `intent.${kind}`,
+                since,
+                limit: PLATFORM_REQUEST_BACKLOG_CAP + 1,
+            });
+            return {
+                count: Math.min(found.length, PLATFORM_REQUEST_BACKLOG_CAP),
+                capped: found.length > PLATFORM_REQUEST_BACKLOG_CAP,
+            };
+        }));
+        const body = {
+            total: counts.reduce((sum, k) => sum + k.count, 0),
+            capped: counts.some((k) => k.capped),
+            since,
+            windowDays: PLATFORM_REQUEST_BACKLOG_WINDOW_DAYS,
+        };
+        return c.json(body);
+    });
     // The same upsert semantics as `/internal/connections/upsert` (§3.5.2) — create under a
     // fresh id, or rotate the one live row in place so its grant tuples survive — behind
     // platform-actor auth instead of the vertical-harness PLATFORM_SECRET. The tenant is
@@ -1370,7 +1574,8 @@ export function createControlPlaneApi(options) {
      * current dispatch — resolved BEFORE any version rebind), restore into the serving
      * script (which re-projects the vertical's roles), and only then `setScopeServingRef` +
      * advance the version pointer. A crash before the flip leaves the scope serving its old
-     * script intact, and the adopt retries idempotently. Already-adopted scopes short-circuit.
+     * script intact, and the adopt retries idempotently. Previews are refused; already-adopted
+     * installs short-circuit.
      * Throws `ControlPlaneError` so callers surface an actionable status, never a bare 500.
      */
     const adoptScopeOntoServing = async (c, tenantId, scopeId) => {
@@ -1379,6 +1584,9 @@ export function createControlPlaneApi(options) {
         if (!scope) {
             throw new ControlPlaneError(404, `unknown scope for tenant: (${tenantId}, ${scopeId})`);
         }
+        if (scope.kind === 'preview') {
+            throw new ControlPlaneError(409, 'preview scopes cannot adopt the production serving script');
+        }
         if (scope.servingRef)
             return { servingRef: scope.servingRef, alreadyAdopted: true };
         if (!scope.vertical) {
@@ -1407,8 +1615,8 @@ export function createControlPlaneApi(options) {
      * pointer to the promoted version so Update stops offering a crossing already made.
      *
      * Gated exactly where the host cascade would have run: PRIVATE (owned, unlisted) only,
-     * active non-fork scopes only. Runs only when a serving script exists — i.e. the serve
-     * actually happened (dispatch-backed + deploy configured); for an embedded vertical the
+     * active non-fork, non-preview scopes only. Runs only when a serving script exists —
+     * i.e. the serve actually happened (dispatch-backed + deploy configured); for an embedded vertical the
      * host cascade already rebound, and `verticalServing` is null, so this is a no-op.
      * Idempotent and retry-safe: the host cascade never rebound these scopes, so their data
      * is still findable on a retry after a failed serve.
@@ -1421,7 +1629,7 @@ export function createControlPlaneApi(options) {
         const v = await verticalOf(actor, slug);
         if (!v || v.ownerTenant === null || v.listed)
             return; // private only, like the host cascade
-        const owned = (await admin.listScopes(actor, { tenantId: v.ownerTenant, vertical: slug, status: ['active'] })).filter((s) => !s.forkedFrom);
+        const owned = (await admin.listScopes(actor, { tenantId: v.ownerTenant, vertical: slug, status: ['active'] })).filter((s) => !s.forkedFrom && s.kind !== 'preview');
         for (const s of owned) {
             if (!s.servingRef) {
                 // Adopt: export from the scope's current (un-rebound) dispatch → serving script,
@@ -1509,6 +1717,84 @@ export function createControlPlaneApi(options) {
         await admin.bindScopeVersion(actor, tenantId, scopeId, serving.versionId);
         return { servingRef: serving.ref, versionId: serving.versionId, tables: restored.tables };
     };
+    /**
+     * Carry a scope's data onto the script a version bind is about to route it to (#1710).
+     * The caller binds only after this returns.
+     *
+     * A scope that is not on the serving script routes by its bound version's own
+     * per-version script (`COALESCE(servingRef, deploymentRef)`), and a Durable Object
+     * namespace belongs to its script. So a bind to a version with a DIFFERENT
+     * `deploymentRef` points the scope at an empty namespace. Previews live there by design
+     * (#527), and so does a forked scope that `scope bind` re-points. This moves the data
+     * first, in adopt-serving's order: export from the script the router resolves the scope
+     * to TODAY, then restore into the incoming version's script.
+     *
+     * After validating the incoming version, it reads no scope data and carries nothing
+     * when the route does not move: the incoming
+     * version has no script of its own (co-located), the two refs are the same script, the
+     * scope stays pinned to the serving script (the version pointer never touches its
+     * route), or the scope's data is not in a named script. It also carries nothing on a
+     * host with no per-version resolver, because such a host has no per-version namespaces
+     * to move between. `dropServingRef` is for a caller that clears the pin in the same act:
+     * the route then does move, off the serving script.
+     *
+     * A failure strands nothing. The pointer has not moved and the source copy is untouched,
+     * and the restore is one transaction in the target DO (drop-then-replay). So a failed
+     * carry leaves the scope serving its data from the old script, and a retry carries again
+     * over any partial copy. A retry after a bind that landed finds the same script on both
+     * ends and carries nothing.
+     *
+     * What it does not do:
+     * - Carry writes that land on the source between the export and the bind. Those are lost.
+     * - Delete the source copy. That copy is unreachable once the caller binds, and it stays
+     *   in the old script, because nothing reaps it yet (#1722). Deleting it here would race
+     *   a concurrent push that is still exporting from it.
+     * - Cross lineages. A version of another vertical reads as absent here, and moving a scope
+     *   between lineages is `rebindScopeOntoVertical`'s job, behind its migration gate.
+     */
+    const carryOntoVersion = async (c, scope, versionId, opts = {}) => {
+        const resolveVersion = options.resolveVerticalVersion;
+        if (!scope.vertical)
+            return null;
+        const actor = c.get('actor');
+        const incoming = await admin.getVersion(actor, versionId, scope.vertical);
+        // The bind refuses a version that is not admitted, except onto a preview. Check that
+        // BEFORE the copy. Otherwise unreviewed code would receive the scope's data through its
+        // own restore handler, only for the bind to refuse afterwards.
+        if (!incoming) {
+            throw new ControlPlaneError(404, `unknown version ${versionId} for vertical '${scope.vertical}'`);
+        }
+        if (incoming.admission !== 'admitted' && scope.kind !== 'preview') {
+            throw new ControlPlaneError(409, `version ${versionId} is not admitted`);
+        }
+        if (!resolveVersion || (scope.servingRef && !opts.dropServingRef))
+            return null;
+        const to = incoming.deploymentRef ?? null;
+        const bound = scope.verticalVersionId
+            ? await admin.getVersion(actor, scope.verticalVersionId, scope.vertical)
+            : undefined;
+        const from = scope.servingRef ?? bound?.deploymentRef ?? null;
+        if (!to || !from || from === to)
+            return null;
+        // Resolve each end at exactly the script named above. No fallback to the prod channel:
+        // a guessed source would be exported, found empty, and carried as if it were the data.
+        const source = scope.servingRef
+            ? await options.resolveVerticalRef?.(scope.servingRef)
+            : await resolveVersion(scope.vertical, scope.verticalVersionId, actor);
+        const dest = await resolveVersion(scope.vertical, versionId, actor);
+        if (!source || !dest) {
+            // Binding anyway would point the scope at an empty store and leave its data behind,
+            // so refuse and name the end that did not resolve. 502, like any unreachable
+            // dependency here (`vertical-client.ts` `reach`, `preview-auth.ts`): this is a wiring or
+            // transient failure, not a deployment that is too old to do it (which is what 501 means).
+            const missing = !source ? `the script holding its data ('${from}')` : `version ${versionId}'s script ('${to}')`;
+            throw new ControlPlaneError(502, `cannot move scope ${scope.id} from '${from}' to '${to}': ${missing} could not be reached, ` +
+                `so the version was not bound (#1710)`);
+        }
+        const dump = await retryTransient(() => source.exportScope(scope.id));
+        const restored = await retryTransient(() => dest.restoreScope(scope.tenantId, scope.id, dump));
+        return { from, to, tables: restored.tables };
+    };
     app.get('/tenants/:tenantId/scopes/:scopeId/tables', async (c) => {
         const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
         const scopeId = scopeIdSchema.parse(c.req.param('scopeId'));
@@ -2924,7 +3210,7 @@ export function createControlPlaneApi(options) {
         if (v.ownerTenant === null) {
             return c.json({ error: 'adopt-serving is a private-vertical operation' }, 409);
         }
-        const owned = (await admin.listScopes(actor, { tenantId: v.ownerTenant, vertical: slug })).filter((s) => !s.forkedFrom && s.status === 'active');
+        const owned = (await admin.listScopes(actor, { tenantId: v.ownerTenant, vertical: slug })).filter((s) => !s.forkedFrom && s.kind !== 'preview' && s.status === 'active');
         const adopted = [];
         const alreadyAdopted = [];
         try {
@@ -2970,13 +3256,26 @@ export function createControlPlaneApi(options) {
     // opts into fork-before-promote (§4): on a migration-digest-crossing bind the
     // pre-migration data is snapshotted first — orchestrated through the vertical
     // when one resolves, in-process otherwise.
+    //
+    // A scope that routes by its bound version's own script (a fork, such as a forked test
+    // environment `substrat scope bind` re-points on every merge) has its data carried into
+    // the new version's script before the pointer moves (#1710). The snapshot is taken
+    // after a successful carry, before binding, in the script the data is leaving. A scope
+    // on the serving script carries nothing, because its route does not follow the version pointer.
     app.post('/tenants/:tenantId/scopes/:scopeId/version', async (c) => {
         const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
         const scopeId = scopeIdSchema.parse(c.req.param('scopeId'));
         const { versionId, snapshot } = bindScopeVersionBody.parse(await c.req.json());
         const actor = c.get('actor');
+        const scope = await admin.getScopeRecord(actor, tenantId, scopeId);
+        // A refusal throws a ControlPlaneError, which the app's error boundary relays with its
+        // status and records as an ops failure. An unknown scope carries nothing, and the bind
+        // below refuses it as it always has.
+        const carry = async () => {
+            if (scope)
+                await carryOntoVersion(c, scope, versionId);
+        };
         if (snapshot) {
-            const scope = await admin.getScopeRecord(actor, tenantId, scopeId);
             if (!scope) {
                 return c.json({ error: `unknown scope for tenant: (${tenantId}, ${scopeId})` }, 404);
             }
@@ -2984,22 +3283,26 @@ export function createControlPlaneApi(options) {
             if (vertical) {
                 // Delegated path: the digest compare lives here (the in-process path does
                 // it below the seam). Snapshot only a migration-crossing bind.
+                let migrationCrossing = false;
                 if (scope.vertical && scope.verticalVersionId) {
                     const [current, incoming] = await Promise.all([
                         admin.getVersion(actor, scope.verticalVersionId, scope.vertical),
                         admin.getVersion(actor, versionId, scope.vertical),
                     ]);
-                    if (current && incoming && current.migrationDigest !== incoming.migrationDigest) {
-                        await orchestratedSnapshot(c, tenantId, scope, {});
-                    }
+                    migrationCrossing = Boolean(current && incoming && current.migrationDigest !== incoming.migrationDigest);
                 }
+                await carry();
+                if (migrationCrossing)
+                    await orchestratedSnapshot(c, tenantId, scope, {});
                 await admin.bindScopeVersion(actor, tenantId, scopeId, versionId);
             }
             else {
+                await carry();
                 await admin.bindScopeVersion(actor, tenantId, scopeId, versionId, { snapshot: true });
             }
             return c.json(await admin.getScopeRecord(actor, tenantId, scopeId));
         }
+        await carry();
         await admin.bindScopeVersion(actor, tenantId, scopeId, versionId);
         return c.json(await admin.getScopeRecord(actor, tenantId, scopeId));
     });
@@ -3427,6 +3730,12 @@ export function createControlPlaneApi(options) {
             // happen — while this one wants the declaration as a node.
             schedules: parsed?.schedules ?? [],
             outbound: parsed?.outbound ?? [],
+            // The peer verticals this version declares it calls (#1706) — `outbound`'s sibling
+            // one band over: the same kind of declared reach, at another of the tenant's apps
+            // rather than at a third-party host. `null` is a FACT, not missing data: a version
+            // pushed before the declaration existed is unenforced, which a reader must be able
+            // to tell from a version that declares it calls nothing.
+            calls: parsed?.calls ?? null,
         });
     });
     app.get('/verticals/:slug/versions/:id/model', async (c) => {
@@ -4258,13 +4567,24 @@ export function createControlPlaneApi(options) {
             scopeId: z.string().min(1).max(64).optional(),
             vertical: z.string().min(1).max(200).optional(),
             hours: z.coerce.number().int().min(1).max(72).default(24),
+            since: z.string().datetime({ offset: true }).optional(),
+            until: z.string().datetime({ offset: true }).optional(),
         })
             .parse({
             tenantId,
             scopeId: c.req.query('scopeId') || undefined,
             vertical: c.req.query('vertical') || undefined,
-            hours: c.req.query('hours'),
+            hours: c.req.query('hours'), since: c.req.query('since'), until: c.req.query('until'),
         });
+        try {
+            resolveObservabilityWindow(input);
+        }
+        catch (e) {
+            throw new ControlPlaneError(400, e.message);
+        }
+        if (input.since !== undefined && !options.observability.absoluteTenantWindows) {
+            throw new ControlPlaneError(501, 'absolute tenant telemetry windows are not supported by this backend');
+        }
         return c.json(await options.observability.tenantMetrics(input));
     });
     // The bucketed twin (#1447): the series a team-level chart plots, one line per app.
@@ -4288,8 +4608,19 @@ export function createControlPlaneApi(options) {
             // zero rows rather than somebody else's, so no ownership check is needed here.
             scopeIds: z.array(z.string().min(1).max(64)).min(1).max(TENANT_SERIES_SCOPE_CAP),
             hours: z.coerce.number().int().min(1).max(72).default(24),
+            since: z.string().datetime({ offset: true }).optional(),
+            until: z.string().datetime({ offset: true }).optional(),
         })
-            .parse({ tenantId, scopeIds, hours: c.req.query('hours') });
+            .parse({ tenantId, scopeIds, hours: c.req.query('hours'), since: c.req.query('since'), until: c.req.query('until') });
+        try {
+            resolveObservabilityWindow(input);
+        }
+        catch (e) {
+            throw new ControlPlaneError(400, e.message);
+        }
+        if (input.since !== undefined && !options.observability.absoluteTenantWindows) {
+            throw new ControlPlaneError(501, 'absolute tenant telemetry windows are not supported by this backend');
+        }
         return c.json(await options.observability.tenantMetricsSeries(input));
     });
     app.get('/observability/tenant-logs', async (c) => {
@@ -4696,11 +5027,21 @@ export function createControlPlaneApi(options) {
      *  between the two converges on retry. Shared by that route and by the create path, which
      *  reaps a HALF-BUILT leftover before re-forking (see `orchestratedPreview`). */
     const reapPreview = async (c, preview) => {
+        // A fork's own sign-in clients go FIRST (#1704): a failure here leaves the preview in
+        // place, so the retry a PR-close job makes still finds it — after the row is gone, nothing
+        // would name those clients again. A clean room never had one.
+        await retireClientsOfReapedScope(previewAuthDeps(c), preview);
         const vertical = await verticalForScope(c, preview);
         if (vertical)
             await vertical.deleteScope({ scopeId: preview.id });
         await options.host.deleteSnapshot(c.get('actor'), preview.tenantId, preview.id);
     };
+    /** The preview-login seam (#1704, `preview-auth.ts`): the directory, and each issuer's deployment. */
+    const previewAuthDeps = (c) => ({
+        admin,
+        actor: c.get('actor'),
+        issuerClient: (scope) => verticalForScope(c, scope),
+    });
     /** Given a base hostname `<label>.<domain>`, mint (or find) the preview's `--<tag>`
      *  hostname `<label>--<tag>.<domain>` bound to `previewId`. Non-canonical, so it never
      *  demotes the prod surface. Shared by the fork and clean-room paths. */
@@ -4801,18 +5142,65 @@ export function createControlPlaneApi(options) {
         const target = (await options.resolveVerticalVersion?.(slug, opts.versionId, actor)) ??
             options.verticals?.[slug] ??
             (await options.resolveVertical?.(slug, actor));
+        // The preview's own login (#1704), once it serves this version. Delivered into `target` —
+        // THIS push's version deployment — and never "whatever the scope is bound to now", so an
+        // older push can never overwrite a newer push's config store. A failure lands an
+        // ops-failure row like a failed restore (#559): the preview exists and serves the PR, but
+        // has no working login. The message names no secret; none ever reaches it.
+        const wireLogin = async (previewId, hostname, parent) => {
+            try {
+                return await wirePreviewAuth(previewAuthDeps(c), {
+                    tenantId,
+                    versionId: opts.versionId,
+                    previewId,
+                    tag: opts.tag,
+                    parent,
+                    previewHostname: hostname,
+                    appName: parent?.name ?? slug.split('/').at(-1),
+                    deliver: async (entries) => {
+                        if (!target)
+                            throw new ControlPlaneError(501, 'no deployment resolves for the PR version to deliver to');
+                        await retryTransient(() => target.configureInstance({ tenantId, scopeId: previewId, entries }));
+                    },
+                });
+            }
+            catch (e) {
+                if (e instanceof ControlPlaneError) {
+                    recordFailure({
+                        actor,
+                        operation: 'preview.create',
+                        stage: 'login',
+                        tenantId,
+                        scopeId: previewId,
+                        vertical: slug,
+                        version: opts.versionId,
+                        status: e.status,
+                        message: e.message,
+                    }, e);
+                }
+                throw e;
+            }
+        };
+        const notesFor = (parent) => parent
+            ? [
+                "Settings: a preview does not carry over the app's per-install settings (the dashboard's Env tab); " +
+                    "it runs on the deployment's defaults.",
+            ]
+            : [];
         // The `--<tag>` URL's base — the source's canonical hostname (fork) or the tenant-app
         // convention (clean-room). Computed once so reuse and fresh mint the same URL.
         const baseHostname = await previewBaseHostname(actor, source, tenantId, slug, surface);
         // Idempotent on (tenant, vertical, tag): a PR *synchronize* rebinds the new version
-        // onto the SAME preview — successive pushes roll their migrations forward on one copy
-        // (§4's rehearsal case) — unless `refresh` asks for a fresh one.
+        // onto the SAME preview, unless `refresh` asks for a fresh one. On a hosted vertical
+        // every push is its own script, so the rebind first CARRIES the preview's data into
+        // the new version's script (#1710). The preview's data then follows it from push to
+        // push, and each push's migrations run forward over it (§4's rehearsal case).
         const existing = (await admin.listScopes(actor, { tenantId, vertical: slug })).find((s) => s.kind === 'preview' && s.slug === previewSlug(slug, opts.tag));
         // A preview only HAS data once its two-phase create finished: the directory row lands
         // first as `provisioning` (K-31), the fork's export→restore runs, and `activateScope`
         // is the last step. So a row still at `provisioning` is a create that DIED mid-fork —
-        // its DO is empty. Reuse must never adopt one: reuse only rebinds the version and the
-        // hostname, it never copies data, so adopting a half-built row hands back a
+        // its DO is empty. Reuse must never adopt one: reuse carries whatever the preview holds
+        // and never re-forks from the source, so adopting a half-built row hands back a
         // permanently EMPTY preview and reports `reused: true` — success for a URL that shows
         // a reviewer no data at all. That is exactly what a CI retry does (the generated
         // workflow retries `preview create` on a transient), so the failure mode is the
@@ -4822,6 +5210,31 @@ export function createControlPlaneApi(options) {
         // scope would otherwise collide with the old row's still-bound `--<tag>` hostname.
         const stale = existing !== undefined && (opts.refresh || existing.status !== 'active');
         if (existing && !stale) {
+            // Data first, pointer last (#1710): move the preview's data into this push's script
+            // BEFORE anything re-points it. A failed carry therefore leaves the preview bound to
+            // its old script with its data, and records why, like a failed fork restore (#559).
+            // `dropServingRef`: a preview provisioned before #527 inherited a serving_ref that is
+            // cleared just below, so its route leaves the serving script too.
+            let carried;
+            try {
+                carried = await carryOntoVersion(c, existing, opts.versionId, { dropServingRef: true });
+            }
+            catch (e) {
+                if (e instanceof ControlPlaneError && e.status >= 500 && e.status !== 501) {
+                    recordFailure({
+                        actor,
+                        operation: 'preview.create',
+                        stage: 'carry',
+                        tenantId,
+                        scopeId: existing.id,
+                        vertical: slug,
+                        version: opts.versionId,
+                        status: e.status,
+                        message: e.message,
+                    }, e);
+                }
+                throw e;
+            }
             // Heal a preview provisioned before #527: clear any inherited serving_ref so routing
             // follows the bound version (its per-version script), not the prod serving script.
             if (existing.servingRef)
@@ -4831,7 +5244,34 @@ export function createControlPlaneApi(options) {
             await admin.setScopeExpiresAt(actor, tenantId, existing.id, expiresAt);
             const hostname = await bindPreviewHostname(actor, baseHostname, tenantId, existing.id, opts.tag, surface);
             await assertServesBoundVersion(existing.id);
-            return { scopeId: existing.id, hostname, url: `https://${hostname}`, versionId: opts.versionId, reused: true };
+            // The fork it was made from, not whichever prod scope this call picked: a reuse is the
+            // same preview. Every push is a new version with an EMPTY config store, and the carry
+            // above moves the scope's data but not the vertical's config store, so the login is
+            // wired again on every reuse, never assumed to have survived.
+            const parent = existing.forkedFrom
+                ? ((await admin.getScopeRecord(actor, tenantId, existing.forkedFrom)) ?? null)
+                : null;
+            const auth = await wireLogin(existing.id, hostname, parent);
+            return {
+                scopeId: existing.id,
+                hostname,
+                url: `https://${hostname}`,
+                versionId: opts.versionId,
+                reused: true,
+                auth,
+                // The carry's loss window, said where the person pushing reads it (the CLI prints
+                // every note): a reviewer who clicked around during the push would otherwise just
+                // find their change gone.
+                notes: [
+                    ...(carried
+                        ? [
+                            `Data: the preview's data was copied into this push's version (${carried.tables} tables). ` +
+                                'Anything written to the preview while this push ran may be lost.',
+                        ]
+                        : []),
+                    ...notesFor(parent),
+                ],
+            };
         }
         // Free the tag: the slug is unique per tenant and the `--<tag>` hostname is still bound
         // to the old row, so the fresh fork below cannot be provisioned until this one is gone.
@@ -4921,7 +5361,16 @@ export function createControlPlaneApi(options) {
         await admin.bindScopeVersion(actor, tenantId, previewId, opts.versionId);
         const hostname = await bindPreviewHostname(actor, baseHostname, tenantId, previewId, opts.tag, surface);
         await assertServesBoundVersion(previewId);
-        return { scopeId: previewId, hostname, url: `https://${hostname}`, versionId: opts.versionId, reused: false };
+        const auth = await wireLogin(previewId, hostname, source);
+        return {
+            scopeId: previewId,
+            hostname,
+            url: `https://${hostname}`,
+            versionId: opts.versionId,
+            reused: false,
+            auth,
+            notes: notesFor(source),
+        };
     };
     // The owning tenant of a builder's OWN vertical — the gate every preview route shares. A
     // preview forks THIS tenant's own scope and binds the version onto that fork (never an
@@ -5003,6 +5452,8 @@ export function createControlPlaneApi(options) {
                 expiresAt: s.expiresAt,
                 hostname: hosts[0]?.hostname ?? null,
                 url: hosts[0]?.hostname ? `https://${hosts[0].hostname}` : null,
+                // What an EXTERNAL issuer would need registered for this preview to sign in (#1704).
+                callbackUrl: hosts[0]?.hostname ? oidcCallbackUrl(hosts[0].hostname) : null,
             });
         }
         return c.json(out);
@@ -5130,6 +5581,70 @@ export function createControlPlaneApi(options) {
     };
     app.delete('/tenants/:tenantId/scopes/:scopeId/system-grants', switchScheduleRoute('off'));
     app.post('/tenants/:tenantId/scopes/:scopeId/system-grants', switchScheduleRoute('on'));
+    // The status read (#1674): "is this module switched off on this scope?", without
+    // reaching for the scope's own SQL console (the CHECKPOINT runbook's old CONFIRM step —
+    // now this route). Same staff-only gate as the switch itself: a tenant credential passes
+    // the `/tenants/<own>` confinement, so it is refused here too, by the same
+    // `confinedTenant` check, not a kind test.
+    app.get('/tenants/:tenantId/scopes/:scopeId/system-grants', async (c) => {
+        if (confinedTenant(c.get('principal')) !== null) {
+            return c.json({ error: 'forbidden: the schedule switch is staff-only' }, 403);
+        }
+        const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
+        const scopeId = scopeIdSchema.parse(c.req.param('scopeId'));
+        const actor = c.get('actor');
+        // K-3 first, so a scope of another tenant reads as absent before anything is reached.
+        if (!(await admin.getScopeRecord(actor, tenantId, scopeId))) {
+            return c.json({ error: `unknown scope for tenant: (${tenantId}, ${scopeId})` }, 404);
+        }
+        return c.json(await admin.systemGrantsStatus(actor, { tenantId, scopeId }));
+    });
+    // -- the peer kill switch (#1706) ------------------------------------------
+    //
+    // One calling vertical's access to ONE scope, off and back on — `revokeFromPeer` and
+    // `restoreToPeer`. The schedule switch above with the subject swapped: `reason` required,
+    // both halves audited intent-then-outcome, idempotent, and no re-provision re-seats what
+    // OFF took, so the POST is the only way back. For a HOSTED scope the grants live in the
+    // vertical's deployment and `peerSwitchDelegation` moves the switch there; a deployment
+    // built before the far end existed answers 501 "redeploy" and nothing is switched.
+    //
+    // **Unlike the schedule switch, a tenant may pull this one**, and that is the point of it:
+    // "may this other app of mine still call into here" is the tenant's question about its own
+    // two apps, not a platform incident lever. A confined credential (#977) is held by the
+    // dashboard on a tenant's behalf and may act on THAT tenant only; no vertical holds one at
+    // all, since a vertical is CP-less, so the switched party cannot switch itself back on.
+    const switchPeerRoute = (to) => async (c) => {
+        const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
+        const scopeId = scopeIdSchema.parse(c.req.param('scopeId'));
+        const pin = confinedTenant(c.get('principal'));
+        if (pin !== null && pin !== tenantId)
+            return c.json({ error: 'forbidden' }, 403);
+        const body = peerSwitchBody.parse(await c.req.json());
+        const actor = c.get('actor');
+        // K-3 first, so a scope of another tenant reads as absent before anything is reached.
+        if (!(await admin.getScopeRecord(actor, tenantId, scopeId))) {
+            return c.json({ error: `unknown scope for tenant: (${tenantId}, ${scopeId})` }, 404);
+        }
+        const input = { vertical: body.vertical, node: { tenantId, scopeId }, reason: body.reason };
+        return c.json(to === 'off' ? await admin.revokeFromPeer(actor, input) : await admin.restoreToPeer(actor, input));
+    };
+    app.delete('/tenants/:tenantId/scopes/:scopeId/peer-grants', switchPeerRoute('off'));
+    app.post('/tenants/:tenantId/scopes/:scopeId/peer-grants', switchPeerRoute('on'));
+    // The status read (#1706): which of the tenant's other apps may call into this scope, and
+    // where each stands — the read the dashboard's install disclosure and the console's scope
+    // view are both built on. Same confinement as the switch, for the same reason.
+    app.get('/tenants/:tenantId/scopes/:scopeId/peer-grants', async (c) => {
+        const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
+        const scopeId = scopeIdSchema.parse(c.req.param('scopeId'));
+        const pin = confinedTenant(c.get('principal'));
+        if (pin !== null && pin !== tenantId)
+            return c.json({ error: 'forbidden' }, 403);
+        const actor = c.get('actor');
+        if (!(await admin.getScopeRecord(actor, tenantId, scopeId))) {
+            return c.json({ error: `unknown scope for tenant: (${tenantId}, ${scopeId})` }, 404);
+        }
+        return c.json(await admin.peerGrantsStatus(actor, { tenantId, scopeId }));
+    });
     // Orgs: the portal-customer grouping (§4.1). Creating one mints no permission —
     // members reach what the org was GRANTED, and granting stays off this surface.
     app.post('/tenants/:tenantId/orgs', async (c) => {