@substrat-run/control-plane-api 0.120.0 → 0.122.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/api.js CHANGED
@@ -1,14 +1,15 @@
1
1
  import { resolveObservabilityWindow } from './observability-window.js';
2
2
  import { Hono } from 'hono';
3
- import { parseHostname, withLabel, RESERVED_LABEL_SEPARATOR, adminAction, ASSET_PART_PREFIX, assetHash, channelName, connectionActivity, connectionActivitySource, connectionCredential, connectionFilter, connectionHealthState, connectionProvider, connectorDispatchKind, PROVISION_SIBLING_KIND, ARCHIVE_SCOPE_KIND, PROVISION_TENANT_KIND, SET_ENTITLEMENTS_KIND, MODEL_USAGE_KIND, SWEEP_RUNS_KIND, toConnectionHealthEntry, CONNECTION_EXPIRY_WARNING_DAYS, CONNECTION_STALE_DAYS, connectionProbe, createTenantInput, entitlementGrantInput, hostname as hostnameSchema, hostnameRegion, hostnameStatus, identityLink, errorCode, issueStatus, issueStatusInput, listPageQuery, matchesOutboundHost, pageOf, platformRequestFilter, principalId as principalIdSchema, promotionAcknowledgement, provisionableJurisdiction, publishVersionInput, queryScopeInput, readScopeTableInput, entityHistoryInput, eventFacetInput, eventCauseInput, eventEffectsInput, invocationEventsInput, deadLettersInput, delegatedReadParams, createOrgInput, roleKey as roleKeySchema, peerSwitch, systemSwitch, DEFAULT_DENIAL_LIMIT, DENIAL_LIMIT_MAX, denialGroupBy, registerVerticalInput, serviceDimensions, sweepRunKind, sweepRunOutcome, scopeDump, dataSubjectId as dataSubjectIdSchema, scopeId as scopeIdSchema, scopeStatus, storageShape, surfaceName, tenantId as tenantIdSchema, tenantStatus, versionOrigin, z, PROBLEM_CONTENT_TYPE, toProblem, redrainEventsInput, REDRAIN_BATCH, } from '@substrat-run/contracts';
3
+ import { parseHostname, withLabel, RESERVED_LABEL_SEPARATOR, adminAction, ASSET_PART_PREFIX, assetHash, channelName, connectionActivity, connectionActivitySource, connectionCredential, connectionFilter, connectionHealthState, connectionProvider, connectorDispatchKind, PROVISION_SIBLING_KIND, ARCHIVE_SCOPE_KIND, PROVISION_TENANT_KIND, SET_ENTITLEMENTS_KIND, MODEL_USAGE_KIND, SWEEP_RUNS_KIND, toConnectionHealthEntry, CONNECTION_EXPIRY_WARNING_DAYS, CONNECTION_STALE_DAYS, connectionProbe, createTenantInput, entitlementGrantInput, hostname as hostnameSchema, hostnameRegion, hostnameStatus, identityLink, errorCode, issueStatus, issueStatusInput, listPageQuery, matchesOutboundHost, pageOf, platformRequestFilter, principalId as principalIdSchema, promotionAcknowledgement, bindAcknowledgement, provisionableJurisdiction, publishVersionInput, queryScopeInput, readScopeTableInput, entityHistoryInput, eventFacetInput, eventCauseInput, eventEffectsInput, invocationEventsInput, deadLettersInput, delegatedReadParams, createOrgInput, roleKey as roleKeySchema, peerSwitch, systemSwitch, DEFAULT_DENIAL_LIMIT, DENIAL_LIMIT_MAX, denialGroupBy, registerVerticalInput, serviceDimensions, sweepRunKind, sweepRunOutcome, scopeDump, dataSubjectId as dataSubjectIdSchema, scopeId as scopeIdSchema, scopeStatus, storageShape, surfaceName, tenantId as tenantIdSchema, tenantStatus, versionOrigin, z, PROBLEM_CONTENT_TYPE, toProblem, redrainEventsInput, importCursorMove, importCursorAcknowledgementMissing, REDRAIN_BATCH, migrationsOnTop, } from '@substrat-run/contracts';
4
4
  import { attributeFailure } from './failure-attribution.js';
5
- import { migrationProgress, ulid } from '@substrat-run/kernel';
5
+ import { BIND_EXPORT_BREAK_REFUSAL, bindExportBreakRefusal, crossVerticalHealth, exportBreakRefusal, isBindExportBreakRefusal, isExportBreakRefusal, migrationProgress, ulid, } from '@substrat-run/kernel';
6
6
  import { TENANT_HEADER, confinedTenant } from './auth.js';
7
7
  import { mintTenantToken } from './tenant-token.js';
8
8
  import { connectionGrantsForScope } from './vertical-client.js';
9
9
  import { oidcCallbackUrl, retireClientsOfReapedScope, wirePreviewAuth } from './preview-auth.js';
10
10
  import { versionReachedAt } from './scope-deployment.js';
11
11
  import { reconcileConnectionGrants } from './connection-grants.js';
12
+ import { reconcileThenReassert } from './reconcile.js';
12
13
  import { ConnectionRelayError, relayConnectionUpsert } from './connection-relay.js';
13
14
  import { ControlPlaneError } from './client.js';
14
15
  import { provisionSiblingScope } from './platform-drain.js';
@@ -56,6 +57,35 @@ function outsideTenant(p, tenantId) {
56
57
  const pin = confinedTenant(p);
57
58
  return pin !== null && pin !== tenantId;
58
59
  }
60
+ /**
61
+ * #1756: a move refused because it would break apps in the scope's own tenant. Carries the
62
+ * listing, because every route that moves a scope pins that tenant in its path, so the caller may
63
+ * read it whole; the counted sentence is the host's (`bindExportBreakRefusal`).
64
+ */
65
+ class ExportBreakRefused extends ControlPlaneError {
66
+ breaks;
67
+ constructor(breaks) {
68
+ super(409, bindExportBreakRefusal(breaks));
69
+ this.breaks = breaks;
70
+ }
71
+ }
72
+ /** The body a route answers an `ExportBreakRefused` with: the sentence and the listing. */
73
+ function exportBreakBody(e) {
74
+ return { error: e.message, ...(e instanceof ExportBreakRefused ? { exportBreaks: { affected: e.breaks } } : {}) };
75
+ }
76
+ /**
77
+ * A fleet-wide report, narrowed to what a caller may see: a confined caller gets its own
78
+ * tenant's rows and a COUNT of the other tenants, never their ids (K-3's forced filter, held on
79
+ * reports that have to be computed fleet-wide). Staff (`pin === null`) get every row.
80
+ */
81
+ function narrowToCaller(rows, pin) {
82
+ if (pin === null)
83
+ return { visible: [...rows], otherTenants: 0 };
84
+ return {
85
+ visible: rows.filter((r) => r.tenantId === pin),
86
+ otherTenants: new Set(rows.filter((r) => r.tenantId !== pin).map((r) => r.tenantId)).size,
87
+ };
88
+ }
59
89
  // -- request schemas ---------------------------------------------------------
60
90
  // Parse, don't trust: every input crosses Zod at the boundary. The ids stay
61
91
  // CALLER-SUPPLIED rather than minted here, exactly as the contract has them —
@@ -165,6 +195,9 @@ const bindScopeVersionBody = z.object({
165
195
  // data first when this bind crosses a migration-digest boundary. Optional and
166
196
  // ignored on a code-only rebind — the digest compare is the gate, not the flag.
167
197
  snapshot: z.boolean().optional(),
198
+ // #1756: a bind that drops or re-versions an export an app in this tenant imports is
199
+ // refused unless acknowledged — the promote gate's `exportBreak`, for one install.
200
+ acknowledge: bindAcknowledgement.optional(),
168
201
  });
169
202
  const rebindScopeVerticalBody = z.object({
170
203
  // The FULL registry id of the target lineage (e.g. `substrat-9yjbbn/manyfold`) —
@@ -181,7 +214,12 @@ const rebindScopeVerticalBody = z.object({
181
214
  // backout copy exactly as in a carried rebind. The scope must be re-provisioned
182
215
  // on the target afterwards (`/verticals/:slug/instances` is idempotent, K-31).
183
216
  abandonData: z.boolean().optional(),
217
+ // #1756: within one lineage the rebind is an adopt, refused when it breaks an app in the tenant.
218
+ acknowledge: bindAcknowledgement.optional(),
184
219
  });
220
+ // #1756: an adopt moves the scope onto what its vertical serves, and is refused when that breaks
221
+ // an app in the tenant. The body is optional, as it always was.
222
+ const adoptServingBody = z.object({ acknowledge: bindAcknowledgement.optional() });
185
223
  // A snapshot request (preview-and-snapshots.md §3/§9). `expiresAt` opts into the GC
186
224
  // sweep; absent = pinned until deliberately deleted. `kind` defaults to 'archive'.
187
225
  const snapshotScopeBody = z.object({
@@ -371,6 +409,19 @@ const tenantRoleAssignmentBody = z
371
409
  * spells out; `.strict()` so a body that tried to name one is refused, not stripped.
372
410
  */
373
411
  const systemSwitchBody = systemSwitch.pick({ moduleId: true, reason: true }).strict();
412
+ /**
413
+ * The fleet read's filter (#1674). `position` defaults to `off` — "what is switched off
414
+ * across the fleet" is the question the route exists for — and `all` asks for both. Paged
415
+ * by `operationId`, ascending by default (oldest switch first, as the admin log reads).
416
+ */
417
+ const systemSwitchesQuery = z.object({
418
+ position: z.enum(['on', 'off', 'all']).default('off'),
419
+ tenantId: tenantIdSchema.optional(),
420
+ scopeId: scopeIdSchema.optional(),
421
+ moduleId: systemSwitch.shape.moduleId.optional(),
422
+ vertical: z.string().min(1).optional(),
423
+ ...listPageQuery.shape,
424
+ });
374
425
  /** The peer kill switch's body (#1706) — the tenant and scope come from the path. */
375
426
  const peerSwitchBody = peerSwitch.pick({ vertical: true, reason: true }).strict();
376
427
  /** A new org under an addressed tenant (#1343) — `tenantId` comes from the path. */
@@ -453,6 +504,7 @@ const TENANT_ROUTES = [
453
504
  { method: 'DELETE', re: /\/verticals\/[^/]+$/, pin: 'owner' },
454
505
  { method: 'GET', re: /\/verticals\/[^/]+\/versions$/, pin: 'owner' },
455
506
  { method: 'GET', re: /\/verticals\/[^/]+\/versions\/[^/]+\/registry$/, pin: 'owner' },
507
+ { method: 'GET', re: /\/verticals\/[^/]+\/versions\/[^/]+\/migrations$/, pin: 'owner' },
456
508
  { method: 'GET', re: /\/verticals\/[^/]+\/versions\/[^/]+\/model$/, pin: 'owner' },
457
509
  { method: 'GET', re: /\/verticals\/[^/]+\/versions\/[^/]+\/flow$/, pin: 'owner' },
458
510
  { method: 'GET', re: /\/verticals\/[^/]+\/versions\/[^/]+\/assets$/, pin: 'owner' },
@@ -460,6 +512,7 @@ const TENANT_ROUTES = [
460
512
  { method: 'GET', re: /\/verticals\/[^/]+\/channels$/, pin: 'owner' },
461
513
  { method: 'GET', re: /\/verticals\/[^/]+\/channels\/[^/]+\/history$/, pin: 'owner' },
462
514
  { method: 'POST', re: /\/verticals\/[^/]+\/channels\/[^/]+\/promote$/, pin: 'owner' },
515
+ { method: 'GET', re: /\/verticals\/[^/]+\/channels\/[^/]+\/promote-impact$/, pin: 'owner' },
463
516
  { method: 'GET', re: /\/verticals\/[^/]+\/previews$/, pin: 'owner' },
464
517
  { method: 'POST', re: /\/verticals\/[^/]+\/previews$/, pin: 'owner' },
465
518
  { method: 'DELETE', re: /\/verticals\/[^/]+\/previews\/[^/]+$/, pin: 'owner' },
@@ -612,6 +665,9 @@ export function createControlPlaneApi(options) {
612
665
  // The declared permission registry of one version (D-39, #336) — owner-narrowed in the
613
666
  // handler like the versions list; the builder-facing Permissions tab reads it.
614
667
  { method: 'GET', re: /\/verticals\/[^/]+\/versions\/[^/]+\/registry$/ },
668
+ // The SQL migrations one version adds on top of another (#1677), owner-narrowed the same
669
+ // way; the promote dialog and `substrat promote` read it.
670
+ { method: 'GET', re: /\/verticals\/[^/]+\/versions\/[^/]+\/migrations$/ },
615
671
  // The emitted entity model of one version (#1214) — owner-narrowed the same way;
616
672
  // the dashboard's Model tab reads it.
617
673
  { method: 'GET', re: /\/verticals\/[^/]+\/versions\/[^/]+\/model$/ },
@@ -783,6 +839,8 @@ export function createControlPlaneApi(options) {
783
839
  if (err instanceof z.ZodError) {
784
840
  return problem(c, { status: 400, body: toProblem(err, c.req.path) });
785
841
  }
842
+ if (err instanceof ExportBreakRefused)
843
+ return c.json(exportBreakBody(err), 409);
786
844
  const { status, body } = mapError(err);
787
845
  // A 5xx is the PLATFORM failing — an unmapped throw, or a downstream vertical's
788
846
  // own 5xx passing through (a DO storage fault during a preview restore is the
@@ -1565,6 +1623,25 @@ export function createControlPlaneApi(options) {
1565
1623
  return (`no deployment is bound for vertical '${slug}': the scope is not on the serving script and its bound ` +
1566
1624
  `version did not resolve. Promote a version to prod, then adopt the scope onto the serving script.`);
1567
1625
  };
1626
+ /**
1627
+ * #1756: refuse, BEFORE any data moves, a move of this scope that would break apps in its
1628
+ * tenant, unless acknowledged. The host refuses the same move whoever calls; this is what keeps
1629
+ * a route that exports, restores or snapshots first from doing so for a move that is refused.
1630
+ * `servingRef` is the route the scope takes in the same act (an adopt).
1631
+ */
1632
+ const refuseOnBreaks = async (actor, tenantId, scopeId, versionId, opts) => {
1633
+ if (opts.acknowledge?.exportBreak)
1634
+ return;
1635
+ const breaks = await admin.bindingImpact(actor, tenantId, scopeId, versionId, opts.servingRef !== undefined ? { servingRef: opts.servingRef } : undefined);
1636
+ if (breaks.length > 0)
1637
+ throw new ExportBreakRefused(breaks);
1638
+ };
1639
+ /** The host's own export-break refusal (the move changed since it was asked), as the same 409. */
1640
+ const relayHostRefusal = (err) => {
1641
+ if (isBindExportBreakRefusal(err))
1642
+ throw new ControlPlaneError(409, err.message);
1643
+ throw err;
1644
+ };
1568
1645
  /**
1569
1646
  * Move ONE legacy scope's data off its per-version dispatch script onto its vertical's
1570
1647
  * stable serving script (#286/#321), then flip routing. The one primitive behind both
@@ -1578,7 +1655,7 @@ export function createControlPlaneApi(options) {
1578
1655
  * installs short-circuit.
1579
1656
  * Throws `ControlPlaneError` so callers surface an actionable status, never a bare 500.
1580
1657
  */
1581
- const adoptScopeOntoServing = async (c, tenantId, scopeId) => {
1658
+ const adoptScopeOntoServing = async (c, tenantId, scopeId, opts = {}) => {
1582
1659
  const actor = c.get('actor');
1583
1660
  const scope = await admin.getScopeRecord(actor, tenantId, scopeId);
1584
1661
  if (!scope) {
@@ -1587,8 +1664,13 @@ export function createControlPlaneApi(options) {
1587
1664
  if (scope.kind === 'preview') {
1588
1665
  throw new ControlPlaneError(409, 'preview scopes cannot adopt the production serving script');
1589
1666
  }
1590
- if (scope.servingRef)
1667
+ if (scope.servingRef) {
1668
+ // #1674 (Copilot review): an adopt that flipped routing and then failed to re-assert is
1669
+ // retried HERE, so the retry re-asserts too — else a store that lost its OFF marker in
1670
+ // the copy would stay on for good. Cheap and idempotent when nothing is owed.
1671
+ await admin.reassertSystemSwitches(actor, { tenantId, scopeId });
1591
1672
  return { servingRef: scope.servingRef, alreadyAdopted: true };
1673
+ }
1592
1674
  if (!scope.vertical) {
1593
1675
  throw new ControlPlaneError(409, 'scope has no vertical — nothing to adopt onto');
1594
1676
  }
@@ -1596,6 +1678,12 @@ export function createControlPlaneApi(options) {
1596
1678
  if (!serving) {
1597
1679
  throw new ControlPlaneError(409, `vertical '${scope.vertical}' has no serving script yet — promote a version to prod first`);
1598
1680
  }
1681
+ // #1756: onto the serving script, the scope runs the served version from the moment routing
1682
+ // flips — whatever its own version exported. Asked before the export below moves anything.
1683
+ await refuseOnBreaks(actor, tenantId, scopeId, serving.versionId, {
1684
+ servingRef: serving.ref,
1685
+ acknowledge: opts.acknowledge,
1686
+ });
1599
1687
  const source = await verticalForScope(c, scope);
1600
1688
  const dest = await options.resolveVerticalRef?.(serving.ref);
1601
1689
  if (!source || !dest) {
@@ -1604,8 +1692,15 @@ export function createControlPlaneApi(options) {
1604
1692
  const dump = await source.exportScope(scopeId);
1605
1693
  const restored = await retryTransient(() => dest.restoreScope(tenantId, scopeId, dump));
1606
1694
  // Data landed — only now flip routing and move the version pointer.
1607
- await admin.setScopeServingRef(actor, tenantId, scopeId, serving.ref);
1608
- await admin.bindScopeVersion(actor, tenantId, scopeId, serving.versionId);
1695
+ await admin
1696
+ .setScopeServingRef(actor, tenantId, scopeId, serving.ref, { acknowledge: opts.acknowledge })
1697
+ .catch(relayHostRefusal);
1698
+ await admin
1699
+ .bindScopeVersion(actor, tenantId, scopeId, serving.versionId, { acknowledge: opts.acknowledge })
1700
+ .catch(relayHostRefusal);
1701
+ // #1674: the scope now routes to a different store, so put the directory's recorded
1702
+ // OFF positions back there — cheap and idempotent when the dump carried them.
1703
+ await admin.reassertSystemSwitches(actor, { tenantId, scopeId });
1609
1704
  return { servingRef: serving.ref, tables: restored.tables };
1610
1705
  };
1611
1706
  /**
@@ -1621,29 +1716,51 @@ export function createControlPlaneApi(options) {
1621
1716
  * Idempotent and retry-safe: the host cascade never rebound these scopes, so their data
1622
1717
  * is still findable on a retry after a failed serve.
1623
1718
  */
1624
- const adoptAndRebindOwnedScopes = async (c, slug, versionId) => {
1719
+ const adoptAndRebindOwnedScopes = async (c, slug, versionId,
1720
+ // #1756: the promote's own acknowledgement. An install adopted here goes from ITS version to
1721
+ // the promoted one, which the promote gate judged only from the channel's previous version:
1722
+ // an install lagging behind that is judged again, and passes on the promote's ack or not at all.
1723
+ acknowledge) => {
1625
1724
  const actor = c.get('actor');
1725
+ const none = { refused: [] };
1626
1726
  const serving = await admin.verticalServing(actor, slug);
1627
1727
  if (!serving)
1628
- return; // embedded / not dispatch-backed — the host cascade handled rebinds
1728
+ return none; // embedded / not dispatch-backed — the host cascade handled rebinds
1629
1729
  const v = await verticalOf(actor, slug);
1630
1730
  if (!v || v.ownerTenant === null || v.listed)
1631
- return; // private only, like the host cascade
1731
+ return none; // private only, like the host cascade
1632
1732
  const owned = (await admin.listScopes(actor, { tenantId: v.ownerTenant, vertical: slug, status: ['active'] })).filter((s) => !s.forkedFrom && s.kind !== 'preview');
1733
+ // An install refused for what it would break is left where it is, and the rest still move:
1734
+ // one lagging app must not keep every other install's pointer, or the store backfill after
1735
+ // this, from happening. The caller answers with what was refused.
1736
+ const refused = [];
1737
+ let refusal;
1633
1738
  for (const s of owned) {
1634
- if (!s.servingRef) {
1635
- // Adopt: export from the scope's current (un-rebound) dispatch → serving script,
1636
- // then bind to the serving version. Data-first, so a failure here leaves the
1637
- // scope intact on its old script for the next promote to retry.
1638
- await adoptScopeOntoServing(c, s.tenantId, s.id);
1739
+ try {
1740
+ if (!s.servingRef) {
1741
+ // Adopt: export from the scope's current (un-rebound) dispatch → serving script,
1742
+ // then bind to the serving version. Data-first, so a failure here leaves the
1743
+ // scope intact on its old script for the next promote to retry.
1744
+ await adoptScopeOntoServing(c, s.tenantId, s.id, { acknowledge });
1745
+ }
1746
+ else if (s.verticalVersionId !== versionId) {
1747
+ // Already on the serving script (born there, or adopted earlier): routing is
1748
+ // pinned to servingRef, so advancing the version pointer only affects Update
1749
+ // offers. Snapshot on a migration-digest crossing (fork-before-promote, §4).
1750
+ await admin.bindScopeVersion(actor, s.tenantId, s.id, versionId, { snapshot: true, acknowledge });
1751
+ }
1639
1752
  }
1640
- else if (s.verticalVersionId !== versionId) {
1641
- // Already on the serving script (born there, or adopted earlier): routing is
1642
- // pinned to servingRef, so advancing the version pointer only affects Update
1643
- // offers. Snapshot on a migration-digest crossing (fork-before-promote, §4).
1644
- await admin.bindScopeVersion(actor, s.tenantId, s.id, versionId, { snapshot: true });
1753
+ catch (e) {
1754
+ if (e instanceof ExportBreakRefused)
1755
+ refused.push(...e.breaks);
1756
+ else if (isBindExportBreakRefusal(e) || (e instanceof ControlPlaneError && e.message.startsWith(BIND_EXPORT_BREAK_REFUSAL))) {
1757
+ refusal ??= e.message; // the host's own refusal (the answer moved under us): counts only
1758
+ }
1759
+ else
1760
+ throw e;
1645
1761
  }
1646
1762
  }
1763
+ return { refused, ...(refusal ? { refusal } : {}) };
1647
1764
  };
1648
1765
  /**
1649
1766
  * Move ONE scope onto a DIFFERENT vertical lineage's serving script (#389) — the
@@ -1677,15 +1794,24 @@ export function createControlPlaneApi(options) {
1677
1794
  throw new ControlPlaneError(409, `vertical '${target}' has no serving script yet — promote a version to prod first`);
1678
1795
  }
1679
1796
  if (scope.vertical === target && scope.servingRef === serving.ref) {
1797
+ // #1674: a rebind retried after its re-assert failed lands here; re-assert, as adopt does.
1798
+ await admin.reassertSystemSwitches(actor, { tenantId, scopeId });
1680
1799
  return { servingRef: serving.ref, versionId: serving.versionId, alreadyBound: true };
1681
1800
  }
1801
+ // #1756: within one lineage this is an adopt, and judged as one before anything moves. A
1802
+ // crossing is not judged (`bindExportBreaksOf`): the producer's slug itself changes.
1803
+ await refuseOnBreaks(actor, tenantId, scopeId, serving.versionId, {
1804
+ servingRef: serving.ref,
1805
+ acknowledge: opts.acknowledge,
1806
+ });
1807
+ const move = { acknowledge: opts.acknowledge };
1682
1808
  if (opts.abandonData) {
1683
1809
  // Directory-only crossing: no bytes move, so the frontier gate has nothing to
1684
1810
  // protect — the target provisions its own schema from scratch. The source
1685
1811
  // script's copy is untouched and remains the backout, same as a carried rebind.
1686
1812
  // The scope serves nothing until `/verticals/:slug/instances` re-provisions it.
1687
- await admin.setScopeServingRef(actor, tenantId, scopeId, serving.ref);
1688
- await admin.bindScopeVersion(actor, tenantId, scopeId, serving.versionId);
1813
+ await admin.setScopeServingRef(actor, tenantId, scopeId, serving.ref, move).catch(relayHostRefusal);
1814
+ await admin.bindScopeVersion(actor, tenantId, scopeId, serving.versionId, move).catch(relayHostRefusal);
1689
1815
  return { servingRef: serving.ref, versionId: serving.versionId, dataAbandoned: true };
1690
1816
  }
1691
1817
  // The frontier gate. Digest equality proves the target's migration set is exactly
@@ -1713,8 +1839,10 @@ export function createControlPlaneApi(options) {
1713
1839
  // `bindScopeVersion` rewrites `scopes.vertical` from the version row, audited. No
1714
1840
  // extra snapshot here (adopt-serving's precedent): the source script's copy is the
1715
1841
  // pre-migration state, and it is never deleted — that copy is the backout.
1716
- await admin.setScopeServingRef(actor, tenantId, scopeId, serving.ref);
1717
- await admin.bindScopeVersion(actor, tenantId, scopeId, serving.versionId);
1842
+ await admin.setScopeServingRef(actor, tenantId, scopeId, serving.ref, move).catch(relayHostRefusal);
1843
+ await admin.bindScopeVersion(actor, tenantId, scopeId, serving.versionId, move).catch(relayHostRefusal);
1844
+ // #1674: re-assert the recorded OFF positions in the store the scope now routes to.
1845
+ await admin.reassertSystemSwitches(actor, { tenantId, scopeId });
1718
1846
  return { servingRef: serving.ref, versionId: serving.versionId, tables: restored.tables };
1719
1847
  };
1720
1848
  /**
@@ -2034,7 +2162,8 @@ export function createControlPlaneApi(options) {
2034
2162
  : null;
2035
2163
  const ranAs = versionReachedAt(reached.via, scope, serving);
2036
2164
  try {
2037
- const result = await vertical.reconcileInstance({
2165
+ // #1674: the recorded OFF positions go back after the reconcile, before the receipt.
2166
+ const result = await reconcileThenReassert(admin, actor, { tenantId, scopeId }, () => vertical.reconcileInstance({
2038
2167
  tenantId,
2039
2168
  scopeId,
2040
2169
  entitlements,
@@ -2045,7 +2174,7 @@ export function createControlPlaneApi(options) {
2045
2174
  // by the vertical, so the reconcile must carry it into the same ready-gate — a bound
2046
2175
  // but unmigrated database fails as loudly as an absent one.
2047
2176
  ...(tenantStores.length ? { tenantStores } : {}),
2048
- });
2177
+ }));
2049
2178
  // #1172: the reconcile succeeded, so record WHICH version it ran against. Without
2050
2179
  // this the sweep would come back and do it again on the next pass — the console
2051
2180
  // button and the automatic phase have to write the same receipt, or pressing the
@@ -3090,6 +3219,9 @@ export function createControlPlaneApi(options) {
3090
3219
  const vertical = await verticalForScope(c, scope);
3091
3220
  if (vertical)
3092
3221
  await retryTransient(() => vertical.restoreScope(tenantId, scopeId, tables));
3222
+ // #1674: a backup from before a module was switched off brings it back on; the
3223
+ // directory's record puts it back off now, not at the next reconcile.
3224
+ await admin.reassertSystemSwitches(actor, { tenantId, scopeId });
3093
3225
  return c.json({ restored: scopeId, tables: tables.length });
3094
3226
  }
3095
3227
  catch (e) {
@@ -3158,10 +3290,15 @@ export function createControlPlaneApi(options) {
3158
3290
  force,
3159
3291
  localApply: !vertical,
3160
3292
  });
3161
- if (vertical) {
3162
- return c.json(await vertical.rewindScope(scopeId, bookmark, { force }));
3163
- }
3164
- return c.json(result);
3293
+ const answer = vertical ? await vertical.rewindScope(scopeId, bookmark, { force }) : result;
3294
+ // #1674: a rewind takes the scope's grants back to the bookmark, and with them any
3295
+ // schedule kill switch pulled since — a tenant owner could otherwise undo a staff
3296
+ // switch this way. Clearing the provisioned receipt, only once the rewind has gone
3297
+ // through, makes the next sweep reconcile the scope, and that reconcile re-asserts
3298
+ // what the directory records as off. Left to the sweep rather than done here: the
3299
+ // scope's store restarts to finish the restore, and a re-assert now would race it.
3300
+ await admin.markScopeProvisioned(actor, tenantId, scopeId, null);
3301
+ return c.json(answer);
3165
3302
  }
3166
3303
  catch (e) {
3167
3304
  if (e instanceof ControlPlaneError) {
@@ -3178,13 +3315,14 @@ export function createControlPlaneApi(options) {
3178
3315
  app.post('/tenants/:tenantId/scopes/:scopeId/adopt-serving', async (c) => {
3179
3316
  const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
3180
3317
  const scopeId = scopeIdSchema.parse(c.req.param('scopeId'));
3318
+ const { acknowledge } = adoptServingBody.parse(await c.req.json().catch(() => ({})));
3181
3319
  try {
3182
- const r = await adoptScopeOntoServing(c, tenantId, scopeId);
3320
+ const r = await adoptScopeOntoServing(c, tenantId, scopeId, { acknowledge });
3183
3321
  return c.json({ adopted: scopeId, ...r });
3184
3322
  }
3185
3323
  catch (e) {
3186
3324
  if (e instanceof ControlPlaneError) {
3187
- return c.json({ error: e.message }, e.status);
3325
+ return c.json(exportBreakBody(e), e.status);
3188
3326
  }
3189
3327
  throw e;
3190
3328
  }
@@ -3211,17 +3349,18 @@ export function createControlPlaneApi(options) {
3211
3349
  return c.json({ error: 'adopt-serving is a private-vertical operation' }, 409);
3212
3350
  }
3213
3351
  const owned = (await admin.listScopes(actor, { tenantId: v.ownerTenant, vertical: slug })).filter((s) => !s.forkedFrom && s.kind !== 'preview' && s.status === 'active');
3352
+ const { acknowledge } = adoptServingBody.parse(await c.req.json().catch(() => ({})));
3214
3353
  const adopted = [];
3215
3354
  const alreadyAdopted = [];
3216
3355
  try {
3217
3356
  for (const s of owned) {
3218
- const r = await adoptScopeOntoServing(c, s.tenantId, s.id);
3357
+ const r = await adoptScopeOntoServing(c, s.tenantId, s.id, { acknowledge });
3219
3358
  (r.alreadyAdopted ? alreadyAdopted : adopted).push(s.id);
3220
3359
  }
3221
3360
  }
3222
3361
  catch (e) {
3223
3362
  if (e instanceof ControlPlaneError) {
3224
- return c.json({ error: e.message, adopted, alreadyAdopted }, e.status);
3363
+ return c.json({ ...exportBreakBody(e), adopted, alreadyAdopted }, e.status);
3225
3364
  }
3226
3365
  throw e;
3227
3366
  }
@@ -3235,21 +3374,34 @@ export function createControlPlaneApi(options) {
3235
3374
  app.post('/tenants/:tenantId/scopes/:scopeId/rebind-vertical', async (c) => {
3236
3375
  const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
3237
3376
  const scopeId = scopeIdSchema.parse(c.req.param('scopeId'));
3238
- const { vertical, ackMigrations, abandonData } = rebindScopeVerticalBody.parse(await c.req.json());
3377
+ const { vertical, ackMigrations, abandonData, acknowledge } = rebindScopeVerticalBody.parse(await c.req.json());
3239
3378
  try {
3240
3379
  const r = await rebindScopeOntoVertical(c, tenantId, scopeId, vertical, {
3241
3380
  ackMigrations,
3242
3381
  abandonData,
3382
+ acknowledge,
3243
3383
  });
3244
3384
  return c.json({ rebound: scopeId, vertical, ...r });
3245
3385
  }
3246
3386
  catch (e) {
3247
3387
  if (e instanceof ControlPlaneError) {
3248
- return c.json({ error: e.message }, e.status);
3388
+ return c.json(exportBreakBody(e), e.status);
3249
3389
  }
3250
3390
  throw e;
3251
3391
  }
3252
3392
  });
3393
+ // #1756: which apps in this tenant binding `versionId` onto the scope would break — the
3394
+ // question the bind below refuses on, as a read, so a client can ask it before it records
3395
+ // anything of its own (the dashboard's Activity line) and show what an acknowledgement would
3396
+ // break. Refuses what the bind would refuse first (unknown scope or version, not admitted).
3397
+ // Callers: staff, and a tenant's own credential pinned to the tenant in the path; a builder's
3398
+ // push token is not in BUILDER_ROUTES. Every row is in that tenant.
3399
+ app.get('/tenants/:tenantId/scopes/:scopeId/binding-impact', async (c) => {
3400
+ const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
3401
+ const scopeId = scopeIdSchema.parse(c.req.param('scopeId'));
3402
+ const versionId = z.string().min(1).parse(c.req.query('versionId'));
3403
+ return c.json({ affected: await admin.bindingImpact(c.get('actor'), tenantId, scopeId, versionId) });
3404
+ });
3253
3405
  // Pin a scope to a vertical version (#31; orchestration.md §4). Refuses a
3254
3406
  // non-admitted version below the seam — that refusal is the registry's reason to
3255
3407
  // exist. A scope operation, so it keeps the scope route shape. `snapshot: true`
@@ -3265,9 +3417,16 @@ export function createControlPlaneApi(options) {
3265
3417
  app.post('/tenants/:tenantId/scopes/:scopeId/version', async (c) => {
3266
3418
  const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
3267
3419
  const scopeId = scopeIdSchema.parse(c.req.param('scopeId'));
3268
- const { versionId, snapshot } = bindScopeVersionBody.parse(await c.req.json());
3420
+ const { versionId, snapshot, acknowledge } = bindScopeVersionBody.parse(await c.req.json());
3269
3421
  const actor = c.get('actor');
3270
3422
  const scope = await admin.getScopeRecord(actor, tenantId, scopeId);
3423
+ // #1756: the apps in this tenant the bind would break, asked BEFORE the carry and the
3424
+ // snapshot below, which move data. The host refuses too, whoever calls, but by then a
3425
+ // refusal would land after the bytes had moved. It refuses what the bind would refuse
3426
+ // first (a version not admitted) the same way. An unknown scope has no impact to read, and
3427
+ // the bind below refuses it as it always has.
3428
+ if (scope)
3429
+ await refuseOnBreaks(actor, tenantId, scopeId, versionId, { acknowledge });
3271
3430
  // A refusal throws a ControlPlaneError, which the app's error boundary relays with its
3272
3431
  // status and records as an ops failure. An unknown scope carries nothing, and the bind
3273
3432
  // below refuses it as it always has.
@@ -3275,6 +3434,15 @@ export function createControlPlaneApi(options) {
3275
3434
  if (scope)
3276
3435
  await carryOntoVersion(c, scope, versionId);
3277
3436
  };
3437
+ // #1674: after the bind, the scope may route to a different store (the carry's
3438
+ // destination); put the directory's recorded OFF positions back there.
3439
+ const reassert = async () => {
3440
+ if (scope)
3441
+ await admin.reassertSystemSwitches(actor, { tenantId, scopeId });
3442
+ };
3443
+ // The host's own refusal, should the impact have changed since it was read above (a
3444
+ // promote landing in between): relayed as the same 409, never as a 500.
3445
+ const bind = (opts) => admin.bindScopeVersion(actor, tenantId, scopeId, versionId, opts).catch(relayHostRefusal);
3278
3446
  if (snapshot) {
3279
3447
  if (!scope) {
3280
3448
  return c.json({ error: `unknown scope for tenant: (${tenantId}, ${scopeId})` }, 404);
@@ -3292,18 +3460,24 @@ export function createControlPlaneApi(options) {
3292
3460
  migrationCrossing = Boolean(current && incoming && current.migrationDigest !== incoming.migrationDigest);
3293
3461
  }
3294
3462
  await carry();
3295
- if (migrationCrossing)
3463
+ if (migrationCrossing) {
3464
+ // Asked again after the carry: a promote landing since the first question would
3465
+ // otherwise leave an archive behind for a bind the host then refuses.
3466
+ await refuseOnBreaks(actor, tenantId, scopeId, versionId, { acknowledge });
3296
3467
  await orchestratedSnapshot(c, tenantId, scope, {});
3297
- await admin.bindScopeVersion(actor, tenantId, scopeId, versionId);
3468
+ }
3469
+ await bind({ acknowledge });
3298
3470
  }
3299
3471
  else {
3300
3472
  await carry();
3301
- await admin.bindScopeVersion(actor, tenantId, scopeId, versionId, { snapshot: true });
3473
+ await bind({ acknowledge, snapshot: true });
3302
3474
  }
3475
+ await reassert();
3303
3476
  return c.json(await admin.getScopeRecord(actor, tenantId, scopeId));
3304
3477
  }
3305
3478
  await carry();
3306
- await admin.bindScopeVersion(actor, tenantId, scopeId, versionId);
3479
+ await bind({ acknowledge });
3480
+ await reassert();
3307
3481
  return c.json(await admin.getScopeRecord(actor, tenantId, scopeId));
3308
3482
  });
3309
3483
  // -- instances (K-31) -------------------------------------------------------
@@ -3405,6 +3579,14 @@ export function createControlPlaneApi(options) {
3405
3579
  // one guessed from the serving pointer would claim a hook ran against code this
3406
3580
  // scope was not running.
3407
3581
  const provisioned = await admin.getScopeRecord(c.get('actor'), input.tenantId, input.scopeId);
3582
+ // #1674: this route is idempotent (K-31), so an already-bound scope that was wiped can
3583
+ // come back through it, re-seated live by the deployment — and the receipt below means
3584
+ // no sweep follows. Re-assert the recorded OFF positions before it. Only once the
3585
+ // directory row exists: a brand-new install's is written after this call (see above),
3586
+ // has no record to re-assert, and `reassertSystemSwitches` refuses an unknown scope.
3587
+ if (provisioned) {
3588
+ await admin.reassertSystemSwitches(c.get('actor'), { tenantId: input.tenantId, scopeId: input.scopeId });
3589
+ }
3408
3590
  if (provisioned?.verticalVersionId) {
3409
3591
  await admin.markScopeProvisioned(c.get('actor'), input.tenantId, input.scopeId, provisioned.verticalVersionId);
3410
3592
  }
@@ -3660,6 +3842,28 @@ export function createControlPlaneApi(options) {
3660
3842
  return c.json({ error: 'forbidden' }, 403);
3661
3843
  }
3662
3844
  input.verticalSlug = slug;
3845
+ // `manifestJson` is a string the schema cannot look inside, and every reader parses it
3846
+ // with `storedDeployManifest` and 500s on a row it refuses: the per-version reads below
3847
+ // (registry, schedules, flow, model, assets), the serving-upload rebuild that promote
3848
+ // and backout run (`serveVersionInPlace`), and `tenant-stores.ts`.
3849
+ // Hold it to that same parser here, at the trust boundary. The STORED
3850
+ // form, not the push-time one: this route keeps accepting what the platform already holds
3851
+ // (no `registry`, pre-#286 nulls), and refuses only what no reader could open. The string
3852
+ // is stored as sent — re-serializing would drop keys a newer publisher carries.
3853
+ if (input.manifestJson != null) {
3854
+ let raw;
3855
+ try {
3856
+ raw = JSON.parse(input.manifestJson);
3857
+ }
3858
+ catch {
3859
+ return c.json({ error: 'manifestJson is not valid JSON' }, 400);
3860
+ }
3861
+ const checked = storedDeployManifest.safeParse(raw);
3862
+ if (!checked.success) {
3863
+ const issue = checked.error.issues[0];
3864
+ return c.json({ error: `manifestJson is not a deploy manifest: ${issue.path.join('.') || '(root)'}: ${issue.message}` }, 400);
3865
+ }
3866
+ }
3663
3867
  await admin.publishVersion(c.get('actor'), input);
3664
3868
  const version = await admin.getVersion(c.get('actor'), input.id, slug);
3665
3869
  return c.json(version, 201);
@@ -3680,6 +3884,29 @@ export function createControlPlaneApi(options) {
3680
3884
  const registry = json ? (storedDeployManifest.parse(JSON.parse(json)).registry ?? null) : null;
3681
3885
  return c.json({ registry });
3682
3886
  });
3887
+ // The SQL migrations one version adds on top of another (#1677): what the promote dialog
3888
+ // and `substrat promote` show before a migration change is acknowledged. `?base=` names the
3889
+ // version to compare against (the one `prod` serves); without it every migration is listed.
3890
+ // Owner-narrowed like `/registry`, and never widened: migration SQL describes a schema.
3891
+ // `migrations` is null for a version that carries none (pushed before the field, or over
3892
+ // the push's cap), which a reader must show as "SQL not available", never as
3893
+ // "no migrations". Bounded in count and bytes (`migrationsOnTop`). The SQL is stored apart
3894
+ // from the manifest (#1764), so this reads it without parsing either manifest.
3895
+ app.get('/verticals/:slug/versions/:id/migrations', async (c) => {
3896
+ const p = c.get('principal');
3897
+ const slug = await resolveVerticalId(c, c.req.param('slug'));
3898
+ if (await notOwned(p, slug)) {
3899
+ return c.json({ error: 'not found' }, 404);
3900
+ }
3901
+ // `versionMigrations` refuses a version of another vertical, so `base` cannot reach one.
3902
+ const migrationsOf = (id) => admin.versionMigrations(c.get('actor'), slug, id);
3903
+ const base = c.req.query('base');
3904
+ const [incoming, baseline] = await Promise.all([
3905
+ migrationsOf(c.req.param('id')),
3906
+ base === undefined ? undefined : migrationsOf(base),
3907
+ ]);
3908
+ return c.json({ migrations: incoming === null ? null : migrationsOnTop(incoming, baseline) });
3909
+ });
3683
3910
  // The emitted entity model (#1214) one version ships: entities, field schemas, parent
3684
3911
  // edges, declared lifecycles — read out of the same retained manifest as the registry
3685
3912
  // above, and owner-narrowed the same way. `model` is null for a version pushed by a
@@ -4024,6 +4251,58 @@ export function createControlPlaneApi(options) {
4024
4251
  return { minted: [], error: message };
4025
4252
  }
4026
4253
  };
4254
+ /**
4255
+ * Which installed apps promoting `versionId` to `channel` would break: the promote gate's own
4256
+ * answer (`promotionImpact`, judged from the channel's previous version), and — for a PRIVATE,
4257
+ * dispatch-backed vertical's prod — what adopting each owned install still on its own version
4258
+ * would break (#1756). That adopt runs after the serve and carries the promote's acknowledgement,
4259
+ * so the acknowledgement must be asked against both, or it would cover breaks nobody was shown.
4260
+ * Over-approximates for a host that cannot serve in place, where no adopt runs: an extra line
4261
+ * costs a read, a missing one a silent stall.
4262
+ */
4263
+ const promoteImpactOf = async (actor, slug, channel, versionId) => {
4264
+ const gate = await admin.promotionImpact(actor, slug, channel, versionId);
4265
+ if (channel !== 'prod')
4266
+ return gate;
4267
+ const v = await verticalOf(actor, slug);
4268
+ if (!v || v.ownerTenant === null || v.listed)
4269
+ return gate;
4270
+ const incoming = await admin.getVersion(actor, versionId, slug);
4271
+ if (!incoming?.deploymentRef || incoming.admission !== 'admitted')
4272
+ return gate;
4273
+ const lagging = (await admin.listScopes(actor, { tenantId: v.ownerTenant, vertical: slug, status: ['active'] })).filter((s) => !s.forkedFrom && s.kind !== 'preview' && !s.servingRef && s.verticalVersionId !== versionId);
4274
+ const seen = new Set(gate.map((b) => `${b.scopeId}|${b.type}`));
4275
+ const out = [...gate];
4276
+ for (const s of lagging) {
4277
+ // After the adopt it runs the promoted version, served in place: that pointer move, from
4278
+ // what it runs now, is the move to judge.
4279
+ for (const b of await admin.bindingImpact(actor, s.tenantId, s.id, versionId)) {
4280
+ const key = `${b.scopeId}|${b.type}`;
4281
+ if (!seen.has(key))
4282
+ (seen.add(key), out.push(b));
4283
+ }
4284
+ }
4285
+ return out;
4286
+ };
4287
+ // #1705 PR 3: which installed apps promoting `?versionId=` would break (an export they import,
4288
+ // dropped or re-versioned), BEFORE promoting, so a dialog can show it and ask for the
4289
+ // acknowledgement up front, as it does for the digests. The promote's own 409 carries the same
4290
+ // listing. Narrowed to the caller exactly as that 409 is. Owner-checked like the promote.
4291
+ app.get('/verticals/:slug/channels/:channel/promote-impact', async (c) => {
4292
+ const p = c.get('principal');
4293
+ const slug = await resolveVerticalId(c, c.req.param('slug'));
4294
+ const channel = channelName.parse(c.req.param('channel'));
4295
+ const versionId = z.string().min(1).parse(c.req.query('versionId'));
4296
+ const pin = confinedTenant(p);
4297
+ if (pin) {
4298
+ const v = await verticalOf(p.actor, slug);
4299
+ if (!v || v.ownerTenant !== pin)
4300
+ return c.json({ error: 'forbidden' }, 403);
4301
+ }
4302
+ const breaks = await promoteImpactOf(c.get('actor'), slug, channel, versionId);
4303
+ const { visible, otherTenants } = narrowToCaller(breaks, pin);
4304
+ return c.json({ affected: visible, ...(otherTenants ? { otherTenants } : {}) });
4305
+ });
4027
4306
  app.post('/verticals/:slug/channels/:channel/promote', async (c) => {
4028
4307
  const p = c.get('principal');
4029
4308
  const slug = await resolveVerticalId(c, c.req.param('slug'));
@@ -4060,15 +4339,48 @@ export function createControlPlaneApi(options) {
4060
4339
  }
4061
4340
  }
4062
4341
  const { versionId, acknowledge } = promoteVersionBody.parse(await c.req.json());
4342
+ // #1705 PR 3: the installed consumers a promotion breaks (an export they import, dropped or
4343
+ // re-versioned). The host refuses it without `exportBreak` whoever calls, in counts only,
4344
+ // because it does not know who is asking. This door knows, so it adds the listing, narrowed
4345
+ // the way the store backfill below is: a confined caller sees its own tenant's apps and the
4346
+ // rest as a count, never another tenant's id. Read only when there is something to say: on
4347
+ // the refusal, and before an acknowledged promotion moves the channel it is measured from.
4348
+ // The whole answer is kept beside the narrowed listing, so the refusal's sentence counts the
4349
+ // same breaks the listing lists (in counts, which name no tenant) — the host's own sentence
4350
+ // counts only the gate's half of them.
4351
+ let counted = [];
4352
+ const breaksOf = async () => {
4353
+ const breaks = await promoteImpactOf(c.get('actor'), slug, channel, versionId);
4354
+ counted = breaks;
4355
+ if (breaks.length === 0)
4356
+ return null;
4357
+ const { visible, otherTenants } = narrowToCaller(breaks, confinedTenant(p));
4358
+ return { affected: visible, ...(otherTenants ? { otherTenants } : {}) };
4359
+ };
4360
+ const exportBreaks = acknowledge?.exportBreak ? await breaksOf() : null;
4063
4361
  // The blast-radius moment: refuses a changed digest without the acknowledgement,
4064
4362
  // and refuses a non-admitted version. Both are enforced below the seam and
4065
4363
  // surface as a 4xx through mapError, not a 500.
4066
- await admin.promoteVersion(c.get('actor'), slug, channel, versionId, acknowledge);
4364
+ try {
4365
+ await admin.promoteVersion(c.get('actor'), slug, channel, versionId, acknowledge);
4366
+ }
4367
+ catch (err) {
4368
+ if (!isExportBreakRefusal(err))
4369
+ throw err;
4370
+ // The refusal stands whether or not the listing can be read. A listing that fails turns
4371
+ // into a note, never into a 500 that would hide that the promote was refused.
4372
+ const listing = await breaksOf().then((exportBreaks) => ({ exportBreaks }), (e) => ({
4373
+ exportBreaks: null,
4374
+ exportBreaksUnavailable: `which apps it breaks could not be read: ${e instanceof Error ? e.message : String(e)}`,
4375
+ }));
4376
+ return c.json({ error: listing.exportBreaks ? exportBreakRefusal(counted) : err.message, ...listing }, 409);
4377
+ }
4067
4378
  // The in-place serve (#286), prod only, AFTER every promote gate has passed —
4068
4379
  // uploading first would deploy to live scopes before the acknowledgement check.
4069
4380
  // A failed serve is NOT a failed promote: the channel moved (audited), old code
4070
4381
  // still serves, and promoting again retries the upload.
4071
4382
  let backfill = { minted: [] };
4383
+ let adopted = { refused: [] };
4072
4384
  if (channel === 'prod') {
4073
4385
  try {
4074
4386
  await serveVersionInPlace(c.get('actor'), slug, versionId);
@@ -4077,7 +4389,9 @@ export function createControlPlaneApi(options) {
4077
4389
  // dispatch-backed vertical (#321), in the correct order (serve → adopt → rebind),
4078
4390
  // so a legacy scope's data survives the promote instead of being stranded on a
4079
4391
  // fresh per-version script. Retry-safe: nothing rebound these scopes yet.
4080
- await adoptAndRebindOwnedScopes(c, slug, versionId);
4392
+ // Only the export-break acknowledgement is the adopt's to carry: the digest ones are the
4393
+ // promote's own, and a bind's strict acknowledgement refuses them (#1756).
4394
+ adopted = await adoptAndRebindOwnedScopes(c, slug, versionId, acknowledge?.exportBreak ? { exportBreak: true } : undefined);
4081
4395
  // #825: and mint whatever THIS version newly declares for the tenants already
4082
4396
  // installed — after the serve, so the declaration being read is the one now live.
4083
4397
  backfill = await backfillFleetStores(c, slug);
@@ -4091,6 +4405,31 @@ export function createControlPlaneApi(options) {
4091
4405
  }, 502);
4092
4406
  }
4093
4407
  }
4408
+ // #1756: served, and every install that could move has moved and been backfilled, but one
4409
+ // still on its own version would break apps in its tenant if moved onto the serving script —
4410
+ // a move the promote gate never judged, since that install was not on the channel's previous
4411
+ // version. It is left where it is. Promoting again acknowledged moves it.
4412
+ if (adopted.refused.length > 0 || adopted.refusal) {
4413
+ const sentence = adopted.refused.length > 0 ? bindExportBreakRefusal(adopted.refused) : adopted.refusal;
4414
+ return c.json({
4415
+ error: `promoted and served, but an app still on its own version was not moved onto the serving script: ` +
4416
+ `${sentence}. Promote again with the export-break acknowledgement to move it`,
4417
+ ...(adopted.refused.length > 0 ? { exportBreaks: { affected: adopted.refused } } : {}),
4418
+ // Narrowed as the success answer's is: a builder never reads another tenant's id here.
4419
+ ...(backfill.minted.length || backfill.error
4420
+ ? (() => {
4421
+ const { visible, otherTenants } = narrowToCaller(backfill.minted, confinedTenant(c.get('principal')));
4422
+ return {
4423
+ storeBackfill: {
4424
+ minted: visible,
4425
+ ...(otherTenants ? { otherTenants } : {}),
4426
+ ...(backfill.error ? { error: backfill.error } : {}),
4427
+ },
4428
+ };
4429
+ })()
4430
+ : {}),
4431
+ }, 409);
4432
+ }
4094
4433
  const promoted = (await admin.listChannels(c.get('actor'), slug)).find((ch) => ch.channel === channel);
4095
4434
  // The store backfill rides the response only when it has something to say (#825), so
4096
4435
  // nothing changes for the overwhelming majority of promotes — and when a store WAS
@@ -4103,12 +4442,11 @@ export function createControlPlaneApi(options) {
4103
4442
  // report must not be the one place that hands them the tenant ids of everyone who
4104
4443
  // installed their vertical. The rest is a count, which tells them the fleet was covered
4105
4444
  // without naming who is in it.
4106
- const p2 = c.get('principal');
4107
- const reportPin = confinedTenant(p2);
4108
- const visible = reportPin ? backfill.minted.filter((m) => m.tenantId === reportPin) : backfill.minted;
4109
- const otherTenants = new Set(backfill.minted.filter((m) => !visible.includes(m)).map((m) => m.tenantId)).size;
4445
+ const { visible, otherTenants } = narrowToCaller(backfill.minted, confinedTenant(c.get('principal')));
4110
4446
  return c.json({
4111
4447
  ...promoted,
4448
+ // What the acknowledged break reached, so a promoter who passed `exportBreak` reads it.
4449
+ ...(exportBreaks ? { exportBreaks } : {}),
4112
4450
  ...(backfill.minted.length || backfill.error
4113
4451
  ? {
4114
4452
  storeBackfill: {
@@ -4566,6 +4904,8 @@ export function createControlPlaneApi(options) {
4566
4904
  // than somebody else's, and needs no ownership check of its own to be safe.
4567
4905
  scopeId: z.string().min(1).max(64).optional(),
4568
4906
  vertical: z.string().min(1).max(200).optional(),
4907
+ // A coarser grouping of the same rows (#1767) — never a wider read.
4908
+ grain: z.enum(['surface', 'scope']).optional(),
4569
4909
  hours: z.coerce.number().int().min(1).max(72).default(24),
4570
4910
  since: z.string().datetime({ offset: true }).optional(),
4571
4911
  until: z.string().datetime({ offset: true }).optional(),
@@ -4574,6 +4914,7 @@ export function createControlPlaneApi(options) {
4574
4914
  tenantId,
4575
4915
  scopeId: c.req.query('scopeId') || undefined,
4576
4916
  vertical: c.req.query('vertical') || undefined,
4917
+ grain: c.req.query('grain') || undefined,
4577
4918
  hours: c.req.query('hours'), since: c.req.query('since'), until: c.req.query('until'),
4578
4919
  });
4579
4920
  try {
@@ -5240,6 +5581,8 @@ export function createControlPlaneApi(options) {
5240
5581
  if (existing.servingRef)
5241
5582
  await admin.setScopeServingRef(actor, tenantId, existing.id, null);
5242
5583
  await admin.bindScopeVersion(actor, tenantId, existing.id, opts.versionId);
5584
+ // #1674: re-assert any recorded OFF in the script the preview now routes to.
5585
+ await admin.reassertSystemSwitches(actor, { tenantId, scopeId: existing.id });
5243
5586
  // Renew (or clear) the preview's GC deadline so a reused preview does not silently die.
5244
5587
  await admin.setScopeExpiresAt(actor, tenantId, existing.id, expiresAt);
5245
5588
  const hostname = await bindPreviewHostname(actor, baseHostname, tenantId, existing.id, opts.tag, surface);
@@ -5599,6 +5942,36 @@ export function createControlPlaneApi(options) {
5599
5942
  }
5600
5943
  return c.json(await admin.systemGrantsStatus(actor, { tenantId, scopeId }));
5601
5944
  });
5945
+ // The fleet read (#1674): every scope with a module switched off, from the directory's
5946
+ // record of the switch, so it needs no walk of every scope's store. The record is not the
5947
+ // gate; the per-scope read above is where a drift between the two shows (`recorded`).
5948
+ //
5949
+ // **Staff and the platform service token ONLY.** Outside `/tenants/:id`, and in neither
5950
+ // BUILDER_ROUTES nor TENANT_ROUTES, so a builder and a tenant credential are both refused
5951
+ // by default-deny before this runs. The `confinedTenant` check is the second lock, not
5952
+ // the first: a fleet-wide read is exactly what a confined credential must never make, and
5953
+ // it must not start answering if this path is ever allowlisted for a narrowed read.
5954
+ app.get('/system-switches', async (c) => {
5955
+ if (confinedTenant(c.get('principal')) !== null) {
5956
+ return c.json({ error: 'forbidden: the schedule switch is staff-only' }, 403);
5957
+ }
5958
+ const q = systemSwitchesQuery.parse({
5959
+ position: c.req.query('position'),
5960
+ tenantId: c.req.query('tenantId'),
5961
+ scopeId: c.req.query('scopeId'),
5962
+ moduleId: c.req.query('moduleId'),
5963
+ vertical: c.req.query('vertical'),
5964
+ limit: c.req.query('limit'),
5965
+ cursor: c.req.query('cursor'),
5966
+ order: c.req.query('order'),
5967
+ });
5968
+ const { position, ...rest } = q;
5969
+ const entries = await admin.listSystemSwitches(c.get('actor'), {
5970
+ ...rest,
5971
+ ...(position === 'all' ? {} : { position }),
5972
+ });
5973
+ return c.json(pageOf(entries, q.limit, (e) => e.operationId));
5974
+ });
5602
5975
  // -- the peer kill switch (#1706) ------------------------------------------
5603
5976
  //
5604
5977
  // One calling vertical's access to ONE scope, off and back on — `revokeFromPeer` and
@@ -5721,6 +6094,51 @@ export function createControlPlaneApi(options) {
5721
6094
  const entries = await admin.listSweepRuns(c.get('actor'), filter);
5722
6095
  return c.json(pageOf(entries, filter.limit, (e) => e.id));
5723
6096
  });
6097
+ // -- the replay lever (#1705 PR 3): move a consumer's watermark on one edge ----
6098
+ // Staff (the console) and a tenant credential (the dashboard, for someone who may manage the
6099
+ // tenant's apps) may pull it: the edge is between two of the tenant's own apps, and what a
6100
+ // replay runs lands in the tenant's own scope. The peer switch's posture, for that reason.
6101
+ // Builders may not: this route is absent from BUILDER_ROUTES, which is default-deny.
6102
+ //
6103
+ // The acknowledgement is refused HERE with the sentence it stands for, rather than left to a
6104
+ // Zod message about a literal. The person pulling the lever has to be told, in words, that a
6105
+ // replay runs handlers again and repeats what they send or call outside the app.
6106
+ app.post('/tenants/:tenantId/scopes/:scopeId/import-cursor', async (c) => {
6107
+ const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
6108
+ const scopeId = scopeIdSchema.parse(c.req.param('scopeId'));
6109
+ const pin = confinedTenant(c.get('principal'));
6110
+ if (pin !== null && pin !== tenantId)
6111
+ return c.json({ error: 'forbidden' }, 403);
6112
+ const raw = await c.req.json().catch(() => ({}));
6113
+ const refused = importCursorAcknowledgementMissing(raw);
6114
+ if (refused)
6115
+ return c.json({ error: refused }, 400);
6116
+ const move = importCursorMove.parse(raw);
6117
+ const actor = c.get('actor');
6118
+ // K-3 first, so a scope of another tenant reads as absent before anything is reached.
6119
+ if (!(await admin.getScopeRecord(actor, tenantId, scopeId))) {
6120
+ return c.json({ error: `unknown scope for tenant: (${tenantId}, ${scopeId})` }, 404);
6121
+ }
6122
+ return c.json(await admin.moveImportCursor(actor, tenantId, scopeId, move));
6123
+ });
6124
+ // -- edge health (#1705 PR 3): where each cross-vertical edge of one tenant stands -
6125
+ // Read live, through the phase's own reach, so it cannot disagree with the next pass about
6126
+ // which edges exist. Staff and the tenant's own credential, tenant-pinned (the dashboard shows
6127
+ // a tenant its own apps' edges). Builders may not: absent from BUILDER_ROUTES. `?scopeId=`
6128
+ // narrows it to one scope's edges, so a per-app view asks nothing about the rest.
6129
+ app.get('/tenants/:tenantId/cross-vertical/edges', async (c) => {
6130
+ const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
6131
+ const pin = confinedTenant(c.get('principal'));
6132
+ if (pin !== null && pin !== tenantId)
6133
+ return c.json({ error: 'forbidden' }, 403);
6134
+ const focus = c.req.query('scopeId');
6135
+ return c.json(await crossVerticalHealth(host, {
6136
+ actor: c.get('actor'),
6137
+ tenantId,
6138
+ ...(focus ? { focus: scopeIdSchema.parse(focus) } : {}),
6139
+ ...(options.crossVertical ? { crossVertical: options.crossVertical } : {}),
6140
+ }));
6141
+ });
5724
6142
  // -- issues (#1233): failures grouped by fingerprint, with a lifecycle --------
5725
6143
  // Staff-only (absent from BUILDER_ROUTES): an issue is a fleet-scoped aggregate
5726
6144
  // with no tenant column, so the forced-filter posture cannot narrow it — a