@substrat-run/control-plane-api 0.139.0 → 0.141.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/dist/api.d.ts +17 -1
  2. package/dist/api.d.ts.map +1 -1
  3. package/dist/api.js +701 -160
  4. package/dist/api.js.map +1 -1
  5. package/dist/audited-change.d.ts +139 -0
  6. package/dist/audited-change.d.ts.map +1 -0
  7. package/dist/audited-change.js +214 -0
  8. package/dist/audited-change.js.map +1 -0
  9. package/dist/cf-observability.d.ts.map +1 -1
  10. package/dist/cf-observability.js +38 -1
  11. package/dist/cf-observability.js.map +1 -1
  12. package/dist/connect-links.d.ts +29 -0
  13. package/dist/connect-links.d.ts.map +1 -0
  14. package/dist/connect-links.js +139 -0
  15. package/dist/connect-links.js.map +1 -0
  16. package/dist/connect-url.d.ts +20 -1
  17. package/dist/connect-url.d.ts.map +1 -1
  18. package/dist/connect-url.js +1 -1
  19. package/dist/connect-url.js.map +1 -1
  20. package/dist/connection-relay.d.ts +24 -0
  21. package/dist/connection-relay.d.ts.map +1 -1
  22. package/dist/connection-relay.js +31 -12
  23. package/dist/connection-relay.js.map +1 -1
  24. package/dist/email-intent.d.ts +71 -0
  25. package/dist/email-intent.d.ts.map +1 -0
  26. package/dist/email-intent.js +186 -0
  27. package/dist/email-intent.js.map +1 -0
  28. package/dist/field-coverage-read.d.ts +44 -0
  29. package/dist/field-coverage-read.d.ts.map +1 -0
  30. package/dist/field-coverage-read.js +45 -0
  31. package/dist/field-coverage-read.js.map +1 -0
  32. package/dist/field-coverage-tally.d.ts +12 -0
  33. package/dist/field-coverage-tally.d.ts.map +1 -1
  34. package/dist/field-coverage-tally.js +55 -14
  35. package/dist/field-coverage-tally.js.map +1 -1
  36. package/dist/index.d.ts +8 -2
  37. package/dist/index.d.ts.map +1 -1
  38. package/dist/index.js +7 -3
  39. package/dist/index.js.map +1 -1
  40. package/dist/observability.d.ts +13 -0
  41. package/dist/observability.d.ts.map +1 -1
  42. package/dist/platform-drain.d.ts +12 -2
  43. package/dist/platform-drain.d.ts.map +1 -1
  44. package/dist/platform-drain.js +15 -3
  45. package/dist/platform-drain.js.map +1 -1
  46. package/dist/reconcile.d.ts +17 -2
  47. package/dist/reconcile.d.ts.map +1 -1
  48. package/dist/reconcile.js +35 -1
  49. package/dist/reconcile.js.map +1 -1
  50. package/dist/scope-copy-cleanup.d.ts +76 -0
  51. package/dist/scope-copy-cleanup.d.ts.map +1 -0
  52. package/dist/scope-copy-cleanup.js +228 -0
  53. package/dist/scope-copy-cleanup.js.map +1 -0
  54. package/dist/staff-client.d.ts +3 -1
  55. package/dist/staff-client.d.ts.map +1 -1
  56. package/dist/staff-client.js +2 -0
  57. package/dist/staff-client.js.map +1 -1
  58. package/dist/vertical-client.d.ts +30 -2
  59. package/dist/vertical-client.d.ts.map +1 -1
  60. package/dist/vertical-client.js +100 -22
  61. package/dist/vertical-client.js.map +1 -1
  62. package/package.json +10 -10
package/dist/api.js CHANGED
@@ -1,15 +1,18 @@
1
1
  import { resolveObservabilityWindow } from './observability-window.js';
2
2
  import { Hono } from 'hono';
3
- import { parseHostname, withLabel, RESERVED_LABEL_SEPARATOR, adminAction, ownerTransferInput, OWNER_TRANSFER_AUDIT_ERROR_MAX, memberInviteInput, memberRoleInput, ASSET_PART_PREFIX, assetHash, channelName, connectionActivity, connectionActivitySource, connectionCredential, connectionFilter, connectionHealthState, connectionProvider, connectorDispatchKind, PROVISION_SIBLING_KIND, ARCHIVE_SCOPE_KIND, PROVISION_TENANT_KIND, SET_ENTITLEMENTS_KIND, MODEL_USAGE_KIND, SWEEP_RUNS_KIND, toConnectionHealthEntry, CONNECTION_EXPIRY_WARNING_DAYS, CONNECTION_STALE_DAYS, connectionProbe, createTenantInput, entitlementGrantInput, hostname as hostnameSchema, hostnameRegion, hostnameStatus, identityLink, errorCode, issueStatus, issueStatusInput, findingKind, findingRuleInput, findingStatus, findingStatusInput, listPageQuery, matchesOutboundHost, pageOf, platformRequestFilter, principalId as principalIdSchema, impersonationStamp, promotionAcknowledgement, bindAcknowledgement, provisionableJurisdiction, publishVersionInput, queryScopeInput, readScopeTableInput, entityHistoryInput, eventFacetInput, eventCauseInput, eventEffectsInput, invocationEventsInput, deadLettersInput, lifecycleFlowInput, operationSeriesCount, operationSeriesInput, delegatedReadParams, createOrgInput, roleKey as roleKeySchema, peerSwitch, systemSwitch, DEFAULT_DENIAL_LIMIT, DENIAL_LIMIT_MAX, denialGroupBy, registerVerticalInput, platformScriptCollision, platformScriptRefusal, PLATFORM_SCRIPT_NAMES, serviceDimensions, sweepRunKind, sweepRunOutcome, scopeDump, dataSubjectId as dataSubjectIdSchema, scopeId as scopeIdSchema, scopeStatus, storageShape, surfaceName, tenantId as tenantIdSchema, tenantStatus, versionOrigin, z, PROBLEM_CONTENT_TYPE, toProblem, redrainEventsInput, importCursorMove, importCursorAcknowledgementMissing, REDRAIN_BATCH, migrationsOnTop, errorCodeOf, capabilityFilterQuery, PLATFORM_FEATURES_HEADER, PLATFORM_FEATURE_SCOPE_SWEEPER, } from '@substrat-run/contracts';
3
+ import { parseHostname, withLabel, RESERVED_LABEL_SEPARATOR, adminAction, ownerTransferInput, memberInviteInput, memberRoleInput, ASSET_PART_PREFIX, assetHash, channelName, connectLinkId, connectionActivity, connectionActivitySource, connectionCredential, connectionCertificate, connectionFilter, connectionHealthState, connectionProvider, connectorDispatchKind, PROVISION_SIBLING_KIND, ARCHIVE_SCOPE_KIND, PROVISION_TENANT_KIND, SET_ENTITLEMENTS_KIND, MODEL_USAGE_KIND, SWEEP_RUNS_KIND, toConnectionHealthEntry, CONNECTION_EXPIRY_WARNING_DAYS, CONNECTION_STALE_DAYS, connectionProbe, createTenantInput, entitlementGrantInput, hostname as hostnameSchema, hostnameRegion, hostnameStatus, identityLink, errorCode, issueStatus, issueStatusInput, findingKind, findingRuleInput, findingStatus, findingStatusInput, listPageQuery, matchesOutboundHost, pageOf, platformRequestFilter, principalId as principalIdSchema, impersonationStamp, promotionAcknowledgement, bindAcknowledgement, provisionableJurisdiction, publishVersionInput, queryScopeInput, readScopeTableInput, entityHistoryInput, eventFacetInput, eventCauseInput, eventEffectsInput, invocationEventsInput, deadLettersInput, lifecycleFlowInput, operationSeriesCount, operationSeriesInput, delegatedReadParams, createOrgInput, roleKey as roleKeySchema, peerSwitch, verticalSlug, systemSwitch, DEFAULT_DENIAL_LIMIT, DENIAL_LIMIT_MAX, denialGroupBy, registerVerticalInput, platformScriptCollision, platformScriptRefusal, PLATFORM_SCRIPT_NAMES, serviceDimensions, sweepRunKind, sweepRunOutcome, scopeDump, dataSubjectId as dataSubjectIdSchema, scopeId as scopeIdSchema, scopeStatus, storageShape, surfaceName, tenantId as tenantIdSchema, tenantStatus, versionOrigin, z, PROBLEM_CONTENT_TYPE, toProblem, redrainEventsInput, importCursorMove, importCursorAcknowledgementMissing, REDRAIN_BATCH, migrationsOnTop, errorCodeOf, capabilityFilterQuery, PLATFORM_FEATURES_HEADER, PLATFORM_FEATURE_SCOPE_SWEEPER, } from '@substrat-run/contracts';
4
+ import { SCOPE_COPY_LEASE_MS, copyRestoreFence } from '@substrat-run/kernel';
4
5
  import { attributeFailure } from './failure-attribution.js';
5
- import { BIND_EXPORT_BREAK_REFUSAL, CARRIED_AWAY_KEY, bindExportBreakRefusal, carriedAwayDump, dumpMetaValue, metaValueIn, crossVerticalHealth, exportBreakRefusal, isBindExportBreakRefusal, isExportBreakRefusal, isPrimaryScope, migrationProgress, ulid, } from '@substrat-run/kernel';
6
+ import { auditedChange, withAuditedOutcomes } from './audited-change.js';
7
+ import { BIND_EXPORT_BREAK_REFUSAL, CARRIED_AWAY_KEY, bindExportBreakRefusal, carriedAwayDump, dumpMetaValue, metaValueIn, crossVerticalHealth, exportBreakRefusal, isBindExportBreakRefusal, isExportBreakRefusal, isPrimaryScope, migrationProgress, ulid, auditWarningOf, } from '@substrat-run/kernel';
6
8
  import { TENANT_HEADER, confinedTenant } from './auth.js';
7
9
  import { mintTenantToken } from './tenant-token.js';
8
10
  import { connectionGrantsForScope } from './vertical-client.js';
11
+ import { assertNoUnreachableScopeCopies, listAllScopeScriptCopies, reapScopeScriptCopies, recordStrandedStorage } from './scope-copy-cleanup.js';
9
12
  import { oidcCallbackUrl, retireClientsOfReapedScope, wirePreviewAuth } from './preview-auth.js';
10
13
  import { versionReachedAt } from './scope-deployment.js';
11
14
  import { reconcileConnectionGrants } from './connection-grants.js';
12
- import { reconcileThenReassert, switchCarryFor } from './reconcile.js';
15
+ import { reconcileThenReassert, reviewedEntityGrants, switchCarryFor } from './reconcile.js';
13
16
  import { ConnectionRelayError, relayConnectionUpsert } from './connection-relay.js';
14
17
  import { ControlPlaneError } from '@substrat-run/control-plane-client';
15
18
  import { provisionSiblingScope } from './platform-drain.js';
@@ -595,7 +598,16 @@ const TENANT_ROUTES = [
595
598
  { method: 'DELETE', re: /\/tenants\/[^/]+\/connections\/[^/]+$/, pin: 'path' },
596
599
  { method: 'GET', re: /\/tenants\/[^/]+\/connections\/[^/]+\/activity$/, pin: 'path' },
597
600
  { method: 'GET', re: /\/tenants\/[^/]+\/connections\/[^/]+\/credential$/, pin: 'path' },
601
+ // #2100: the PUBLIC certificate a connection signs in with — same pin as the credential view.
602
+ { method: 'GET', re: /\/tenants\/[^/]+\/connections\/[^/]+\/certificate$/, pin: 'path' },
598
603
  { method: 'POST', re: /\/tenants\/[^/]+\/connections\/[^/]+\/verify$/, pin: 'path' },
604
+ // A vertical's mailed connect links (connections.md §3.5.4): the consent landing's liveness
605
+ // read, the callback's spend and its undo, and the integrations card's list and revoke.
606
+ { method: 'GET', re: /\/tenants\/[^/]+\/connect-links$/, pin: 'path' },
607
+ { method: 'GET', re: /\/tenants\/[^/]+\/connect-links\/[^/]+$/, pin: 'path' },
608
+ { method: 'POST', re: /\/tenants\/[^/]+\/connect-links\/[^/]+\/consume$/, pin: 'path' },
609
+ { method: 'POST', re: /\/tenants\/[^/]+\/connect-links\/[^/]+\/restore$/, pin: 'path' },
610
+ { method: 'DELETE', re: /\/tenants\/[^/]+\/connect-links\/[^/]+$/, pin: 'path' },
599
611
  // Members: identity links mirrored from the dashboard, orgs, role assignments.
600
612
  { method: 'GET', re: /\/tenants\/[^/]+\/identities$/, pin: 'path' },
601
613
  { method: 'PUT', re: /\/tenants\/[^/]+\/identities$/, pin: 'path' },
@@ -633,6 +645,8 @@ const TENANT_ROUTES = [
633
645
  { method: 'GET', re: /\/tenants\/[^/]+\/scopes\/[^/]+\/migrations$/, pin: 'path' },
634
646
  { method: 'GET', re: /\/tenants\/[^/]+\/scopes\/[^/]+\/owner-seat$/, pin: 'path' },
635
647
  { method: 'GET', re: /\/tenants\/[^/]+\/scopes\/[^/]+\/peer-grants$/, pin: 'path' },
648
+ { method: 'GET', re: /\/tenants\/[^/]+\/scopes\/[^/]+\/peer-bindings$/, pin: 'path' },
649
+ { method: 'PUT', re: /\/tenants\/[^/]+\/scopes\/[^/]+\/peer-bindings$/, pin: 'path' },
636
650
  // The peer switch (#1706): on is POST, off is DELETE, on the tenant's own scope.
637
651
  { method: 'POST', re: /\/tenants\/[^/]+\/scopes\/[^/]+\/peer-grants$/, pin: 'path' },
638
652
  { method: 'DELETE', re: /\/tenants\/[^/]+\/scopes\/[^/]+\/peer-grants$/, pin: 'path' },
@@ -684,6 +698,7 @@ const TENANT_ROUTES = [
684
698
  { method: 'GET', re: /\/observability\/tenant-metrics$/, pin: 'query' },
685
699
  { method: 'GET', re: /\/observability\/tenant-metrics-series$/, pin: 'query' },
686
700
  { method: 'GET', re: /\/observability\/tenant-logs$/, pin: 'query' },
701
+ { method: 'GET', re: /\/observability\/tenant-field-coverage$/, pin: 'query' },
687
702
  // #1746: the request record's histogram, facets and list — same grain, same forced tenant.
688
703
  { method: 'GET', re: /\/observability\/tenant-request-volume$/, pin: 'query' },
689
704
  { method: 'GET', re: /\/observability\/tenant-request-facets$/, pin: 'query' },
@@ -786,6 +801,7 @@ const namedTenants = (c) => {
786
801
  */
787
802
  export function createControlPlaneApi(options) {
788
803
  const { host, authenticate, authenticateBuilder, authenticateTenantService } = options;
804
+ const copyLeaseMs = options.copyLeaseMs ?? SCOPE_COPY_LEASE_MS;
789
805
  const app = new Hono();
790
806
  // The CLI version advisory (#971), stamped on EVERY response — including the 401 the
791
807
  // auth middleware below answers with, since a CLI too old to authenticate is exactly
@@ -947,6 +963,7 @@ export function createControlPlaneApi(options) {
947
963
  { method: 'GET', re: /\/observability\/tenant-metrics$/ },
948
964
  { method: 'GET', re: /\/observability\/tenant-metrics-series$/ },
949
965
  { method: 'GET', re: /\/observability\/tenant-logs$/ },
966
+ { method: 'GET', re: /\/observability\/tenant-field-coverage$/ },
950
967
  // #1746: the same lines, counted and listed. Tenant-keyed, so no fleet-wide fallback.
951
968
  { method: 'GET', re: /\/observability\/tenant-request-volume$/ },
952
969
  { method: 'GET', re: /\/observability\/tenant-request-facets$/ },
@@ -1588,7 +1605,7 @@ export function createControlPlaneApi(options) {
1588
1605
  return c.json({ error: 'body tenantId disagrees with the route tenant' }, 400);
1589
1606
  }
1590
1607
  try {
1591
- return c.json(await relayConnectionUpsert(c.var.host, c.get('actor'), { ...body, tenantId }, { probeCandidate }));
1608
+ return c.json(await relayConnectionUpsert(c.var.host, c.get('actor'), { ...body, tenantId }, { probeCandidate, prepareCandidate }));
1592
1609
  }
1593
1610
  catch (err) {
1594
1611
  if (err instanceof ConnectionRelayError && err.status < 500) {
@@ -1618,6 +1635,80 @@ export function createControlPlaneApi(options) {
1618
1635
  await c.var.admin.revokeConnection(c.get('actor'), row.id);
1619
1636
  return c.body(null, 204);
1620
1637
  });
1638
+ // -- a vertical's mailed connect links, tenant-scoped (connections.md §3.5.4) ---------
1639
+ //
1640
+ // The dashboard's half of a link a VERTICAL minted (`/internal/connections/connect-links`):
1641
+ // its consent landing asks whether the row still stands, its callback spends the row
1642
+ // before storing and restores it if the store fails, and its integrations card lists and
1643
+ // revokes them. The tenant is the path's; the scope is named on every call and is half
1644
+ // the store's key, so another scope's link — or another tenant's — reads as absent (K-3).
1645
+ // Not in BUILDER_ROUTES: a builder has no business spending a tenant's consent round.
1646
+ const connectLinkScope = z.object({ scopeId: scopeIdSchema });
1647
+ app.get('/tenants/:tenantId/connect-links', async (c) => {
1648
+ const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
1649
+ const q = connectLinkScope
1650
+ .extend({ provider: connectionProvider.optional(), outstanding: z.literal('1').optional() })
1651
+ .parse({
1652
+ scopeId: c.req.query('scopeId'),
1653
+ provider: c.req.query('provider') || undefined,
1654
+ outstanding: c.req.query('outstanding') || undefined,
1655
+ });
1656
+ return c.json(await c.var.admin.listConnectLinks(c.get('actor'), {
1657
+ tenantId,
1658
+ scopeId: q.scopeId,
1659
+ ...(q.provider ? { provider: q.provider } : {}),
1660
+ ...(q.outstanding ? { outstandingOnly: true } : {}),
1661
+ }));
1662
+ });
1663
+ app.get('/tenants/:tenantId/connect-links/:linkId', async (c) => {
1664
+ const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
1665
+ const { scopeId } = connectLinkScope.parse({ scopeId: c.req.query('scopeId') });
1666
+ const link = await c.var.admin.getConnectLink(c.get('actor'), {
1667
+ tenantId,
1668
+ scopeId,
1669
+ id: connectLinkId.parse(c.req.param('linkId')),
1670
+ });
1671
+ return link ? c.json(link) : c.json({ error: 'unknown connect link' }, 404);
1672
+ });
1673
+ // The callback's spend. A refusal is a 200 carrying the typed reason, not an error status:
1674
+ // "already used" is the answer to the question asked, and the callback renders each one.
1675
+ app.post('/tenants/:tenantId/connect-links/:linkId/consume', async (c) => {
1676
+ const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
1677
+ const body = connectLinkScope
1678
+ .extend({
1679
+ provider: connectionProvider,
1680
+ accountRef: z.string().max(256).optional(),
1681
+ accountLabel: z.string().max(256).optional(),
1682
+ })
1683
+ .parse(await c.req.json());
1684
+ return c.json(await c.var.admin.consumeConnectLink(c.get('actor'), {
1685
+ ...body,
1686
+ tenantId,
1687
+ id: connectLinkId.parse(c.req.param('linkId')),
1688
+ }));
1689
+ });
1690
+ app.post('/tenants/:tenantId/connect-links/:linkId/restore', async (c) => {
1691
+ const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
1692
+ const { scopeId } = connectLinkScope.parse(await c.req.json());
1693
+ const restored = await c.var.admin.restoreConnectLink(c.get('actor'), {
1694
+ tenantId,
1695
+ scopeId,
1696
+ id: connectLinkId.parse(c.req.param('linkId')),
1697
+ });
1698
+ return c.json({ restored });
1699
+ });
1700
+ // Idempotent, as the store is: a revoked or spent link answers 204 again. 404 only for a
1701
+ // link this scope never held.
1702
+ app.delete('/tenants/:tenantId/connect-links/:linkId', async (c) => {
1703
+ const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
1704
+ const { scopeId } = connectLinkScope.parse({ scopeId: c.req.query('scopeId') });
1705
+ const link = await c.var.admin.revokeConnectLink(c.get('actor'), {
1706
+ tenantId,
1707
+ scopeId,
1708
+ id: connectLinkId.parse(c.req.param('linkId')),
1709
+ });
1710
+ return link ? c.body(null, 204) : c.json({ error: 'unknown connect link' }, 404);
1711
+ });
1621
1712
  /**
1622
1713
  * The tenant-scoped connection lookup the two inspection routes share. Same K-3
1623
1714
  * existence hiding as the revoke above — a foreign tenant's connection id reads
@@ -1630,6 +1721,8 @@ export function createControlPlaneApi(options) {
1630
1721
  * in which case the upsert behaves exactly as it always did, storing unverified.
1631
1722
  */
1632
1723
  const probeCandidate = async (provider, secret) => options.connectionInspectors?.[provider]?.probeCandidate?.(secret);
1724
+ /** #2100: the provider's own half of a credential, made before the probe sees it. */
1725
+ const prepareCandidate = (provider) => options.connectionInspectors?.[provider]?.prepareCandidate;
1633
1726
  const inspectableConnection = async (c) => {
1634
1727
  const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
1635
1728
  const id = c.req.param('id');
@@ -1703,6 +1796,20 @@ export function createControlPlaneApi(options) {
1703
1796
  }
1704
1797
  return c.json(connectionCredential.parse(await credential(c.var.host, row)));
1705
1798
  });
1799
+ /**
1800
+ * The public certificate a connection signs in with (#2100) — what the tenant downloads and
1801
+ * uploads to the provider. Public by construction: the private key never leaves the sealed
1802
+ * credential, and only a provider whose keypair the platform generates has one at all.
1803
+ */
1804
+ app.get('/tenants/:tenantId/connections/:id/certificate', async (c) => {
1805
+ const row = await inspectableConnection(c);
1806
+ if (!row)
1807
+ return c.json({ error: 'unknown connection' }, 404);
1808
+ const certificate = await options.connectionInspectors?.[row.provider]?.certificate?.(c.var.host, row);
1809
+ if (!certificate)
1810
+ return c.json({ error: 'this connection signs in with no certificate' }, 404);
1811
+ return c.json(connectionCertificate.parse(certificate));
1812
+ });
1706
1813
  // -- the scope directory (§3.2/§4.2) ---------------------------------------
1707
1814
  app.get('/scopes', async (c) => {
1708
1815
  const p = c.get('principal');
@@ -1962,6 +2069,7 @@ export function createControlPlaneApi(options) {
1962
2069
  exact: source.exact,
1963
2070
  loadStamp: source.loadStamp,
1964
2071
  expect: source.expect,
2072
+ ...(source.fence ? { fence: source.fence } : {}),
1965
2073
  ...(markCopy ? { markCopy } : {}),
1966
2074
  }));
1967
2075
  };
@@ -1999,13 +2107,9 @@ export function createControlPlaneApi(options) {
1999
2107
  if (!source || !dest) {
2000
2108
  throw new ControlPlaneError(501, 'adopt-serving needs dispatch resolution for both ends');
2001
2109
  }
2002
- const dump = await source.exportScope(scopeId);
2003
2110
  // #1742: the recorded OFF positions ride the restore, applied in the replay's own event.
2004
- const restored = await restoreCarryingSwitches(actor, dest, tenantId, scopeId, dump, { scopeId, exact: true });
2005
- // Data landed — only now flip routing and move the version pointer.
2006
- await c.var.admin
2007
- .setScopeServingRef(actor, tenantId, scopeId, serving.ref, { acknowledge: opts.acknowledge })
2008
- .catch(relayHostRefusal);
2111
+ // Data landed — only then does routing flip and the version pointer move.
2112
+ const restored = await moveOntoServing(c, scope, source, dest, serving.ref, { acknowledge: opts.acknowledge });
2009
2113
  await c.var.admin
2010
2114
  .bindScopeVersion(actor, tenantId, scopeId, serving.versionId, { acknowledge: opts.acknowledge })
2011
2115
  .catch(relayHostRefusal);
@@ -2085,8 +2189,9 @@ export function createControlPlaneApi(options) {
2085
2189
  * independent, and the scope's DO carries the SOURCE lineage's applied-migration
2086
2190
  * journal. The digest gate refuses the crossing unless the scope's bound version and
2087
2191
  * the target's serving version carry the same migration digest — or the operator
2088
- * acknowledges having read both surfaces (`ackMigrations`). The source script's copy
2089
- * is never deleted — flipping `servingRef` + version back is the backout.
2192
+ * acknowledges having read both surfaces (`ackMigrations`). Once the move is confirmed the
2193
+ * source script's copy is wiped by the sweep (#1722, `moveOntoServing`), so it is no backout:
2194
+ * rollback is an explicit snapshot or backup taken before the rebind.
2090
2195
  */
2091
2196
  const rebindScopeOntoVertical = async (c, tenantId, scopeId, target, opts) => {
2092
2197
  const actor = c.get('actor');
@@ -2118,10 +2223,25 @@ export function createControlPlaneApi(options) {
2118
2223
  const move = { acknowledge: opts.acknowledge };
2119
2224
  if (opts.abandonData) {
2120
2225
  // Directory-only crossing: no bytes move, so the frontier gate has nothing to
2121
- // protect — the target provisions its own schema from scratch. The source
2122
- // script's copy is untouched and remains the backout, same as a carried rebind.
2123
- // The scope serves nothing until `/verticals/:slug/instances` re-provisions it.
2124
- await c.var.admin.setScopeServingRef(actor, tenantId, scopeId, serving.ref, move).catch(relayHostRefusal);
2226
+ // protect — the target provisions its own schema from scratch. The source script's
2227
+ // copy is the only copy of the scope's data, and its deployment predates the export a
2228
+ // fenced wipe needs, so it is kept (`retained`), never swept, and reap and erasure still
2229
+ // reach it. The scope serves nothing until `/verticals/:slug/instances` re-provisions it.
2230
+ const sourceRef = await routeOf(c, scope);
2231
+ const moveId = sourceRef && sourceRef !== serving.ref ? ulid() : null;
2232
+ if (moveId && sourceRef) {
2233
+ await c.var.admin.recordScopeScriptCopy(actor, tenantId, scopeId, sourceRef, moveId, { role: 'source' });
2234
+ }
2235
+ try {
2236
+ const erasureEpoch = await c.var.admin.scopeErasureEpoch(actor, tenantId, scopeId);
2237
+ await c.var.admin.setScopeServingRef(actor, tenantId, scopeId, serving.ref, {
2238
+ ...move, expectedErasureEpoch: erasureEpoch, ...(moveId ? { confirmMove: { moveId, source: 'retained' } } : {}),
2239
+ }).catch(relayHostRefusal);
2240
+ }
2241
+ finally {
2242
+ if (moveId)
2243
+ await c.var.admin.settleScopeScriptCopy(actor, tenantId, scopeId, sourceRef, moveId, 'retained');
2244
+ }
2125
2245
  await c.var.admin.bindScopeVersion(actor, tenantId, scopeId, serving.versionId, move).catch(relayHostRefusal);
2126
2246
  return { servingRef: serving.ref, versionId: serving.versionId, dataAbandoned: true };
2127
2247
  }
@@ -2144,18 +2264,68 @@ export function createControlPlaneApi(options) {
2144
2264
  if (!source || !dest) {
2145
2265
  throw new ControlPlaneError(501, 'rebind-vertical needs dispatch resolution for both ends');
2146
2266
  }
2147
- const dump = await source.exportScope(scopeId);
2148
- const restored = await restoreCarryingSwitches(actor, dest, tenantId, scopeId, dump, { scopeId, exact: true }); // #1742, as adopt
2149
- // Data landed on the target script — only now flip routing and cross the pointer.
2150
- // `bindScopeVersion` rewrites `scopes.vertical` from the version row, audited. No
2151
- // extra snapshot here (adopt-serving's precedent): the source script's copy is the
2152
- // pre-migration state, and it is never deleted — that copy is the backout.
2153
- await c.var.admin.setScopeServingRef(actor, tenantId, scopeId, serving.ref, move).catch(relayHostRefusal);
2267
+ // #1742, as adopt. Data landed on the target script — only then do routing flip and the
2268
+ // pointer cross. `bindScopeVersion` rewrites `scopes.vertical` from the version row, audited.
2269
+ // No snapshot is taken here: the old copy is swept once the move is confirmed, so an
2270
+ // operator who wants a way back takes a snapshot or backup first.
2271
+ const restored = await moveOntoServing(c, scope, source, dest, serving.ref, move);
2154
2272
  await c.var.admin.bindScopeVersion(actor, tenantId, scopeId, serving.versionId, move).catch(relayHostRefusal);
2155
2273
  // #1674: re-assert the recorded OFF positions in the store the scope now routes to.
2156
2274
  await c.var.admin.reassertSystemSwitches(actor, { tenantId, scopeId }, { appliedInUnit: restored.switchedOff });
2157
2275
  return { servingRef: serving.ref, versionId: serving.versionId, tables: restored.tables };
2158
2276
  };
2277
+ /**
2278
+ * The copy move under adopt-serving and rebind-vertical (#1722): export from the script the
2279
+ * scope routes to, restore onto the serving script, then flip the route. Both ends are in the
2280
+ * ledger, leased to this move, before any bytes move, and the destination's entry names the
2281
+ * stamp its restore leaves, so a crash-recovery sweep can wipe an unbound restore if this
2282
+ * request dies. The route flips only while that lease is live and unclaimed, and the same
2283
+ * write settles the destination `done`.
2284
+ *
2285
+ * The old source is then `eligible`, as a carry's is: the scheduled sweep wipes it, fenced on
2286
+ * the stamp and revision this export read, so a copy that took a write since is kept for staff
2287
+ * rather than deleted. The old copy is not a backout; rollback is an explicit snapshot or
2288
+ * backup taken before the move (the decided policy). A source whose deployment predates the
2289
+ * stamped export cannot be wiped under a fence, and stays `retained`. A move that fails or is
2290
+ * never confirmed keeps its source too: the catch below settles both ends `retained`.
2291
+ */
2292
+ const moveOntoServing = async (c, scope, source, dest, servingRef, routeOpts) => {
2293
+ const actor = c.get('actor');
2294
+ const { tenantId, id: scopeId } = scope;
2295
+ const sourceRef = await routeOf(c, scope);
2296
+ const moveId = sourceRef !== servingRef ? ulid() : null;
2297
+ const restoredStamp = ulid();
2298
+ const recordedAt = Date.now();
2299
+ if (moveId && sourceRef) {
2300
+ await c.var.admin.recordScopeScriptCopy(actor, tenantId, scopeId, sourceRef, moveId, { role: 'source', leaseMs: copyLeaseMs });
2301
+ }
2302
+ try {
2303
+ if (moveId) {
2304
+ await c.var.admin.recordScopeScriptCopy(actor, tenantId, scopeId, servingRef, moveId, { role: 'destination', loadStamp: restoredStamp, leaseMs: copyLeaseMs });
2305
+ }
2306
+ const erasureEpoch = await c.var.admin.scopeErasureEpoch(actor, tenantId, scopeId);
2307
+ const { tables: dump, loadStamp: sourceStamp, revision: sourceRevision } = await retryTransient(() => source.exportScopeStamped(scopeId));
2308
+ const restored = await restoreCarryingSwitches(actor, dest, tenantId, scopeId, dump, {
2309
+ scopeId, exact: true, loadStamp: restoredStamp,
2310
+ ...(moveId ? { fence: copyRestoreFence(moveId, recordedAt, copyLeaseMs, erasureEpoch) } : {}),
2311
+ });
2312
+ await c.var.admin.setScopeServingRef(actor, tenantId, scopeId, servingRef, {
2313
+ ...routeOpts, expectedErasureEpoch: erasureEpoch,
2314
+ ...(moveId ? { confirmMove: sourceStamp
2315
+ ? { moveId, source: 'eligible', sourceMarker: { loadStamp: sourceStamp, revision: sourceRevision } }
2316
+ : { moveId, source: 'retained' } } : {}),
2317
+ }).catch(relayHostRefusal);
2318
+ return restored;
2319
+ }
2320
+ catch (error) {
2321
+ if (moveId)
2322
+ await Promise.allSettled([
2323
+ ...(sourceRef ? [c.var.admin.settleScopeScriptCopy(actor, tenantId, scopeId, sourceRef, moveId, 'retained')] : []),
2324
+ c.var.admin.settleScopeScriptCopy(actor, tenantId, scopeId, servingRef, moveId, 'retained'),
2325
+ ]);
2326
+ throw error;
2327
+ }
2328
+ };
2159
2329
  /** The script a scope's requests reach: its serving pin, else its bound version's own script. */
2160
2330
  const routeOf = async (c, scope) => {
2161
2331
  if (scope.servingRef)
@@ -2249,43 +2419,74 @@ export function createControlPlaneApi(options) {
2249
2419
  throw new ControlPlaneError(502, `cannot move scope ${scope.id} from '${from}' to '${to}': ${missing} could not be reached, ` +
2250
2420
  `so the version was not bound (#1710)`);
2251
2421
  }
2252
- const { tables: dump, loadStamp: sourceStamp, revision: sourceRevision } = await retryTransient(() => source.exportScopeStamped(scope.id));
2253
- // What the destination holds now, read before the binding is checked again. The restore
2254
- // sends it back and is refused if the store moved since: another carry into the same script
2255
- // loaded it, or it went live under another carry's bind and took a write. That is a CI retry
2256
- // of the same version, held before its restore while the first run bound and served.
2257
- const destMarker = await retryTransient(() => dest.loadMarker(scope.id));
2258
- const now = await c.var.admin.getScopeRecord(actor, scope.tenantId, scope.id);
2259
- if (!now ||
2260
- (now.verticalVersionId ?? null) !== (scope.verticalVersionId ?? null) ||
2261
- (now.servingRef ?? null) !== (scope.servingRef ?? null)) {
2262
- throw new ControlPlaneError(412, `scope ${scope.id} was re-pointed while its data was being copied out of '${from}', so nothing was ` +
2263
- `moved or bound (#1722). Reload the scope and retry.`);
2264
- }
2265
- if (dumpMetaValue(dump, CARRIED_AWAY_KEY) !== null) {
2266
- throw new ControlPlaneError(412, `the copy of scope ${scope.id} in '${from}' was already carried to another script and wiped, so nothing ` +
2267
- `was moved or bound (#1722). Reload the scope and retry.`);
2268
- }
2422
+ // Before exporting or restoring anything, retain the script identity in the directory, so a
2423
+ // failed carry stays named for reap and erasure. Both entries are leased to this move: if this
2424
+ // request dies, a sweep settles them once the lease runs out, and the move can then no longer
2425
+ // confirm its bind.
2426
+ const moveId = ulid();
2269
2427
  // #1742: the recorded OFF positions ride the restore, as on adopt and rebind. #1722: so does
2270
- // a stamp of this carry's own, which a refused bind's wipe of the copy expects.
2428
+ // a stamp of this carry's own, which a refused bind's wipe of the copy expects, and which the
2429
+ // destination's entry names before the restore so a crash-recovery sweep can fence on it.
2271
2430
  const restoredStamp = ulid();
2272
- const restored = await restoreCarryingSwitches(actor, dest, scope.tenantId, scope.id, dump, {
2273
- scopeId: scope.id,
2274
- exact: true,
2275
- loadStamp: restoredStamp,
2276
- ...(destMarker === 'unfenced' ? {} : { expect: destMarker }),
2277
- });
2278
- return {
2279
- from,
2280
- to,
2281
- tables: restored.tables,
2282
- switchedOff: restored.switchedOff,
2283
- source,
2284
- dest,
2285
- sourceStamp,
2286
- sourceRevision,
2287
- restoredStamp,
2288
- };
2431
+ const recordedAt = Date.now();
2432
+ await c.var.admin.recordScopeScriptCopy(actor, scope.tenantId, scope.id, from, moveId, { role: 'source', leaseMs: copyLeaseMs });
2433
+ try {
2434
+ // The unbound destination can also survive a refused bind or a failed best-effort
2435
+ // cleanup. Name it before any bytes are restored so reap and erasure can find it.
2436
+ await c.var.admin.recordScopeScriptCopy(actor, scope.tenantId, scope.id, to, moveId, { role: 'destination', loadStamp: restoredStamp, leaseMs: copyLeaseMs });
2437
+ const erasureEpoch = await c.var.admin.scopeErasureEpoch(actor, scope.tenantId, scope.id);
2438
+ const { tables: dump, loadStamp: sourceStamp, revision: sourceRevision } = await retryTransient(() => source.exportScopeStamped(scope.id));
2439
+ // What the destination holds now, read before the binding is checked again. The restore
2440
+ // sends it back and is refused if the store moved since: another carry into the same script
2441
+ // loaded it, or it went live under another carry's bind and took a write. That is a CI retry
2442
+ // of the same version, held before its restore while the first run bound and served.
2443
+ const destMarker = await retryTransient(() => dest.loadMarker(scope.id));
2444
+ const now = await c.var.admin.getScopeRecord(actor, scope.tenantId, scope.id);
2445
+ if (!now ||
2446
+ (now.verticalVersionId ?? null) !== (scope.verticalVersionId ?? null) ||
2447
+ (now.servingRef ?? null) !== (scope.servingRef ?? null)) {
2448
+ throw new ControlPlaneError(412, `scope ${scope.id} was re-pointed while its data was being copied out of '${from}', so nothing was ` +
2449
+ `moved or bound (#1722). Reload the scope and retry.`);
2450
+ }
2451
+ if (dumpMetaValue(dump, CARRIED_AWAY_KEY) !== null) {
2452
+ throw new ControlPlaneError(412, `the copy of scope ${scope.id} in '${from}' was already carried to another script and wiped, so nothing ` +
2453
+ `was moved or bound (#1722). Reload the scope and retry.`);
2454
+ }
2455
+ const restored = await restoreCarryingSwitches(actor, dest, scope.tenantId, scope.id, dump, {
2456
+ scopeId: scope.id,
2457
+ exact: true,
2458
+ loadStamp: restoredStamp,
2459
+ ...(destMarker === 'unfenced' ? {} : { expect: destMarker }),
2460
+ // The store refuses this load once the move's lease can have run out (#1722).
2461
+ fence: copyRestoreFence(moveId, recordedAt, copyLeaseMs, erasureEpoch),
2462
+ });
2463
+ return {
2464
+ moveId,
2465
+ erasureEpoch,
2466
+ from,
2467
+ to,
2468
+ tables: restored.tables,
2469
+ switchedOff: restored.switchedOff,
2470
+ source,
2471
+ dest,
2472
+ sourceStamp,
2473
+ sourceRevision,
2474
+ restoredStamp,
2475
+ };
2476
+ }
2477
+ catch (error) {
2478
+ // Neither script can be discarded based on this unconfirmed move. A retry or reap
2479
+ // can still reach both. A crash leaves them pending and blocks key finalization.
2480
+ const settled = await Promise.allSettled([
2481
+ c.var.admin.settleScopeScriptCopy(actor, scope.tenantId, scope.id, from, moveId, 'retained'),
2482
+ c.var.admin.settleScopeScriptCopy(actor, scope.tenantId, scope.id, to, moveId, 'retained'),
2483
+ ]);
2484
+ for (const result of settled) {
2485
+ if (result.status === 'rejected')
2486
+ recordCarryCleanup(c, scope, versionId, 'copy-ledger', result.reason);
2487
+ }
2488
+ throw error;
2489
+ }
2289
2490
  };
2290
2491
  /**
2291
2492
  * Wipe one copy of a scope that no route reaches (#1722), without reaping it. Fenced where the
@@ -2403,14 +2604,29 @@ export function createControlPlaneApi(options) {
2403
2604
  await dropUnboundCopy(c, scope, versionId, stale);
2404
2605
  throw new CarrySourceChanged(scope.id, stale.from, stale.to, attempt);
2405
2606
  }
2607
+ // The fresh carry restored over the stale one's copy in the same script and holds its own
2608
+ // entries for both ends, so the stale move leaves nothing of its own behind (#1722).
2609
+ await supersedeMove(c, scope, versionId, stale);
2406
2610
  carried = next;
2407
2611
  }
2612
+ // #1722: the write that moves the route confirms the move, while its lease is live. That is the
2613
+ // bind itself, unless a pin still routes the scope to the source until it is cleared below.
2614
+ // The source is confirmed `retained`, not `eligible`: `settleCarriedSource` may still have to
2615
+ // carry it again (an overtaken destination), and promotes it only once that is ruled out, so
2616
+ // no sweep can wipe it first. A request that dies in between leaves it kept, not swept.
2617
+ const unpins = Boolean(opts.clearServingRef && scope.servingRef);
2618
+ const confirmMove = carried
2619
+ ? { confirmMove: { moveId: carried.moveId, source: 'retained',
2620
+ sourceMarker: { loadStamp: carried.sourceStamp, revision: carried.sourceRevision } } }
2621
+ : {};
2408
2622
  try {
2409
2623
  await c.var.admin
2410
2624
  .bindScopeVersion(actor, scope.tenantId, scope.id, versionId, {
2411
2625
  acknowledge: opts.acknowledge,
2412
2626
  ...(opts.snapshot ? { snapshot: true } : {}),
2413
2627
  expectedVersionId: scope.verticalVersionId ?? null,
2628
+ ...(carried ? { expectedErasureEpoch: carried.erasureEpoch } : {}),
2629
+ ...(unpins ? {} : confirmMove),
2414
2630
  })
2415
2631
  .catch(relayHostRefusal);
2416
2632
  }
@@ -2419,8 +2635,15 @@ export function createControlPlaneApi(options) {
2419
2635
  await dropUnboundCopy(c, scope, versionId, carried);
2420
2636
  throw e;
2421
2637
  }
2422
- if (opts.clearServingRef && scope.servingRef) {
2423
- await c.var.admin.setScopeServingRef(actor, scope.tenantId, scope.id, null).catch(relayHostRefusal);
2638
+ if (unpins) {
2639
+ try {
2640
+ await c.var.admin.setScopeServingRef(actor, scope.tenantId, scope.id, null, carried ? { expectedErasureEpoch: carried.erasureEpoch, ...confirmMove } : undefined).catch(relayHostRefusal);
2641
+ }
2642
+ catch (e) {
2643
+ if (carried)
2644
+ await dropUnboundCopy(c, scope, versionId, carried);
2645
+ throw e;
2646
+ }
2424
2647
  }
2425
2648
  try {
2426
2649
  await c.var.admin.reassertSystemSwitches(actor, { tenantId: scope.tenantId, scopeId: scope.id }, { appliedInUnit: carried?.switchedOff });
@@ -2432,19 +2655,36 @@ export function createControlPlaneApi(options) {
2432
2655
  await settleCarriedSource(c, scope, versionId, carried);
2433
2656
  }
2434
2657
  };
2658
+ /** Settle a carry that a fresh carry into the same script replaced. Best effort, recorded. */
2659
+ const supersedeMove = async (c, scope, versionId, stale) => {
2660
+ const settled = await Promise.allSettled([stale.from, stale.to].map((ref) => c.var.admin.settleScopeScriptCopy(c.get('actor'), scope.tenantId, scope.id, ref, stale.moveId, 'done')));
2661
+ for (const result of settled) {
2662
+ if (result.status === 'rejected')
2663
+ recordCarryCleanup(c, scope, versionId, 'copy-ledger', result.reason);
2664
+ }
2665
+ };
2435
2666
  /**
2436
2667
  * The copy a carry restored for a bind that did not land, wiped unless the scope routes to
2437
2668
  * that script now. Best effort: it never throws, and a failure is recorded.
2438
2669
  */
2439
2670
  const dropUnboundCopy = async (c, scope, versionId, carried) => {
2671
+ let route = null;
2440
2672
  try {
2441
- const route = await currentRoute(c, scope, versionId, carried);
2442
- if (route === carried.to)
2673
+ route = await currentRoute(c, scope, versionId, carried);
2674
+ if (route === carried.to) {
2675
+ await c.var.admin.settleScopeScriptCopy(c.get('actor'), scope.tenantId, scope.id, carried.to, carried.moveId, 'done');
2443
2676
  return;
2444
- await wipeCarriedCopy(carried.dest, scope, { loadStamp: carried.restoredStamp }, route ?? carried.from);
2677
+ }
2678
+ const wiped = await wipeCarriedCopy(carried.dest, scope, { loadStamp: carried.restoredStamp }, route ?? carried.from);
2679
+ await c.var.admin.settleScopeScriptCopy(c.get('actor'), scope.tenantId, scope.id, carried.to, carried.moveId, wiped ? 'done' : 'retained');
2445
2680
  }
2446
2681
  catch (e) {
2447
2682
  recordCarryCleanup(c, scope, versionId, 'unbound-copy', e);
2683
+ await c.var.admin.settleScopeScriptCopy(c.get('actor'), scope.tenantId, scope.id, carried.to, carried.moveId, 'retained').catch((err) => recordCarryCleanup(c, scope, versionId, 'unbound-copy-ledger', err));
2684
+ }
2685
+ finally {
2686
+ await c.var.admin.settleScopeScriptCopy(c.get('actor'), scope.tenantId, scope.id, carried.from, carried.moveId, route === carried.from ? 'done' : 'retained')
2687
+ .catch((err) => recordCarryCleanup(c, scope, versionId, 'source-copy-ledger', err));
2448
2688
  }
2449
2689
  };
2450
2690
  /**
@@ -2526,8 +2766,10 @@ export function createControlPlaneApi(options) {
2526
2766
  // The caller read the route elsewhere, so a copy that changed in any way since the export (a
2527
2767
  // load as much as a write: a staff restore-forward into it racing this carry, Codex #2008 r8)
2528
2768
  // is kept in the store rather than left an unmarked orphan.
2529
- if (await wipeCarriedCopy(carried.source, scope, expect, carried.to, true))
2769
+ if (await wipeCarriedCopy(carried.source, scope, expect, carried.to, true)) {
2770
+ await c.var.admin.settleScopeScriptCopy(c.get('actor'), scope.tenantId, scope.id, carried.from, carried.moveId, 'done', expect);
2530
2771
  return;
2772
+ }
2531
2773
  // A rollback that restored into it since made it the live store again: it is no kept copy, and
2532
2774
  // a marker this refusal set while the rollback had not yet bound comes off.
2533
2775
  const route = await currentRoute(c, scope, versionId, carried);
@@ -2535,8 +2777,11 @@ export function createControlPlaneApi(options) {
2535
2777
  await releaseIfLive(c, scope, versionId, carried, carried.source, carried.from);
2536
2778
  return;
2537
2779
  }
2538
- if (route === carried.to && (await bringClearForward(c, scope, versionId, carried)))
2780
+ if (route === carried.to && (await bringClearForward(c, scope, versionId, carried))) {
2781
+ await c.var.admin.settleScopeScriptCopy(c.get('actor'), scope.tenantId, scope.id, carried.from, carried.moveId, 'done', expect);
2539
2782
  return;
2783
+ }
2784
+ await c.var.admin.settleScopeScriptCopy(c.get('actor'), scope.tenantId, scope.id, carried.from, carried.moveId, 'kept', expect);
2540
2785
  recordCarryCleanup(c, scope, versionId, 'source-copy-kept', new ControlPlaneError(409, `scope ${scope.id}'s copy in '${carried.from}' changed after it was carried to '${carried.to}', so it was ` +
2541
2786
  `kept: it holds what reached it in between, which the carry did not copy (#1722)`));
2542
2787
  };
@@ -2556,35 +2801,68 @@ export function createControlPlaneApi(options) {
2556
2801
  */
2557
2802
  const settleCarriedSource = async (c, scope, versionId, carried) => {
2558
2803
  const actor = c.get('actor');
2804
+ const finishDestination = async () => {
2805
+ if ((await currentRoute(c, scope, versionId, carried)) === carried.to) {
2806
+ await c.var.admin.settleScopeScriptCopy(actor, scope.tenantId, scope.id, carried.to, carried.moveId, 'done');
2807
+ }
2808
+ else {
2809
+ await dropUnboundCopy(c, scope, versionId, carried);
2810
+ }
2811
+ };
2559
2812
  try {
2560
2813
  const route = await currentRoute(c, scope, versionId, carried);
2561
- if (route === carried.from)
2814
+ if (route !== carried.to) {
2815
+ await dropUnboundCopy(c, scope, versionId, carried);
2562
2816
  return;
2817
+ }
2563
2818
  // The store this carry landed in is live; a kept marker another carry set while racing it
2564
2819
  // does not belong on it (#1722 r8).
2565
- if (route === carried.to)
2566
- await releaseIfLive(c, scope, versionId, carried, carried.dest, carried.to);
2567
- if (route === carried.to && (await isCarriedAway(carried.dest, scope))) {
2820
+ await releaseIfLive(c, scope, versionId, carried, carried.dest, carried.to);
2821
+ let sourceMarker = { loadStamp: carried.sourceStamp, revision: carried.sourceRevision };
2822
+ if (await isCarriedAway(carried.dest, scope)) {
2568
2823
  const { tables: dump, loadStamp: stamp, revision } = await retryTransient(() => carried.source.exportScopeStamped(scope.id));
2569
2824
  if (dumpMetaValue(dump, CARRIED_AWAY_KEY) !== null) {
2570
2825
  throw new ControlPlaneError(500, `both copies of scope ${scope.id} hold the carried_away tombstone; nothing was wiped`);
2571
2826
  }
2572
- await restoreCarryingSwitches(actor, carried.dest, scope.tenantId, scope.id, dump, {
2573
- scopeId: scope.id,
2574
- exact: true,
2575
- loadStamp: ulid(),
2576
- });
2827
+ // The re-carry is a copy move of its own (#1722): in the ledger and leased before it
2828
+ // writes, and fenced, so it cannot land after a reap or an erasure has settled the scope.
2829
+ const recarryId = ulid();
2830
+ const restoredStamp = ulid();
2831
+ const recordedAt = Date.now();
2832
+ await c.var.admin.recordScopeScriptCopy(actor, scope.tenantId, scope.id, carried.to, recarryId, { role: 'destination', loadStamp: restoredStamp, leaseMs: copyLeaseMs });
2833
+ try {
2834
+ const erasureEpoch = await c.var.admin.scopeErasureEpoch(actor, scope.tenantId, scope.id);
2835
+ await restoreCarryingSwitches(actor, carried.dest, scope.tenantId, scope.id, dump, {
2836
+ scopeId: scope.id,
2837
+ exact: true,
2838
+ loadStamp: restoredStamp,
2839
+ fence: copyRestoreFence(recarryId, recordedAt, copyLeaseMs, erasureEpoch),
2840
+ });
2841
+ }
2842
+ finally {
2843
+ // The scope routes to this store, so the route reaches whatever landed in it.
2844
+ await c.var.admin.settleScopeScriptCopy(actor, scope.tenantId, scope.id, carried.to, recarryId, 'done');
2845
+ }
2577
2846
  if (await isCarriedAway(carried.dest, scope)) {
2578
2847
  throw new ControlPlaneError(500, `scope ${scope.id}'s store in '${carried.to}' was wiped again; its source copy stays`);
2579
2848
  }
2580
2849
  // Re-carried, so the fence expects what the re-export read.
2581
- await keepOrWipeSource(c, scope, versionId, carried, { loadStamp: stamp, revision });
2850
+ sourceMarker = { loadStamp: stamp, revision };
2851
+ }
2852
+ // Only now may the sweep wipe the source: the destination holds the data.
2853
+ const eligible = await c.var.admin.settleScopeScriptCopy(actor, scope.tenantId, scope.id, carried.from, carried.moveId, 'eligible', sourceMarker);
2854
+ if (!eligible) {
2855
+ await finishDestination(); // a later move through this script superseded this carry
2582
2856
  return;
2583
2857
  }
2584
- await keepOrWipeSource(c, scope, versionId, carried, { loadStamp: carried.sourceStamp, revision: carried.sourceRevision });
2858
+ await keepOrWipeSource(c, scope, versionId, carried, sourceMarker);
2859
+ await finishDestination();
2585
2860
  }
2586
2861
  catch (e) {
2587
2862
  recordCarryCleanup(c, scope, versionId, 'source-copy', e);
2863
+ // This call can no longer restore another destination copy. Clear its pending
2864
+ // move if it is still serving, even when the source wipe needs a sweep retry.
2865
+ await finishDestination().catch((err) => recordCarryCleanup(c, scope, versionId, 'destination-copy-ledger', err));
2588
2866
  }
2589
2867
  };
2590
2868
  app.get('/tenants/:tenantId/scopes/:scopeId/tables', async (c) => {
@@ -2825,6 +3103,9 @@ export function createControlPlaneApi(options) {
2825
3103
  ? await c.var.admin.verticalServing(actor, scope.vertical).catch(() => null)
2826
3104
  : null;
2827
3105
  const ranAs = versionReachedAt(reached.via, scope, serving);
3106
+ // #2071: the shapes this reconcile tops holders up to, from the reviewed registry of the
3107
+ // version it reaches — never anything the vertical's own code names.
3108
+ const entityGrants = scope.vertical && ranAs ? await reviewedEntityGrants(c.var.admin, actor, scope.vertical, ranAs) : undefined;
2828
3109
  try {
2829
3110
  // #1674: the recorded OFF positions go back after the reconcile, before the receipt.
2830
3111
  const result = await reconcileThenReassert(c.var.admin, actor, { tenantId, scopeId }, (carry) => vertical.reconcileInstance({
@@ -2834,6 +3115,7 @@ export function createControlPlaneApi(options) {
2834
3115
  identityLinks,
2835
3116
  connectionGrants,
2836
3117
  connectionKeys,
3118
+ ...(entityGrants ? { entityGrants } : {}),
2837
3119
  ...carry,
2838
3120
  // Handed over exactly as at provision: a store minted HERE has never been migrated
2839
3121
  // by the vertical, so the reconcile must carry it into the same ready-gate — a bound
@@ -3256,28 +3538,46 @@ export function createControlPlaneApi(options) {
3256
3538
  return { refusal: c.json({ error: await diagnoseUnboundScope(actor, scope) }, 501) };
3257
3539
  return { tenantId, scopeId, actor, vertical, person: memberPerson(principal) };
3258
3540
  };
3259
- /** Run one audited member change: intent, the vertical's call, then the outcome row. */
3260
- const auditedMemberChange = async (c, entry, run, applied) => {
3261
- const actor = c.get('actor');
3262
- const base = { ...entry, operationId: ulid() };
3263
- await c.var.admin.recordMemberChange(actor, { ...base, phase: 'intent' });
3264
- let result;
3265
- try {
3266
- result = await run();
3267
- }
3268
- catch (e) {
3269
- const status = e instanceof ControlPlaneError ? e.status : 500;
3270
- const error = (e instanceof Error ? e.message : String(e)).slice(0, OWNER_TRANSFER_AUDIT_ERROR_MAX);
3271
- await c.var.admin
3272
- .recordMemberChange(actor, { ...base, phase: status < 500 ? 'refused' : 'failed', error })
3273
- .catch(() => undefined);
3274
- if (e instanceof ControlPlaneError) {
3275
- return { ok: false, response: c.json({ error: e.message }, e.status) };
3541
+ /**
3542
+ * Answer for one `auditedChange` (#2064). A refusal or failure answers with the vertical's own
3543
+ * status (a non-`ControlPlaneError` throw is rethrown). A change that went through answers
3544
+ * `ok`, with its `operationId`, whether or not its outcome row could be written. When it could
3545
+ * not, the answer also carries `auditWarning`, which says so. That is a success on purpose:
3546
+ * the change happened, so no client should read a retryable error and make it twice (a second
3547
+ * invite, a second hand-over). The missing row is logged, and the scheduled settle closes it.
3548
+ */
3549
+ const auditedAnswer = (c, done, what, ok) => {
3550
+ const { operationId } = done;
3551
+ if ('error' in done) {
3552
+ if (done.error instanceof ControlPlaneError) {
3553
+ return c.json({ error: done.error.message, operationId }, done.error.status);
3276
3554
  }
3277
- throw e;
3555
+ throw done.error;
3278
3556
  }
3279
- await c.var.admin.recordMemberChange(actor, { ...base, phase: 'applied', ...applied(result) });
3280
- return { ok: true, result };
3557
+ if ('unrecorded' in done) {
3558
+ return ok(done.result, {
3559
+ operationId,
3560
+ auditWarning: auditWarningOf(what, done.unrecorded),
3561
+ });
3562
+ }
3563
+ return ok(done.result, { operationId });
3564
+ };
3565
+ /** Run one audited member change: intent, the vertical's call, then the outcome row. */
3566
+ const auditedMemberChange = (c, entry, run, applied) => {
3567
+ const actor = c.get('actor');
3568
+ const operationId = ulid();
3569
+ return auditedChange({
3570
+ flow: 'member-change',
3571
+ operationId,
3572
+ run,
3573
+ // A 4xx is the vertical's refusal: nothing was written there.
3574
+ refused: (e) => e instanceof ControlPlaneError && e.status < 500,
3575
+ record: (row) => c.var.admin.recordMemberChange(actor, {
3576
+ ...entry,
3577
+ operationId,
3578
+ ...(row.phase === 'applied' ? { phase: 'applied', ...applied(row.result) } : row),
3579
+ }),
3580
+ });
3281
3581
  };
3282
3582
  const NO_PERSON = 'a member change needs the signed-in person it is made for — this credential names nobody';
3283
3583
  app.get('/tenants/:tenantId/scopes/:scopeId/members', async (c) => {
@@ -3306,7 +3606,7 @@ export function createControlPlaneApi(options) {
3306
3606
  }
3307
3607
  const { tenantId, scopeId, person } = target;
3308
3608
  const done = await auditedMemberChange(c, { tenantId, scopeId, change: 'invite', caller: person, roleKey: body.roleKey }, () => target.vertical.inviteMember({ tenantId, scopeId, caller: person, origin, roleKey: body.roleKey, email: body.email ?? null }), (link) => ({ principal: link.principal }));
3309
- return done.ok ? c.json(done.result, 201) : done.response;
3609
+ return auditedAnswer(c, done, 'the invite', (link, audit) => c.json({ ...link, ...audit }, 201));
3310
3610
  });
3311
3611
  app.post('/tenants/:tenantId/scopes/:scopeId/members/:principal/role', async (c) => {
3312
3612
  const target = await memberTarget(c);
@@ -3318,7 +3618,7 @@ export function createControlPlaneApi(options) {
3318
3618
  const body = memberRoleInput.parse(await c.req.json());
3319
3619
  const { tenantId, scopeId, person } = target;
3320
3620
  const done = await auditedMemberChange(c, { tenantId, scopeId, change: 'role', caller: person, principal: member, from: body.from, to: body.to }, () => target.vertical.changeMemberRole({ tenantId, scopeId, caller: person, principal: member, from: body.from, to: body.to }), () => ({}));
3321
- return done.ok ? c.json({ principal: member, from: body.from, to: body.to }) : done.response;
3621
+ return auditedAnswer(c, done, 'the role change', (_, audit) => c.json({ principal: member, from: body.from, to: body.to, ...audit }));
3322
3622
  });
3323
3623
  app.post('/tenants/:tenantId/scopes/:scopeId/members/:principal/remove', async (c) => {
3324
3624
  const target = await memberTarget(c);
@@ -3329,7 +3629,7 @@ export function createControlPlaneApi(options) {
3329
3629
  const member = principalIdSchema.parse(c.req.param('principal'));
3330
3630
  const { tenantId, scopeId, person } = target;
3331
3631
  const done = await auditedMemberChange(c, { tenantId, scopeId, change: 'remove', caller: person, principal: member }, () => target.vertical.removeMember({ tenantId, scopeId, caller: person, principal: member }), (removal) => ({ revoked: removal.revoked }));
3332
- return done.ok ? c.json(done.result) : done.response;
3632
+ return auditedAnswer(c, done, 'the removal', (removal, audit) => c.json({ ...removal, ...audit }));
3333
3633
  });
3334
3634
  // The owner HAND-OVER (#1665). The move runs in the VERTICAL, behind `/internal/owner-transfer`
3335
3635
  // (vertical-host says in what order and why). This route decides who may ask, and audits every
@@ -3360,41 +3660,19 @@ export function createControlPlaneApi(options) {
3360
3660
  // An abandon (`abandon: true`) is audited the same way, every row marked, so the log tells
3361
3661
  // a hand-over closed without finishing from one that finished.
3362
3662
  const base = { tenantId, scopeId, operationId, from: body.from, to: body.to, ...(body.abandon ? { abandon: true } : {}) };
3363
- await c.var.admin.recordOwnerTransfer(actor, { ...base, phase: 'intent' });
3364
- let moved;
3365
- try {
3366
- moved = await vertical.transferOwner({ tenantId, scopeId, ...body });
3367
- }
3368
- catch (e) {
3369
- const refused = e instanceof ControlPlaneError && e.status === 409;
3370
- const error = (e instanceof Error ? e.message : String(e)).slice(0, OWNER_TRANSFER_AUDIT_ERROR_MAX);
3371
- await c.var.admin
3372
- .recordOwnerTransfer(actor, { ...base, phase: refused ? 'refused' : 'failed', error })
3373
- .catch(() => undefined);
3374
- if (e instanceof ControlPlaneError) {
3375
- return c.json({ error: e.message, operationId }, e.status);
3376
- }
3377
- throw e;
3378
- }
3379
- try {
3380
- await c.var.admin.recordOwnerTransfer(actor, {
3381
- ...base,
3382
- phase: 'applied',
3383
- outcome: moved.outcome,
3384
- fromRevoked: moved.fromRevoked,
3385
- });
3386
- }
3387
- catch (e) {
3388
- // The owner HAS moved; only the outcome row is missing. Say exactly that, so nobody retries
3389
- // a hand-over believing it failed, and name the operation whose intent row stands alone.
3390
- const why = e instanceof Error ? e.message : String(e);
3391
- return c.json({
3392
- error: `the hand-over completed, but its outcome could not be written to the admin log: ${why}`,
3393
- operationId,
3394
- ...moved,
3395
- }, 500);
3396
- }
3397
- return c.json({ operationId, ...moved });
3663
+ const done = await auditedChange({
3664
+ flow: 'owner-transfer',
3665
+ operationId,
3666
+ run: () => vertical.transferOwner({ tenantId, scopeId, ...body }),
3667
+ // Only the vertical's 409 is a refusal: anything else may have stopped part-way.
3668
+ refused: (e) => e instanceof ControlPlaneError && e.status === 409,
3669
+ record: (row) => c.var.admin.recordOwnerTransfer(actor, row.phase === 'applied'
3670
+ ? { ...base, phase: 'applied', outcome: row.result.outcome, fromRevoked: row.result.fromRevoked }
3671
+ : { ...base, ...row }),
3672
+ });
3673
+ // A missing outcome row is said in `auditWarning` on a success, so nobody retries a hand-over
3674
+ // believing it failed, under the operation whose intent row stands alone.
3675
+ return auditedAnswer(c, done, 'the hand-over', (moved, audit) => c.json({ ...audit, ...moved }));
3398
3676
  });
3399
3677
  // Deliver per-instance CONFIG to the scope's own storage (vertical-auth-detach.md
3400
3678
  // §2.2) — the missing "delivery" step behind the dashboard's Env tab. Same K-3
@@ -3490,12 +3768,53 @@ export function createControlPlaneApi(options) {
3490
3768
  // on); the DIRECTORY half — provenance row, activation, version bind — runs here.
3491
3769
  // With no vertical client resolved (co-located host, tests, self-host) the host's
3492
3770
  // in-process snapshotScope does both halves against its own SCOPE namespace.
3771
+ /**
3772
+ * The script a slug-resolved scope lives in, by the hosted resolver's rule: the vertical's
3773
+ * stable serving script, else its prod channel's version script (control-plane worker
3774
+ * `resolveVerticalFor`). Null unless the platform can also reach that script by ref.
3775
+ */
3776
+ const slugRefOf = async (c, slug) => {
3777
+ const actor = c.get('actor');
3778
+ const serving = await c.var.admin.verticalServing(actor, slug).catch(() => null);
3779
+ let ref = serving?.ref ?? null;
3780
+ if (!ref) {
3781
+ const prod = (await c.var.admin.listChannels(actor, slug).catch(() => [])).find((ch) => ch.channel === 'prod');
3782
+ ref = prod ? ((await c.var.admin.getVersion(actor, prod.versionId, slug))?.deploymentRef ?? null) : null;
3783
+ }
3784
+ return ref && (await options.resolveVerticalRef?.(ref)) ? ref : null;
3785
+ };
3493
3786
  const orchestratedSnapshot = async (c, tenantId, scope, opts) => {
3494
3787
  const actor = c.get('actor');
3495
- const vertical = await verticalForScope(c, scope);
3788
+ const deployment = await deploymentForScope(c, scope);
3789
+ const vertical = deployment?.client;
3496
3790
  if (!vertical)
3497
3791
  return c.var.host.snapshotScope(actor, tenantId, scope.id, opts);
3498
3792
  const snapId = scopeIdSchema.parse(ulid());
3793
+ // #1722: the script the copy lands in (the source's own), named in the ledger before the
3794
+ // copy, so a bind that fails, or a request that dies, leaves a fork reap and erasure reach.
3795
+ // A pinned or bound source names it with the same value its client was resolved from. A
3796
+ // source resolved by slug is resolved once, here (`slugRefOf`), and copied through THAT
3797
+ // ref's client, so a serving move or a promote between two reads cannot put the bytes in
3798
+ // one script and the ledger entry on another. Without a ref (no per-script resolution, or
3799
+ // a slug script the platform cannot reach by ref) the fork has no entry, and reap and
3800
+ // erasure reach it through the slug resolution instead.
3801
+ let ref = null;
3802
+ let copier = vertical;
3803
+ if (options.resolveVerticalRef) {
3804
+ if (deployment.via === 'serving-script')
3805
+ ref = scope.servingRef ?? null;
3806
+ else if (deployment.via === 'bound-version' && scope.vertical && scope.verticalVersionId) {
3807
+ ref = (await c.var.admin.getVersion(actor, scope.verticalVersionId, scope.vertical))?.deploymentRef ?? null;
3808
+ }
3809
+ else if (deployment.via === 'slug' && scope.vertical) {
3810
+ ref = await slugRefOf(c, scope.vertical);
3811
+ const byRef = ref ? await options.resolveVerticalRef(ref) : undefined;
3812
+ if (byRef)
3813
+ copier = byRef;
3814
+ else
3815
+ ref = null;
3816
+ }
3817
+ }
3499
3818
  // Directory row FIRST, as `provisioning` (K-31's two-phase shape, used as
3500
3819
  // intended): a crash between the row and the data copy leaves an inert
3501
3820
  // provisioning row — which, carrying provenance and an expiry, the GC sweep
@@ -3510,12 +3829,34 @@ export function createControlPlaneApi(options) {
3510
3829
  forkedAt: new Date().toISOString(),
3511
3830
  expiresAt: opts.expiresAt,
3512
3831
  });
3513
- await retryTransient(() => vertical.snapshotScope({ sourceScopeId: scope.id, newScopeId: snapId, tenantId }));
3514
- await c.var.admin.activateScope(actor, tenantId, snapId);
3515
- // Bound to the SOURCE's current version: source and fork share a deployment, so
3516
- // the fork resolves to the DO namespace its bytes actually live in.
3517
- if (scope.verticalVersionId) {
3518
- await c.var.admin.bindScopeVersion(actor, tenantId, snapId, scope.verticalVersionId);
3832
+ const moveId = ref ? ulid() : null;
3833
+ try {
3834
+ if (moveId && ref) {
3835
+ // No load stamp: the vertical's in-script copy leaves none, so a crash-recovery sweep keeps
3836
+ // this copy (`retained`) rather than wiping it, and reap and erasure still reach it.
3837
+ await c.var.admin.recordScopeScriptCopy(actor, tenantId, snapId, ref, moveId, { role: 'destination', leaseMs: copyLeaseMs });
3838
+ }
3839
+ await retryTransient(() => copier.snapshotScope({ sourceScopeId: scope.id, newScopeId: snapId, tenantId }));
3840
+ await c.var.admin.activateScope(actor, tenantId, snapId);
3841
+ // Bound to the SOURCE's current version: source and fork share a deployment, so
3842
+ // the fork resolves to the DO namespace its bytes actually live in.
3843
+ if (scope.verticalVersionId) {
3844
+ await c.var.admin.bindScopeVersion(actor, tenantId, snapId, scope.verticalVersionId);
3845
+ }
3846
+ // The fork's route is the script its bytes were copied into, never the serving pointer
3847
+ // `provisionScope` stamped (which may have moved since), and the write that sets it confirms
3848
+ // the move. A fork of a bound source routes by its bound version, whose script is `ref`: its
3849
+ // stamped pin is cleared, so a later bind of the fork carries its data like any other scope's.
3850
+ // A fork of a pinned or slug-resolved source is pinned to `ref`, the script it lives in.
3851
+ if (moveId && ref) {
3852
+ await c.var.admin.setScopeServingRef(actor, tenantId, snapId, deployment.via === 'bound-version' ? null : ref, { confirmMove: { moveId, source: 'retained' } });
3853
+ }
3854
+ }
3855
+ catch (e) {
3856
+ if (moveId && ref) {
3857
+ await c.var.admin.settleScopeScriptCopy(actor, tenantId, snapId, ref, moveId, 'retained').catch(() => undefined);
3858
+ }
3859
+ throw e;
3519
3860
  }
3520
3861
  return snapId;
3521
3862
  };
@@ -3575,6 +3916,37 @@ export function createControlPlaneApi(options) {
3575
3916
  throw e;
3576
3917
  }
3577
3918
  };
3919
+ /**
3920
+ * Delete a scope's storage in every script that may hold it, before the directory forgets it.
3921
+ * Answers whether some of it is stranded (a script that predates `/internal/delete-scope`).
3922
+ *
3923
+ * #1722: with per-script resolution, the ledger names every copy a move or a fork wrote, and
3924
+ * the route names the live store; `reapScopeScriptCopies` reaches both, under the reap claim.
3925
+ * A scope nothing routes (no pin, no bound version) may still keep its store where its vertical
3926
+ * resolves by slug, which is where it was provisioned, so that deployment is reached too, as
3927
+ * it was before the ledger. Without per-script resolution there is only that deployment, and
3928
+ * a ledgered copy the platform cannot reach refuses the reap.
3929
+ */
3930
+ const reapScopeStorage = async (c, scope) => {
3931
+ const actor = c.get('actor');
3932
+ // The deployment the scope's vertical resolves to, deleted through; a 501 there is recorded
3933
+ // as stranded storage, as `reapScopeScriptCopies` records its own.
3934
+ const viaDeployment = async () => {
3935
+ const stranded = await deleteScopeStorageOrStrand(await verticalForScope(c, scope), scope.tenantId, scope.id);
3936
+ if (stranded) {
3937
+ await recordStrandedStorage(c.var.admin, actor, scope.tenantId, scope.id, `the deployment its vertical '${scope.vertical}' resolves to`);
3938
+ }
3939
+ return stranded;
3940
+ };
3941
+ if (!options.resolveVerticalRef) {
3942
+ await assertNoUnreachableScopeCopies(c.var.admin, actor, scope.tenantId, scope.id);
3943
+ return viaDeployment();
3944
+ }
3945
+ const unrouted = (await routeOf(c, scope)) === null;
3946
+ const { storageStranded } = await reapScopeScriptCopies({ admin: c.var.admin, actor, resolveRef: options.resolveVerticalRef }, scope.tenantId, scope.id);
3947
+ const bySlug = unrouted ? await viaDeployment() : false;
3948
+ return storageStranded || bySlug;
3949
+ };
3578
3950
  app.delete('/tenants/:tenantId/scopes/:scopeId', async (c) => {
3579
3951
  const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
3580
3952
  const scopeId = scopeIdSchema.parse(c.req.param('scopeId'));
@@ -3590,8 +3962,7 @@ export function createControlPlaneApi(options) {
3590
3962
  // local/placeholder wipe, hostnames + directory row, audit) — the same
3591
3963
  // storage-before-row ordering deleteSnapshot itself keeps, so a crash
3592
3964
  // between the two converges on retry.
3593
- const vertical = await verticalForScope(c, scope);
3594
- const storageStranded = await deleteScopeStorageOrStrand(vertical, tenantId, scopeId);
3965
+ const storageStranded = await reapScopeStorage(c, scope);
3595
3966
  await c.var.host.deleteSnapshot(actor, tenantId, scopeId);
3596
3967
  return c.json({ deleted: scopeId, ...(storageStranded ? { storageStranded: true } : {}) });
3597
3968
  }
@@ -3717,7 +4088,42 @@ export function createControlPlaneApi(options) {
3717
4088
  const scope = await c.var.admin.getScopeRecord(c.get('actor'), tenantId, scopeId);
3718
4089
  if (!scope)
3719
4090
  return c.json({ error: `unknown scope for tenant: (${tenantId}, ${scopeId})` }, 404);
3720
- return c.json(await c.var.admin.shredSubject(c.get('actor'), tenantId, scopeId, subjectId));
4091
+ const copies = await listAllScopeScriptCopies(c.var.admin, c.get('actor'), tenantId, scopeId);
4092
+ // A move in flight can still restore a copy no inventory read now would name. Its entries
4093
+ // settle when it confirms or fails, or, if it died, when the sweep finds its lease run out.
4094
+ if (copies.some((copy) => copy.state === 'pending')) {
4095
+ throw new ControlPlaneError(412, `scope ${scopeId} has a copy move in flight; retry the erasure after it settles`);
4096
+ }
4097
+ const erasureEpoch = await c.var.admin.scopeErasureEpoch(c.get('actor'), tenantId, scopeId);
4098
+ const currentRef = await routeOf(c, scope);
4099
+ if (!options.resolveVerticalRef) {
4100
+ if (currentRef || copies.some((copy) => copy.state !== 'done')) {
4101
+ throw new ControlPlaneError(502, `scope ${scopeId} has script copies, but this control plane cannot reach their deployments`);
4102
+ }
4103
+ return c.json(await c.var.admin.shredSubject(c.get('actor'), tenantId, scopeId, subjectId));
4104
+ }
4105
+ // A scope nothing routes may still keep its store where its vertical resolves by slug (where
4106
+ // it was provisioned), so erasure reaches that deployment too, as reap does (`reapScopeStorage`).
4107
+ const bySlug = currentRef ? undefined : await verticalForScope(c, scope);
4108
+ if (!currentRef && !bySlug && copies.every((copy) => copy.state === 'done')) {
4109
+ return c.json(await c.var.admin.shredSubject(c.get('actor'), tenantId, scopeId, subjectId));
4110
+ }
4111
+ const refs = new Set(copies.filter((copy) => copy.state !== 'done').map((copy) => copy.scriptRef));
4112
+ if (currentRef)
4113
+ refs.add(currentRef);
4114
+ const redactions = [];
4115
+ for (const ref of refs) {
4116
+ const script = await options.resolveVerticalRef(ref);
4117
+ if (!script)
4118
+ throw new ControlPlaneError(502, `scope ${scopeId}'s copy in '${ref}' cannot be reached for erasure`);
4119
+ redactions.push(await script.redactSubject(scopeId, subjectId));
4120
+ }
4121
+ if (bySlug)
4122
+ redactions.push(await bySlug.redactSubject(scopeId, subjectId));
4123
+ return c.json(await c.var.admin.finalizeSubjectShred(c.get('actor'), tenantId, scopeId, subjectId, redactions, {
4124
+ versionId: scope.verticalVersionId, servingRef: scope.servingRef ?? null, epoch: erasureEpoch,
4125
+ copyCount: copies.length,
4126
+ }).catch(relayHostRefusal));
3721
4127
  });
3722
4128
  // -- directory backups (#40) -----------------------------------------------
3723
4129
  // The platform's own disaster recovery, on the same store posture as the scope
@@ -4164,10 +4570,9 @@ export function createControlPlaneApi(options) {
4164
4570
  }
4165
4571
  }
4166
4572
  try {
4167
- const vertical = await verticalForScope(c, scope);
4168
4573
  // By this point the backup contract has resolved (a copy landed, or the caller
4169
4574
  // explicitly declined one), so stranding is a bookkeeping fact, not data loss.
4170
- const storageStranded = await deleteScopeStorageOrStrand(vertical, tenantId, scopeId);
4575
+ const storageStranded = await reapScopeStorage(c, scope);
4171
4576
  await c.var.admin.reapScope(actor, tenantId, scopeId, {
4172
4577
  ...(backup ? { backupRef: backupRefOf(backup) } : {}),
4173
4578
  });
@@ -6224,6 +6629,9 @@ export function createControlPlaneApi(options) {
6224
6629
  // `emailSender` grant itself is never touched by a push. Refreshes on every re-push
6225
6630
  // like the rest of the install spec, exactly because it grants nothing.
6226
6631
  ...(manifest.sendsEmail ? { sendsEmail: true } : {}),
6632
+ // Where the scope lifecycle is held (#1713): `'router'` keeps the vertical's scopes out of
6633
+ // the lifecycle delivery. Refreshes on every re-push like the rest of the install spec.
6634
+ ...(manifest.lifecycle ? { lifecycle: manifest.lifecycle } : {}),
6227
6635
  // The declared surfaces (K-26) ride like envSpec: registry metadata for the
6228
6636
  // hostname-binding picker, never behavior. Not part of any admission digest.
6229
6637
  ...(manifest.surfaces ? { surfaces: manifest.surfaces } : {}),
@@ -6517,6 +6925,45 @@ export function createControlPlaneApi(options) {
6517
6925
  const services = await appServices(c.get('actor'), input.tenantId, input.scopeId, input.vertical);
6518
6926
  return c.json(await options.observability.tenantLogs({ ...input, services }));
6519
6927
  });
6928
+ // #1331: one app's sampled field reports, joined to the router's own request lines.
6929
+ app.get('/observability/tenant-field-coverage', async (c) => {
6930
+ if (!options.observability?.fieldCoverage) {
6931
+ return c.json({ error: 'field coverage is not configured on this control plane' }, 501);
6932
+ }
6933
+ const tenantId = confinedTenant(c.get('principal')) ?? c.req.query('tenantId');
6934
+ if (!tenantId)
6935
+ throw new ControlPlaneError(400, 'tenantId is required');
6936
+ const input = z.object({
6937
+ tenantId: tenantIdSchema,
6938
+ scopeId: z.string().min(1).max(64),
6939
+ versionId: z.string().min(1).max(64),
6940
+ hours: z.coerce.number().int().min(1).max(72).default(24),
6941
+ since: z.string().datetime({ offset: true }).optional(),
6942
+ until: z.string().datetime({ offset: true }).optional(),
6943
+ }).parse({ tenantId, scopeId: c.req.query('scopeId'), versionId: c.req.query('versionId'),
6944
+ hours: c.req.query('hours'), since: c.req.query('since') || undefined, until: c.req.query('until') || undefined });
6945
+ try {
6946
+ resolveObservabilityWindow(input);
6947
+ }
6948
+ catch (e) {
6949
+ throw new ControlPlaneError(400, e.message);
6950
+ }
6951
+ const scopes = await host.admin.listScopes(c.get('actor'), { tenantId: input.tenantId });
6952
+ const installed = scopes.find((s) => s.id === input.scopeId);
6953
+ if (!installed?.vertical)
6954
+ return c.json({ error: 'app not found' }, 404);
6955
+ const services = scriptFamiliesOfScopes(scopes, { scopeId: input.scopeId });
6956
+ const json = await c.var.admin.versionManifest(c.get('actor'), installed.vertical, input.versionId);
6957
+ if (!json)
6958
+ return c.json({ error: 'version not found' }, 404);
6959
+ const outputSurface = storedDeployManifest.parse(JSON.parse(json)).outputSurface ?? [];
6960
+ const declared = Object.fromEntries(outputSurface.map((op) => [op.operationId, op.fields]));
6961
+ return c.json(await options.observability.fieldCoverage({
6962
+ tenantId: input.tenantId, vertical: installed.vertical, scopeId: input.scopeId, services, declared,
6963
+ versionId: input.versionId, hours: input.hours,
6964
+ ...(input.since ? { since: input.since, until: input.until } : {}),
6965
+ }));
6966
+ });
6520
6967
  // #1746: the per-request record, read at the tenant grain. The three routes take ONE
6521
6968
  // query — tenant, window, facet filters (`tenant-request-query.ts`) — so the histogram,
6522
6969
  // the facet counts and the list beside them always describe the same set of requests.
@@ -6962,15 +7409,29 @@ export function createControlPlaneApi(options) {
6962
7409
  * its hostnames. Storage-before-row, the same ordering the DELETE route uses — a crash
6963
7410
  * between the two converges on retry. Shared by that route and by the create path, which
6964
7411
  * reaps a HALF-BUILT leftover before re-forking (see `orchestratedPreview`). */
7412
+ /** Answers whether some of the preview's storage was stranded (and recorded as such). */
6965
7413
  const reapPreview = async (c, preview) => {
6966
7414
  // A fork's own sign-in clients go FIRST (#1704): a failure here leaves the preview in
6967
7415
  // place, so the retry a PR-close job makes still finds it — after the row is gone, nothing
6968
7416
  // would name those clients again. A clean room never had one.
6969
7417
  await retireClientsOfReapedScope(previewAuthDeps(c), preview);
6970
- const vertical = await verticalForScope(c, preview);
6971
- if (vertical)
6972
- await vertical.deleteScope({ tenantId: preview.tenantId, scopeId: preview.id });
7418
+ let storageStranded = false;
7419
+ if (options.resolveVerticalRef) {
7420
+ ({ storageStranded } = await reapScopeScriptCopies({
7421
+ admin: c.var.admin,
7422
+ actor: c.get('actor'),
7423
+ resolveRef: options.resolveVerticalRef,
7424
+ }, preview.tenantId, preview.id));
7425
+ }
7426
+ else {
7427
+ await assertNoUnreachableScopeCopies(c.var.admin, c.get('actor'), preview.tenantId, preview.id);
7428
+ const vertical = await verticalForScope(c, preview);
7429
+ if (vertical)
7430
+ await vertical.deleteScope({ tenantId: preview.tenantId, scopeId: preview.id });
7431
+ }
7432
+ // Co-located hosts have no script ref; their storage is wiped by deleteSnapshot.
6973
7433
  await c.var.host.deleteSnapshot(c.get('actor'), preview.tenantId, preview.id);
7434
+ return { storageStranded };
6974
7435
  };
6975
7436
  /** The preview-login seam (#1704, `preview-auth.ts`): the directory, and each issuer's deployment. */
6976
7437
  const previewAuthDeps = (c) => ({
@@ -7267,6 +7728,7 @@ export function createControlPlaneApi(options) {
7267
7728
  }
7268
7729
  };
7269
7730
  let row;
7731
+ /** `fenced`: the ledger stamp and lease of this fork's restore (#1722), when it has one. */
7270
7732
  let restore;
7271
7733
  /** #2009: the directory's classification of the row below, which every restore of it carries. */
7272
7734
  const markCopyOfRow = () => {
@@ -7305,7 +7767,7 @@ export function createControlPlaneApi(options) {
7305
7767
  // pushes a fresh version per try) — #559 (2).
7306
7768
  // #2009: a load classifies nothing by itself, so the restore carries the directory's
7307
7769
  // classification of the row below, as every restore onto a non-primary scope does.
7308
- restore = () => target.restoreScope(tenantId, previewId, tables, { sourceScopeId: source.id, exact: true, ...markCopyOfRow() });
7770
+ restore = (fenced) => target.restoreScope(tenantId, previewId, tables, { sourceScopeId: source.id, exact: true, ...markCopyOfRow(), ...fenced });
7309
7771
  }
7310
7772
  else {
7311
7773
  // A clean-room preview (#509 (b)): an EMPTY scope, no source to export. No `forkedFrom`
@@ -7324,19 +7786,49 @@ export function createControlPlaneApi(options) {
7324
7786
  expiresAt: expiresAt ?? undefined,
7325
7787
  };
7326
7788
  // #2009: marked a copy on the directory's word, as the fork above.
7327
- restore = target ? () => target.restoreScope(tenantId, previewId, [], markCopyOfRow()) : null;
7789
+ restore = target ? (fenced) => target.restoreScope(tenantId, previewId, [], { ...markCopyOfRow(), ...fenced }) : null;
7328
7790
  }
7791
+ // #1722: the PR version's script, named in the ledger before the restore writes into it.
7792
+ // Nothing routes the new row there until the bind, so without the entry a failure between
7793
+ // the two leaves a fork of the source's data that reap and erasure never reach.
7794
+ const targetRef = restore && options.resolveVerticalRef
7795
+ ? ((await c.var.admin.getVersion(actor, opts.versionId, slug))?.deploymentRef ?? null)
7796
+ : null;
7797
+ let fork = null;
7329
7798
  try {
7330
7799
  // Inside the try: a host writes the directory row FIRST and then migrates, projects and
7331
7800
  // seats, so a provision that throws can leave a `provisioning` row too. A provision
7332
7801
  // that never wrote one (the tag's slug taken by a racing create) marks nothing: the
7333
7802
  // write below targets this create's own id, which does not exist, and is swallowed.
7334
7803
  await c.var.host.provisionScope(actor, row);
7335
- if (restore)
7336
- await restoreOrRecord(restore);
7804
+ if (restore) {
7805
+ let fenced = {};
7806
+ if (targetRef) {
7807
+ const moveId = ulid();
7808
+ const restoredStamp = ulid();
7809
+ const recordedAt = Date.now();
7810
+ await c.var.admin.recordScopeScriptCopy(actor, tenantId, previewId, targetRef, moveId, { role: 'destination', loadStamp: restoredStamp, leaseMs: copyLeaseMs });
7811
+ fork = { moveId };
7812
+ const erasureEpoch = await c.var.admin.scopeErasureEpoch(actor, tenantId, previewId);
7813
+ fenced = { loadStamp: restoredStamp, fence: copyRestoreFence(moveId, recordedAt, copyLeaseMs, erasureEpoch) };
7814
+ }
7815
+ const load = restore;
7816
+ await restoreOrRecord(() => load(fenced));
7817
+ }
7337
7818
  await c.var.admin.activateScope(actor, tenantId, previewId);
7819
+ // Bind the PR version. A private vertical's push self-admitted, so this is accepted; a
7820
+ // preview scope also admits a pending version (#513), which is what a clean-room rehearsal
7821
+ // of not-yet-admitted code needs. It routes the row onto the fork, so it confirms the move.
7822
+ await c.var.admin.bindScopeVersion(actor, tenantId, previewId, opts.versionId, fork ? { confirmMove: { moveId: fork.moveId, source: 'retained' } } : undefined);
7338
7823
  }
7339
7824
  catch (e) {
7825
+ // The fork stays named for reap and erasure; if this request had died instead, the sweep
7826
+ // would settle the entry once its lease ran out.
7827
+ const unbound = fork;
7828
+ if (unbound && targetRef) {
7829
+ await c.var.admin.settleScopeScriptCopy(actor, tenantId, previewId, targetRef, unbound.moveId, 'retained')
7830
+ .catch(() => undefined);
7831
+ }
7340
7832
  // Expire the row this create left behind, so the next create reaps it at once rather
7341
7833
  // than refusing it as in flight for the whole bound (#1920) — that next create is the
7342
7834
  // CLI's retry after a known-dead attempt (#1918). The row stays until then, or until
@@ -7347,10 +7839,6 @@ export function createControlPlaneApi(options) {
7347
7839
  .catch(() => undefined);
7348
7840
  throw e;
7349
7841
  }
7350
- // Bind the PR version. A private vertical's push self-admitted, so this is accepted; a
7351
- // preview scope also admits a pending version (#513), which is what a clean-room rehearsal
7352
- // of not-yet-admitted code needs.
7353
- await c.var.admin.bindScopeVersion(actor, tenantId, previewId, opts.versionId);
7354
7842
  const hostname = await bindPreviewHostname(c.var.admin, actor, baseHostname, tenantId, previewId, opts.tag, surface);
7355
7843
  await assertServesBoundVersion(previewId);
7356
7844
  const auth = await wireLogin(previewId, hostname, source);
@@ -7466,8 +7954,8 @@ export function createControlPlaneApi(options) {
7466
7954
  // Storage-before-row, the same ordering as the fork hard-delete: wipe the DO in the
7467
7955
  // PR version's deployment, then deleteSnapshot (fork-only re-check, hostnames + row,
7468
7956
  // audit). A crash between the two converges on retry.
7469
- await reapPreview(c, preview);
7470
- return c.json({ deleted: preview.id });
7957
+ const { storageStranded } = await reapPreview(c, preview);
7958
+ return c.json({ deleted: preview.id, ...(storageStranded ? { storageStranded: true } : {}) });
7471
7959
  }
7472
7960
  catch (e) {
7473
7961
  if (e instanceof ControlPlaneError)
@@ -7667,6 +8155,56 @@ export function createControlPlaneApi(options) {
7667
8155
  }
7668
8156
  return c.json(await c.var.admin.peerGrantsStatus(actor, { tenantId, scopeId }));
7669
8157
  });
8158
+ // #1720: a caller's explicit target is a tenant decision. The dashboard checks its own
8159
+ // app-management permission; this route confines the tenant and both directory endpoints.
8160
+ const peerBindingBody = z.object({ vertical: verticalSlug, targetScopeId: scopeIdSchema.nullable() }).strict();
8161
+ app.get('/tenants/:tenantId/scopes/:scopeId/peer-bindings', async (c) => {
8162
+ const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
8163
+ const scopeId = scopeIdSchema.parse(c.req.param('scopeId'));
8164
+ const vertical = c.req.query('vertical');
8165
+ const pin = confinedTenant(c.get('principal'));
8166
+ if (pin !== null && pin !== tenantId)
8167
+ return c.json({ error: 'forbidden' }, 403);
8168
+ const actor = c.get('actor');
8169
+ if (!(await c.var.admin.getScopeRecord(actor, tenantId, scopeId)))
8170
+ return c.json({ error: 'app not found' }, 404);
8171
+ return c.json(vertical === undefined
8172
+ ? await c.var.admin.peerBindings(actor, tenantId, scopeId)
8173
+ : await c.var.admin.peerBinding(actor, tenantId, scopeId, verticalSlug.parse(vertical)) ?? null);
8174
+ });
8175
+ app.put('/tenants/:tenantId/scopes/:scopeId/peer-bindings', async (c) => {
8176
+ const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
8177
+ const scopeId = scopeIdSchema.parse(c.req.param('scopeId'));
8178
+ const pin = confinedTenant(c.get('principal'));
8179
+ // The target choice is the tenant's alone. Staff may inspect it through GET, but cannot
8180
+ // choose a tenant's site on its behalf.
8181
+ if (pin !== tenantId)
8182
+ return c.json({ error: 'forbidden' }, 403);
8183
+ const body = peerBindingBody.parse(await c.req.json());
8184
+ const actor = c.get('actor');
8185
+ if (!(await c.var.admin.getScopeRecord(actor, tenantId, scopeId)))
8186
+ return c.json({ error: 'app not found' }, 404);
8187
+ const operationId = ulid();
8188
+ const base = { tenantId, scopeId, operationId, vertical: body.vertical, targetScopeId: body.targetScopeId };
8189
+ const done = await auditedChange({
8190
+ flow: 'peer-binding', operationId,
8191
+ run: async () => {
8192
+ try {
8193
+ return await c.var.admin.setPeerBinding(actor, tenantId, scopeId, body.vertical, body.targetScopeId);
8194
+ }
8195
+ catch (e) {
8196
+ if (errorCodeOf(e) === 'not_found')
8197
+ throw new ControlPlaneError(404, e instanceof Error ? e.message : String(e));
8198
+ throw e;
8199
+ }
8200
+ },
8201
+ refused: (e) => e instanceof ControlPlaneError && e.status < 500,
8202
+ record: (row) => c.var.admin.recordPeerBindingChange(actor, row.phase === 'applied'
8203
+ ? { ...base, phase: 'applied', changed: row.result.changed, previousScopeId: row.result.previous }
8204
+ : { ...base, ...row }),
8205
+ });
8206
+ return auditedAnswer(c, done, 'the peer binding', (result, audit) => c.json({ ...result, ...audit }));
8207
+ });
7670
8208
  // Orgs: the portal-customer grouping (§4.1). Creating one mints no permission —
7671
8209
  // members reach what the org was GRANTED, and granting stays off this surface.
7672
8210
  app.post('/tenants/:tenantId/orgs', async (c) => {
@@ -7693,9 +8231,12 @@ export function createControlPlaneApi(options) {
7693
8231
  order: c.req.query('order'),
7694
8232
  });
7695
8233
  const entries = await c.var.admin.auditLog(c.get('actor'), filter);
8234
+ // #2064: each audited change's row says the outcome its operation stands at — the latest
8235
+ // outcome row wins — beside the raw row, which is returned as written.
8236
+ const resolved = await withAuditedOutcomes(c.var.admin, c.get('actor'), entries);
7696
8237
  // The cursor IS the last entry's id (ULID order is chronological), so the
7697
8238
  // page carries its own continuation and the console never assembles one.
7698
- return c.json(pageOf(entries, filter.limit, (e) => e.id));
8239
+ return c.json(pageOf(resolved, filter.limit, (e) => e.id));
7699
8240
  });
7700
8241
  // The recorded operational failures (#559) — the console's failures view, and the
7701
8242
  // "what does this `reference = <id>` belong to" lookup. Newest first by default,