@substrat-run/control-plane-api 0.135.0 → 0.137.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/dist/aggregate-reads.d.ts.map +1 -1
  2. package/dist/aggregate-reads.js +4 -1
  3. package/dist/aggregate-reads.js.map +1 -1
  4. package/dist/aggregate-source.d.ts +17 -1
  5. package/dist/aggregate-source.d.ts.map +1 -1
  6. package/dist/aggregate-source.js +16 -10
  7. package/dist/aggregate-source.js.map +1 -1
  8. package/dist/api.d.ts.map +1 -1
  9. package/dist/api.js +1030 -68
  10. package/dist/api.js.map +1 -1
  11. package/dist/cf-observability.d.ts +21 -0
  12. package/dist/cf-observability.d.ts.map +1 -1
  13. package/dist/cf-observability.js +150 -45
  14. package/dist/cf-observability.js.map +1 -1
  15. package/dist/connect-url.d.ts +2 -2
  16. package/dist/connect-url.d.ts.map +1 -1
  17. package/dist/connect-url.js +7 -1
  18. package/dist/connect-url.js.map +1 -1
  19. package/dist/connection-relay.d.ts +4 -2
  20. package/dist/connection-relay.d.ts.map +1 -1
  21. package/dist/connection-relay.js +10 -1
  22. package/dist/connection-relay.js.map +1 -1
  23. package/dist/deploy.d.ts +3 -0
  24. package/dist/deploy.d.ts.map +1 -1
  25. package/dist/deploy.js +3 -4
  26. package/dist/deploy.js.map +1 -1
  27. package/dist/failure-attribution.js +1 -1
  28. package/dist/failure-attribution.js.map +1 -1
  29. package/dist/field-coverage-tally.d.ts +58 -0
  30. package/dist/field-coverage-tally.d.ts.map +1 -0
  31. package/dist/field-coverage-tally.js +234 -0
  32. package/dist/field-coverage-tally.js.map +1 -0
  33. package/dist/index.d.ts +8 -5
  34. package/dist/index.d.ts.map +1 -1
  35. package/dist/index.js +7 -3
  36. package/dist/index.js.map +1 -1
  37. package/dist/observability.d.ts +20 -1
  38. package/dist/observability.d.ts.map +1 -1
  39. package/dist/observability.js +11 -1
  40. package/dist/observability.js.map +1 -1
  41. package/dist/platform-drain.d.ts +24 -2
  42. package/dist/platform-drain.d.ts.map +1 -1
  43. package/dist/platform-drain.js +36 -3
  44. package/dist/platform-drain.js.map +1 -1
  45. package/dist/platform-entry.d.ts +62 -6
  46. package/dist/platform-entry.d.ts.map +1 -1
  47. package/dist/platform-entry.generated.d.ts +8 -1
  48. package/dist/platform-entry.generated.d.ts.map +1 -1
  49. package/dist/platform-entry.generated.js +10 -3
  50. package/dist/platform-entry.generated.js.map +1 -1
  51. package/dist/platform-entry.js +151 -9
  52. package/dist/platform-entry.js.map +1 -1
  53. package/dist/provider-error.d.ts +9 -0
  54. package/dist/provider-error.d.ts.map +1 -0
  55. package/dist/provider-error.js +9 -0
  56. package/dist/provider-error.js.map +1 -0
  57. package/dist/reconcile.d.ts +10 -3
  58. package/dist/reconcile.d.ts.map +1 -1
  59. package/dist/reconcile.js +7 -2
  60. package/dist/reconcile.js.map +1 -1
  61. package/dist/service-family.d.ts +19 -0
  62. package/dist/service-family.d.ts.map +1 -0
  63. package/dist/service-family.js +27 -0
  64. package/dist/service-family.js.map +1 -0
  65. package/dist/tenant-request-query.d.ts.map +1 -1
  66. package/dist/tenant-request-query.js +3 -1
  67. package/dist/tenant-request-query.js.map +1 -1
  68. package/dist/vertical-client.d.ts +191 -40
  69. package/dist/vertical-client.d.ts.map +1 -1
  70. package/dist/vertical-client.js +351 -171
  71. package/dist/vertical-client.js.map +1 -1
  72. package/package.json +7 -7
package/dist/api.js CHANGED
@@ -1,8 +1,8 @@
1
1
  import { resolveObservabilityWindow } from './observability-window.js';
2
2
  import { Hono } from 'hono';
3
- import { parseHostname, withLabel, RESERVED_LABEL_SEPARATOR, adminAction, ownerTransferInput, OWNER_TRANSFER_AUDIT_ERROR_MAX, ASSET_PART_PREFIX, assetHash, channelName, connectionActivity, connectionActivitySource, connectionCredential, connectionFilter, connectionHealthState, connectionProvider, connectorDispatchKind, PROVISION_SIBLING_KIND, ARCHIVE_SCOPE_KIND, PROVISION_TENANT_KIND, SET_ENTITLEMENTS_KIND, MODEL_USAGE_KIND, SWEEP_RUNS_KIND, toConnectionHealthEntry, CONNECTION_EXPIRY_WARNING_DAYS, CONNECTION_STALE_DAYS, connectionProbe, createTenantInput, entitlementGrantInput, hostname as hostnameSchema, hostnameRegion, hostnameStatus, identityLink, errorCode, issueStatus, issueStatusInput, listPageQuery, matchesOutboundHost, pageOf, platformRequestFilter, principalId as principalIdSchema, impersonationStamp, promotionAcknowledgement, bindAcknowledgement, provisionableJurisdiction, publishVersionInput, queryScopeInput, readScopeTableInput, entityHistoryInput, eventFacetInput, eventCauseInput, eventEffectsInput, invocationEventsInput, deadLettersInput, lifecycleFlowInput, delegatedReadParams, createOrgInput, roleKey as roleKeySchema, peerSwitch, systemSwitch, DEFAULT_DENIAL_LIMIT, DENIAL_LIMIT_MAX, denialGroupBy, registerVerticalInput, serviceDimensions, sweepRunKind, sweepRunOutcome, scopeDump, dataSubjectId as dataSubjectIdSchema, scopeId as scopeIdSchema, scopeStatus, storageShape, surfaceName, tenantId as tenantIdSchema, tenantStatus, versionOrigin, z, PROBLEM_CONTENT_TYPE, toProblem, redrainEventsInput, importCursorMove, importCursorAcknowledgementMissing, REDRAIN_BATCH, migrationsOnTop, errorCodeOf, capabilityFilterQuery, } from '@substrat-run/contracts';
3
+ import { parseHostname, withLabel, RESERVED_LABEL_SEPARATOR, adminAction, ownerTransferInput, OWNER_TRANSFER_AUDIT_ERROR_MAX, ASSET_PART_PREFIX, assetHash, channelName, connectionActivity, connectionActivitySource, connectionCredential, connectionFilter, connectionHealthState, connectionProvider, connectorDispatchKind, PROVISION_SIBLING_KIND, ARCHIVE_SCOPE_KIND, PROVISION_TENANT_KIND, SET_ENTITLEMENTS_KIND, MODEL_USAGE_KIND, SWEEP_RUNS_KIND, toConnectionHealthEntry, CONNECTION_EXPIRY_WARNING_DAYS, CONNECTION_STALE_DAYS, connectionProbe, createTenantInput, entitlementGrantInput, hostname as hostnameSchema, hostnameRegion, hostnameStatus, identityLink, errorCode, issueStatus, issueStatusInput, listPageQuery, matchesOutboundHost, pageOf, platformRequestFilter, principalId as principalIdSchema, impersonationStamp, promotionAcknowledgement, bindAcknowledgement, provisionableJurisdiction, publishVersionInput, queryScopeInput, readScopeTableInput, entityHistoryInput, eventFacetInput, eventCauseInput, eventEffectsInput, invocationEventsInput, deadLettersInput, lifecycleFlowInput, delegatedReadParams, createOrgInput, roleKey as roleKeySchema, peerSwitch, systemSwitch, DEFAULT_DENIAL_LIMIT, DENIAL_LIMIT_MAX, denialGroupBy, registerVerticalInput, platformScriptCollision, platformScriptRefusal, PLATFORM_SCRIPT_NAMES, serviceDimensions, sweepRunKind, sweepRunOutcome, scopeDump, dataSubjectId as dataSubjectIdSchema, scopeId as scopeIdSchema, scopeStatus, storageShape, surfaceName, tenantId as tenantIdSchema, tenantStatus, versionOrigin, z, PROBLEM_CONTENT_TYPE, toProblem, redrainEventsInput, importCursorMove, importCursorAcknowledgementMissing, REDRAIN_BATCH, migrationsOnTop, errorCodeOf, capabilityFilterQuery, PLATFORM_FEATURES_HEADER, PLATFORM_FEATURE_SCOPE_SWEEPER, } from '@substrat-run/contracts';
4
4
  import { attributeFailure } from './failure-attribution.js';
5
- import { BIND_EXPORT_BREAK_REFUSAL, bindExportBreakRefusal, crossVerticalHealth, exportBreakRefusal, isBindExportBreakRefusal, isExportBreakRefusal, migrationProgress, ulid, } from '@substrat-run/kernel';
5
+ import { BIND_EXPORT_BREAK_REFUSAL, CARRIED_AWAY_KEY, bindExportBreakRefusal, carriedAwayDump, dumpMetaValue, metaValueIn, crossVerticalHealth, exportBreakRefusal, isBindExportBreakRefusal, isExportBreakRefusal, isPrimaryScope, migrationProgress, ulid, } from '@substrat-run/kernel';
6
6
  import { TENANT_HEADER, confinedTenant } from './auth.js';
7
7
  import { mintTenantToken } from './tenant-token.js';
8
8
  import { connectionGrantsForScope } from './vertical-client.js';
@@ -37,6 +37,7 @@ import { TENANT_SERIES_SCOPE_CAP, REQUEST_FACET_KEYS } from './observability.js'
37
37
  import { parseTenantRequestQuery, parseTenantWindowQuery, refineTenantWindow, scriptFamiliesOfScopes } from './tenant-request-query.js';
38
38
  import { namespacesForScript } from './do-namespaces.js';
39
39
  import { backupDirectoryIfDue } from './directory-backup.js';
40
+ import { deployedDoClasses, platformSweeperDecision } from './platform-entry.js';
40
41
  import { STORAGE_PAGE_DEFAULT, STORAGE_PAGE_MAX, readStoragePage } from './storage-meter.js';
41
42
  import { isCustomHostname, validateBindableHostname, } from './custom-hostnames.js';
42
43
  /**
@@ -46,6 +47,13 @@ import { isCustomHostname, validateBindableHostname, } from './custom-hostnames.
46
47
  */
47
48
  export const CLI_MIN_VERSION_HEADER = 'x-substrat-cli-min-version';
48
49
  export const CLI_LATEST_VERSION_HEADER = 'x-substrat-cli-latest-version';
50
+ /**
51
+ * The directory's classification of a scope that is not primary (#2005), which a verb sends so
52
+ * the store it writes is marked a copy whatever its own marker says; undefined for a primary one.
53
+ */
54
+ function copyLineageOf(scope) {
55
+ return isPrimaryScope(scope) ? undefined : { kind: scope.kind, forkedFrom: scope.forkedFrom };
56
+ }
49
57
  /**
50
58
  * "This principal is confined to a tenant, and this is not that tenant."
51
59
  *
@@ -70,6 +78,27 @@ class ExportBreakRefused extends ControlPlaneError {
70
78
  this.breaks = breaks;
71
79
  }
72
80
  }
81
+ /**
82
+ * A carry whose source store kept changing between its export and its bind (#1722, Codex #2008
83
+ * r12): every re-carry from a fresh export found the source changed again before it could bind.
84
+ * Nothing was bound and the destination's copy was dropped, so the caller retries the bind. Typed,
85
+ * so a client tells it from the other 409s: `carrySourceChanged` names the two scripts and how many
86
+ * exports were tried.
87
+ */
88
+ class CarrySourceChanged extends ControlPlaneError {
89
+ scopeId;
90
+ from;
91
+ to;
92
+ attempts;
93
+ constructor(scopeId, from, to, attempts) {
94
+ super(409, `scope ${scopeId}'s store in '${from}' changed while it was being carried to '${to}', on each of ` +
95
+ `${attempts} exports, so nothing was bound (#1722). Retry the bind.`);
96
+ this.scopeId = scopeId;
97
+ this.from = from;
98
+ this.to = to;
99
+ this.attempts = attempts;
100
+ }
101
+ }
73
102
  /** The body a route answers an `ExportBreakRefused` with: the sentence and the listing. */
74
103
  function exportBreakBody(e) {
75
104
  return { error: e.message, ...(e instanceof ExportBreakRefused ? { exportBreaks: { affected: e.breaks } } : {}) };
@@ -253,6 +282,9 @@ const rebindScopeVerticalBody = z.object({
253
282
  // #1756: an adopt moves the scope onto what its vertical serves, and is refused when that breaks
254
283
  // an app in the tenant. The body is optional, as it always was.
255
284
  const adoptServingBody = z.object({ acknowledge: bindAcknowledgement.optional() });
285
+ // The BULK form starts a run over every still-legacy scope of a vertical, so it reads its body
286
+ // strictly (`readJsonBody`); the single-scope form above keeps the lenient parse it always had.
287
+ const adoptVerticalServingBody = adoptServingBody.strict();
256
288
  // A snapshot request (preview-and-snapshots.md §3/§9). `expiresAt` opts into the GC
257
289
  // sweep; absent = pinned until deliberately deleted. `kind` defaults to 'archive'.
258
290
  const snapshotScopeBody = z.object({
@@ -265,9 +297,43 @@ const snapshotScopeBody = z.object({
265
297
  // false — the explicit "I accept an unrecoverable wipe"
266
298
  // undefined — back up if a store is configured, proceed without one if not, which is
267
299
  // what keeps every pre-#493 caller (and self-host) working unchanged
268
- const reapScopeBody = z.object({
300
+ const reapScopeBody = z
301
+ .object({
269
302
  backup: z.boolean().optional(),
270
- });
303
+ })
304
+ .strict();
305
+ /**
306
+ * Read the JSON body of a route whose default is an ACT (a reap, a bulk adoption, a fleet
307
+ * repair), under one grammar. The old `await c.req.json().catch(() => ({}))` turned a body
308
+ * that did not parse into the defaults, so a body of `{"backup": true` (cut short on the wire)
309
+ * ran the action it was cut short of describing.
310
+ *
311
+ * **Only a ZERO-LENGTH body takes the defaults.** Any other body must hold a JSON value, or it
312
+ * is a 400. That is deliberately not "an empty-looking body": a body of whitespace, or of a
313
+ * byte-order mark and nothing else, is a request that did not arrive intact, and reading it as
314
+ * "no body" would run the default act on it (a tenant reap of ` \t\n ` reaped with no backup).
315
+ * So the check is on the BYTES, before any decoding or trimming, and a leading BOM is stripped
316
+ * only so that JSON following it can parse. Do not "simplify" this to `text.trim() === ''`.
317
+ *
318
+ * The schema decides the rest. One that guards an act is `.strict()`, so a misspelt key is
319
+ * refused rather than dropped.
320
+ */
321
+ async function readJsonBody(c, schema) {
322
+ const bytes = await c.req.arrayBuffer();
323
+ let raw = {};
324
+ if (bytes.byteLength > 0) {
325
+ // `ignoreBOM: true` KEEPS the mark in the string; the default decoder would drop it and
326
+ // let a BOM-only body decode to '' and look empty.
327
+ const text = new TextDecoder('utf-8', { ignoreBOM: true }).decode(bytes).replace(/^\uFEFF/, '');
328
+ try {
329
+ raw = JSON.parse(text);
330
+ }
331
+ catch {
332
+ throw new ControlPlaneError(400, 'the body is not valid JSON');
333
+ }
334
+ }
335
+ return schema.parse(raw);
336
+ }
271
337
  // A directory-restore request (#40). `capturedAt` addresses the copy — there is one
272
338
  // directory, so that is its whole address. `overwrite` is the guard against the
273
339
  // dangerous case: replaying a restore onto a control plane that already recovered.
@@ -695,17 +761,22 @@ export function createControlPlaneApi(options) {
695
761
  // The CLI version advisory (#971), stamped on EVERY response — including the 401 the
696
762
  // auth middleware below answers with, since a CLI too old to authenticate is exactly
697
763
  // the one that needs to hear it. Registered first so nothing can return around it.
698
- // Nothing configured ⇒ no header touched, byte-identical to before this existed.
764
+ // Nothing configured ⇒ no advisory header.
765
+ //
766
+ // Beside it, the capability half of the same handshake (#1902): what this plane provides
767
+ // that a CLI may leave out on its strength. Stamped unconfigured, because it is a fact about
768
+ // this code rather than about a deployment — the uploader this package ships supplies the
769
+ // scope sweeper. The CLI asks before omitting one and refuses on silence, which is what an
770
+ // older plane answers.
699
771
  const advisory = options.cliAdvisory;
700
- if (advisory?.minVersion || advisory?.latestVersion) {
701
- app.use('*', async (c, next) => {
702
- await next();
703
- if (advisory.minVersion)
704
- c.header(CLI_MIN_VERSION_HEADER, advisory.minVersion);
705
- if (advisory.latestVersion)
706
- c.header(CLI_LATEST_VERSION_HEADER, advisory.latestVersion);
707
- });
708
- }
772
+ app.use('*', async (c, next) => {
773
+ await next();
774
+ c.header(PLATFORM_FEATURES_HEADER, PLATFORM_FEATURE_SCOPE_SWEEPER);
775
+ if (advisory?.minVersion)
776
+ c.header(CLI_MIN_VERSION_HEADER, advisory.minVersion);
777
+ if (advisory?.latestVersion)
778
+ c.header(CLI_LATEST_VERSION_HEADER, advisory.latestVersion);
779
+ });
709
780
  // Fail closed, before any route runs: no principal, no reach.
710
781
  //
711
782
  // The TENANT-SCOPED credential is read FIRST (#977), ahead of staff. Not for
@@ -962,6 +1033,9 @@ export function createControlPlaneApi(options) {
962
1033
  }
963
1034
  if (err instanceof ExportBreakRefused)
964
1035
  return c.json(exportBreakBody(err), 409);
1036
+ if (err instanceof CarrySourceChanged) {
1037
+ return c.json({ error: err.message, carrySourceChanged: { scopeId: err.scopeId, from: err.from, to: err.to, attempts: err.attempts } }, 409);
1038
+ }
965
1039
  const { status, body } = mapError(err);
966
1040
  // A 5xx is the PLATFORM failing — an unmapped throw, or a downstream vertical's
967
1041
  // own 5xx passing through (a DO storage fault during a preview restore is the
@@ -1055,7 +1129,7 @@ export function createControlPlaneApi(options) {
1055
1129
  app.post('/tenants/:tenantId/reap', async (c) => {
1056
1130
  const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
1057
1131
  const actor = c.get('actor');
1058
- const { backup: wantsBackup } = reapScopeBody.parse(await c.req.json().catch(() => ({})));
1132
+ const { backup: wantsBackup } = await readJsonBody(c, reapScopeBody);
1059
1133
  const tenant = await c.var.admin.getTenant(actor, tenantId);
1060
1134
  if (!tenant)
1061
1135
  return c.json({ error: `unknown tenant: ${tenantId}` }, 404);
@@ -1843,12 +1917,23 @@ export function createControlPlaneApi(options) {
1843
1917
  */
1844
1918
  const restoreCarryingSwitches = async (actor, dest, tenantId, scopeId, tables,
1845
1919
  /** #1869: the scope `tables` were captured from — `scopeId` itself unless the copy moves —
1846
- * and `exact` when the platform exported them itself rather than a caller supplying them. */
1920
+ * and `exact` when the platform exported them itself rather than a caller supplying them.
1921
+ * #1722: `loadStamp`, the stamp a carry leaves on the copy it lands, and `expect`, the marker
1922
+ * the carry read from `dest`, so a store that moved since is never overwritten. */
1847
1923
  source) => {
1924
+ // #2005: every carry, reuse, adopt and governed restore lands here. When the directory says
1925
+ // the destination is not primary, the vertical marks it a copy in its own storage — a carry
1926
+ // of a copy made before every copy carried the marker brings none, and a CP-less host reads
1927
+ // the marker for primacy. Idempotent there; an install is never marked.
1928
+ const rec = await host.admin.getScopeRecord(actor, tenantId, scopeId);
1929
+ const markCopy = rec ? copyLineageOf(rec) : undefined;
1848
1930
  return retryTransient(async () => dest.restoreScope(tenantId, scopeId, tables, {
1849
1931
  ...(await switchCarryFor(host.admin, actor, { tenantId, scopeId })),
1850
1932
  sourceScopeId: source.scopeId,
1851
1933
  exact: source.exact,
1934
+ loadStamp: source.loadStamp,
1935
+ expect: source.expect,
1936
+ ...(markCopy ? { markCopy } : {}),
1852
1937
  }));
1853
1938
  };
1854
1939
  const adoptScopeOntoServing = async (c, tenantId, scopeId, opts = {}) => {
@@ -2042,6 +2127,16 @@ export function createControlPlaneApi(options) {
2042
2127
  await c.var.admin.reassertSystemSwitches(actor, { tenantId, scopeId }, { appliedInUnit: restored.switchedOff });
2043
2128
  return { servingRef: serving.ref, versionId: serving.versionId, tables: restored.tables };
2044
2129
  };
2130
+ /** The script a scope's requests reach: its serving pin, else its bound version's own script. */
2131
+ const routeOf = async (c, scope) => {
2132
+ if (scope.servingRef)
2133
+ return scope.servingRef;
2134
+ if (!scope.vertical || !scope.verticalVersionId)
2135
+ return null;
2136
+ return (await c.var.admin.getVersion(c.get('actor'), scope.verticalVersionId, scope.vertical))?.deploymentRef ?? null;
2137
+ };
2138
+ /** How many exports a carry tries before its source holds still long enough to bind (#1722 r12). */
2139
+ const CARRY_EXPORT_ATTEMPTS = 3;
2045
2140
  /**
2046
2141
  * Carry a scope's data onto the script a version bind is about to route it to (#1710).
2047
2142
  * The caller binds only after this returns.
@@ -2069,11 +2164,23 @@ export function createControlPlaneApi(options) {
2069
2164
  * over any partial copy. A retry after a bind that landed finds the same script on both
2070
2165
  * ends and carries nothing.
2071
2166
  *
2167
+ * #1722: the export is checked before anything is restored. The scope is read again, and a
2168
+ * binding or pin that moved since the caller read it is refused (412): another carry bound
2169
+ * away from this script, which may already have wiped it, and an export from a wiped DO comes
2170
+ * back as an empty store. A dump that carries the `carried_away` tombstone is refused the same
2171
+ * way. The caller binds with `bindAfterCarry`, which wipes the source copy once the bind
2172
+ * lands, so neither check can pass on an export from a copy that is about to go.
2173
+ *
2174
+ * The restore is conditional too. Before the check above, the carry reads the destination's
2175
+ * `LoadMarker` (load stamp and write revision), and the restore is refused inside its own
2176
+ * transaction if the store moved since. So a carry held before its restore cannot overwrite
2177
+ * a store that another carry into the same script has loaded, or that went live under that
2178
+ * carry's bind and took a write: a CI retry of the same version. A deployment built before
2179
+ * #1722 cannot read a marker, and there the binding re-read is all that guards the restore.
2180
+ * Its window: a bind that lands between that re-read and the restore, followed by a write.
2181
+ *
2072
2182
  * What it does not do:
2073
2183
  * - Carry writes that land on the source between the export and the bind. Those are lost.
2074
- * - Delete the source copy. That copy is unreachable once the caller binds, and it stays
2075
- * in the old script, because nothing reaps it yet (#1722). Deleting it here would race
2076
- * a concurrent push that is still exporting from it.
2077
2184
  * - Cross lineages. A version of another vertical reads as absent here, and moving a scope
2078
2185
  * between lineages is `rebindScopeOntoVertical`'s job, behind its migration gate.
2079
2186
  */
@@ -2095,10 +2202,7 @@ export function createControlPlaneApi(options) {
2095
2202
  if (!resolveVersion || (scope.servingRef && !opts.dropServingRef))
2096
2203
  return null;
2097
2204
  const to = incoming.deploymentRef ?? null;
2098
- const bound = scope.verticalVersionId
2099
- ? await c.var.admin.getVersion(actor, scope.verticalVersionId, scope.vertical)
2100
- : undefined;
2101
- const from = scope.servingRef ?? bound?.deploymentRef ?? null;
2205
+ const from = await routeOf(c, scope);
2102
2206
  if (!to || !from || from === to)
2103
2207
  return null;
2104
2208
  // Resolve each end at exactly the script named above. No fallback to the prod channel:
@@ -2116,13 +2220,343 @@ export function createControlPlaneApi(options) {
2116
2220
  throw new ControlPlaneError(502, `cannot move scope ${scope.id} from '${from}' to '${to}': ${missing} could not be reached, ` +
2117
2221
  `so the version was not bound (#1710)`);
2118
2222
  }
2119
- const dump = await retryTransient(() => source.exportScope(scope.id));
2120
- // #1742: the recorded OFF positions ride the restore, as on adopt and rebind.
2223
+ const { tables: dump, loadStamp: sourceStamp, revision: sourceRevision } = await retryTransient(() => source.exportScopeStamped(scope.id));
2224
+ // What the destination holds now, read before the binding is checked again. The restore
2225
+ // sends it back and is refused if the store moved since: another carry into the same script
2226
+ // loaded it, or it went live under another carry's bind and took a write. That is a CI retry
2227
+ // of the same version, held before its restore while the first run bound and served.
2228
+ const destMarker = await retryTransient(() => dest.loadMarker(scope.id));
2229
+ const now = await c.var.admin.getScopeRecord(actor, scope.tenantId, scope.id);
2230
+ if (!now ||
2231
+ (now.verticalVersionId ?? null) !== (scope.verticalVersionId ?? null) ||
2232
+ (now.servingRef ?? null) !== (scope.servingRef ?? null)) {
2233
+ throw new ControlPlaneError(412, `scope ${scope.id} was re-pointed while its data was being copied out of '${from}', so nothing was ` +
2234
+ `moved or bound (#1722). Reload the scope and retry.`);
2235
+ }
2236
+ if (dumpMetaValue(dump, CARRIED_AWAY_KEY) !== null) {
2237
+ throw new ControlPlaneError(412, `the copy of scope ${scope.id} in '${from}' was already carried to another script and wiped, so nothing ` +
2238
+ `was moved or bound (#1722). Reload the scope and retry.`);
2239
+ }
2240
+ // #1742: the recorded OFF positions ride the restore, as on adopt and rebind. #1722: so does
2241
+ // a stamp of this carry's own, which a refused bind's wipe of the copy expects.
2242
+ const restoredStamp = ulid();
2121
2243
  const restored = await restoreCarryingSwitches(actor, dest, scope.tenantId, scope.id, dump, {
2122
2244
  scopeId: scope.id,
2123
2245
  exact: true,
2246
+ loadStamp: restoredStamp,
2247
+ ...(destMarker === 'unfenced' ? {} : { expect: destMarker }),
2248
+ });
2249
+ return {
2250
+ from,
2251
+ to,
2252
+ tables: restored.tables,
2253
+ switchedOff: restored.switchedOff,
2254
+ source,
2255
+ dest,
2256
+ sourceStamp,
2257
+ sourceRevision,
2258
+ restoredStamp,
2259
+ };
2260
+ };
2261
+ /**
2262
+ * Wipe one copy of a scope that no route reaches (#1722), without reaping it. Fenced where the
2263
+ * deployment can fence it: the wipe runs only if nothing was loaded into that DO since
2264
+ * `expectLoadStamp`, compared inside the wipe's own transaction, and `false` means something
2265
+ * was (a rollback restored there), so the copy stays. A deployment built before the fence
2266
+ * answers `'unfenced'`, and the wipe is then an unconditional load of the tombstone through
2267
+ * the restore verb every deployment has. The caller re-reads the route first; that re-read is
2268
+ * all that guards an unfenced wipe.
2269
+ */
2270
+ const wipeCarriedCopy = async (holder, scope, expect, carriedTo,
2271
+ /** The scope was read routing elsewhere: a copy changed in any way since is kept (#1722 r8). */
2272
+ protectIfChanged = false) => {
2273
+ const at = new Date().toISOString();
2274
+ // #2005 (Codex #2008 r10): the directory's word that the scope is not primary goes with the
2275
+ // wipe, on both paths, so a copy made before the marker leaves it marked — tombstoned or kept.
2276
+ const markCopy = copyLineageOf(scope);
2277
+ const fenced = await retryTransient(() => holder.wipeCarriedCopy({
2278
+ scopeId: scope.id,
2279
+ expectLoadStamp: expect.loadStamp,
2280
+ ...(expect.revision !== undefined ? { expectRevision: expect.revision } : {}),
2281
+ ...(protectIfChanged ? { protectIfChanged: true } : {}),
2282
+ carriedTo,
2283
+ at,
2284
+ ...(markCopy ? { markCopy } : {}),
2285
+ }));
2286
+ if (fenced !== 'unfenced')
2287
+ return fenced.wiped;
2288
+ await retryTransient(() => holder.restoreScope(scope.tenantId, scope.id, carriedAwayDump({ to: carriedTo, at }), {
2289
+ sourceScopeId: scope.id,
2290
+ exact: true,
2291
+ // #2005: as every restore onto a non-primary scope, so the wiped copy stays a copy.
2292
+ ...(markCopy ? { markCopy } : {}),
2293
+ }));
2294
+ return true;
2295
+ };
2296
+ /**
2297
+ * Where the scope routes now, read again after a carry's bind (null for a scope gone from the
2298
+ * directory). Bound to the carry's own version and unpinned, that is the carry's destination,
2299
+ * which needs no version read.
2300
+ */
2301
+ const currentRoute = async (c, scope, versionId, carried) => {
2302
+ const now = await c.var.admin.getScopeRecord(c.get('actor'), scope.tenantId, scope.id);
2303
+ if (!now)
2304
+ return null;
2305
+ if (!now.servingRef && now.verticalVersionId === versionId)
2306
+ return carried.to;
2307
+ return routeOf(c, now);
2308
+ };
2309
+ /** Whether the scope's store in `holder` is a wiped copy: it holds the `carried_away` tombstone. */
2310
+ const isCarriedAway = async (holder, scope) => {
2311
+ const page = await holder.readScopeTable(scope.id, { table: '_substrat_meta', limit: 100, offset: 0 });
2312
+ return metaValueIn(page.rows, page.columns, CARRIED_AWAY_KEY) !== null;
2313
+ };
2314
+ const recordCarryCleanup = (c, scope, versionId, stage, e) => {
2315
+ const { status } = mapError(e);
2316
+ recordFailure({
2317
+ actor: c.get('actor'),
2318
+ operation: 'scope.carry',
2319
+ stage,
2320
+ tenantId: scope.tenantId,
2321
+ scopeId: scope.id,
2322
+ vertical: scope.vertical ?? null,
2323
+ version: versionId,
2324
+ status,
2325
+ message: e instanceof Error ? e.message : String(e),
2326
+ }, e);
2327
+ };
2328
+ /**
2329
+ * Bind a scope onto `versionId` after `carryOntoVersion` (#1722), and settle the copies. The
2330
+ * one tail every carry caller runs: a preview push, `scope bind`, and the serving-pin repair.
2331
+ *
2332
+ * 1. The bind is compare-and-set on the binding the caller read, which is the version the carry
2333
+ * exported from. Two pushes that both read v1 cannot both bind: the second is refused (412)
2334
+ * and never routes the preview to what it carried, which may be an export of a copy the
2335
+ * first one had already wiped. On any failed bind, the copy this carry restored is wiped, unless
2336
+ * the scope now routes to that same script: a CI retry of the same version restores into
2337
+ * the winner's live store, and wiping it there would lose everything.
2338
+ * 2. The pin is cleared (`clearServingRef`) only after the bind lands, so a failure in between
2339
+ * leaves the route where the data still is, and a retry carries again.
2340
+ * 3. The recorded OFF positions are re-asserted where the scope now routes (#1674).
2341
+ * 4. The copy left in the old script is wiped (`settleCarriedSource`). A failure there is
2342
+ * recorded and never fails the bind, which has landed: the copy then stays, as before #1722.
2343
+ */
2344
+ const bindAfterCarry = async (c, scope, versionId, carried, opts = {}) => {
2345
+ const actor = c.get('actor');
2346
+ // 0. The source, read again right before the bind (Codex #2008 r12). A store that changed since
2347
+ // the export (a write, a load, or a staff clear of its copy marker) holds something the
2348
+ // destination does not, so binding now would serve the stale export. The carry is run
2349
+ // again from a fresh export, a bounded number of times, and then refused with a typed 409.
2350
+ // What changes after this read is the source wipe's to fence (`keepOrWipeSource`).
2351
+ for (let attempt = 1; carried; attempt++) {
2352
+ const stale = carried;
2353
+ let now;
2354
+ try {
2355
+ now = await retryTransient(() => stale.source.loadMarker(scope.id));
2356
+ }
2357
+ catch (e) {
2358
+ await dropUnboundCopy(c, scope, versionId, stale); // no bind follows
2359
+ throw e;
2360
+ }
2361
+ if (now === 'unfenced' || (now.loadStamp === stale.sourceStamp && now.revision === stale.sourceRevision))
2362
+ break;
2363
+ let next = null;
2364
+ if (attempt < CARRY_EXPORT_ATTEMPTS && opts.recarry) {
2365
+ try {
2366
+ next = await opts.recarry();
2367
+ }
2368
+ catch (e) {
2369
+ await dropUnboundCopy(c, scope, versionId, stale);
2370
+ throw e;
2371
+ }
2372
+ }
2373
+ if (!next) {
2374
+ await dropUnboundCopy(c, scope, versionId, stale);
2375
+ throw new CarrySourceChanged(scope.id, stale.from, stale.to, attempt);
2376
+ }
2377
+ carried = next;
2378
+ }
2379
+ try {
2380
+ await c.var.admin
2381
+ .bindScopeVersion(actor, scope.tenantId, scope.id, versionId, {
2382
+ acknowledge: opts.acknowledge,
2383
+ ...(opts.snapshot ? { snapshot: true } : {}),
2384
+ expectedVersionId: scope.verticalVersionId ?? null,
2385
+ })
2386
+ .catch(relayHostRefusal);
2387
+ }
2388
+ catch (e) {
2389
+ if (carried)
2390
+ await dropUnboundCopy(c, scope, versionId, carried);
2391
+ throw e;
2392
+ }
2393
+ if (opts.clearServingRef && scope.servingRef) {
2394
+ await c.var.admin.setScopeServingRef(actor, scope.tenantId, scope.id, null).catch(relayHostRefusal);
2395
+ }
2396
+ try {
2397
+ await c.var.admin.reassertSystemSwitches(actor, { tenantId: scope.tenantId, scopeId: scope.id }, { appliedInUnit: carried?.switchedOff });
2398
+ }
2399
+ finally {
2400
+ // Even when the re-assert fails: the route has moved, and a retry finds the same script
2401
+ // on both ends, carries nothing, and so would never come back for this copy.
2402
+ if (carried)
2403
+ await settleCarriedSource(c, scope, versionId, carried);
2404
+ }
2405
+ };
2406
+ /**
2407
+ * The copy a carry restored for a bind that did not land, wiped unless the scope routes to
2408
+ * that script now. Best effort: it never throws, and a failure is recorded.
2409
+ */
2410
+ const dropUnboundCopy = async (c, scope, versionId, carried) => {
2411
+ try {
2412
+ const route = await currentRoute(c, scope, versionId, carried);
2413
+ if (route === carried.to)
2414
+ return;
2415
+ await wipeCarriedCopy(carried.dest, scope, { loadStamp: carried.restoredStamp }, route ?? carried.from);
2416
+ }
2417
+ catch (e) {
2418
+ recordCarryCleanup(c, scope, versionId, 'unbound-copy', e);
2419
+ }
2420
+ };
2421
+ /**
2422
+ * Clear the kept-copy marker of a store the scope routes to (#1722, Codex #2008 r8). A carry
2423
+ * keeps a changed source when it reads the route elsewhere; a rollback binding it a moment later
2424
+ * makes that store live, and a live store holds nothing divergent. Fenced on the revision read
2425
+ * now; re-reads the route first, so it never releases a copy the scope has left.
2426
+ */
2427
+ const releaseIfLive = async (c, scope, versionId, carried, holder, script) => {
2428
+ if (!(await holder.keptCopy(scope.id)))
2429
+ return;
2430
+ if ((await currentRoute(c, scope, versionId, carried)) !== script)
2431
+ return;
2432
+ const marker = await holder.loadMarker(scope.id);
2433
+ if (marker === 'unfenced')
2434
+ return;
2435
+ const markCopy = copyLineageOf(scope);
2436
+ await holder.releaseKeptCopy({
2437
+ scopeId: scope.id,
2438
+ revision: marker.revision,
2439
+ loadStamp: marker.loadStamp, // Codex #2008 r13: the store read, not only its revision
2440
+ ...(markCopy ? { markCopy } : {}),
2441
+ });
2442
+ };
2443
+ /**
2444
+ * A source kept ONLY because staff cleared its copy marker after the carry's export (Codex #2008
2445
+ * r12; the store records that itself, `KeptCopy.clearedOnly`). Its data is what the carry
2446
+ * copied; the fresher decision is the clear, so it is brought to the store the scope runs on:
2447
+ * the destination's marker is cleared, and only then is the source discarded. True when both
2448
+ * landed. Anything else (a write since, a move, a refusal) leaves the source kept, and the caller
2449
+ * records it as any kept copy.
2450
+ *
2451
+ * Every step is fenced on THIS carry, not on whatever the stores hold now (Codex #2008 r13). The
2452
+ * destination must still hold this carry's load (its `restoredStamp`): a governed restore that
2453
+ * replaced it since brings a copy marker that is genuine, and clearing it would let another
2454
+ * scope's copy run its executors. That stamp and the revision read are compared inside the clear's
2455
+ * own transaction, so nothing can land between the check and the clear. The binding is read again
2456
+ * right before the discard, and the discard is fenced on the source's stamp and kept revision.
2457
+ */
2458
+ const bringClearForward = async (c, scope, versionId, carried) => {
2459
+ if (!isPrimaryScope(scope))
2460
+ return false; // only staff clear, and only a primary's marker
2461
+ const kept = await carried.source.keptCopy(scope.id);
2462
+ if (!kept?.clearedOnly)
2463
+ return false;
2464
+ const destMarker = await carried.dest.loadMarker(scope.id);
2465
+ if (destMarker === 'unfenced' || destMarker.loadStamp !== carried.restoredStamp)
2466
+ return false;
2467
+ if ((await currentRoute(c, scope, versionId, carried)) !== carried.to)
2468
+ return false;
2469
+ try {
2470
+ await carried.dest.clearCopyMark(scope.id, { kind: scope.kind, forkedFrom: scope.forkedFrom }, {
2471
+ expect: { loadStamp: carried.restoredStamp, revision: destMarker.revision },
2472
+ });
2473
+ }
2474
+ catch (e) {
2475
+ if (e instanceof ControlPlaneError && (e.status === 412 || e.status === 409))
2476
+ return false;
2477
+ throw e;
2478
+ }
2479
+ if ((await currentRoute(c, scope, versionId, carried)) !== carried.to)
2480
+ return false;
2481
+ const discarded = await carried.source.discardKeptCopy({
2482
+ scopeId: scope.id,
2483
+ revision: kept.clearedOnly.revision,
2484
+ // Clear-only means nothing was loaded there since the export: the stamp the carry read.
2485
+ loadStamp: carried.sourceStamp,
2486
+ carriedTo: carried.to,
2487
+ at: new Date().toISOString(),
2124
2488
  });
2125
- return { from, to, tables: restored.tables, switchedOff: restored.switchedOff };
2489
+ return 'discarded' in discarded;
2490
+ };
2491
+ /**
2492
+ * The source wipe, fenced on what the export read. Refused means the copy changed after the
2493
+ * export: a write from a request still routed there before the bind, which the carry never
2494
+ * copied. The copy is then kept, holding that write, and recorded so it can be recovered.
2495
+ */
2496
+ const keepOrWipeSource = async (c, scope, versionId, carried, expect) => {
2497
+ // The caller read the route elsewhere, so a copy that changed in any way since the export (a
2498
+ // load as much as a write: a staff restore-forward into it racing this carry, Codex #2008 r8)
2499
+ // is kept in the store rather than left an unmarked orphan.
2500
+ if (await wipeCarriedCopy(carried.source, scope, expect, carried.to, true))
2501
+ return;
2502
+ // A rollback that restored into it since made it the live store again: it is no kept copy, and
2503
+ // a marker this refusal set while the rollback had not yet bound comes off.
2504
+ const route = await currentRoute(c, scope, versionId, carried);
2505
+ if (route === carried.from) {
2506
+ await releaseIfLive(c, scope, versionId, carried, carried.source, carried.from);
2507
+ return;
2508
+ }
2509
+ if (route === carried.to && (await bringClearForward(c, scope, versionId, carried)))
2510
+ return;
2511
+ recordCarryCleanup(c, scope, versionId, 'source-copy-kept', new ControlPlaneError(409, `scope ${scope.id}'s copy in '${carried.from}' changed after it was carried to '${carried.to}', so it was ` +
2512
+ `kept: it holds what reached it in between, which the carry did not copy (#1722)`));
2513
+ };
2514
+ /**
2515
+ * Wipe the copy a landed carry left in its source script (#1722), once nothing routes there.
2516
+ *
2517
+ * The route is read again first. A scope that routes to the source again was bound back to it
2518
+ * (a rollback that has landed), and its copy stays. A scope that routes to this carry's
2519
+ * destination has its store checked for the `carried_away` tombstone first: an unfenced wipe
2520
+ * from an earlier carry out of this script can overtake this carry's restore when this carry
2521
+ * IS the rollback, and then the source, which nothing has wiped yet, is carried again before
2522
+ * it goes. The fenced wipe in a deployment that has it makes the overtaking impossible.
2523
+ *
2524
+ * Left open, for a script built before the fence only: an unfenced wipe whose request reaches
2525
+ * the old script after a rollback's bind and this check, both, still destroys the rollback's
2526
+ * restore. Narrow, and recorded here rather than claimed closed.
2527
+ */
2528
+ const settleCarriedSource = async (c, scope, versionId, carried) => {
2529
+ const actor = c.get('actor');
2530
+ try {
2531
+ const route = await currentRoute(c, scope, versionId, carried);
2532
+ if (route === carried.from)
2533
+ return;
2534
+ // The store this carry landed in is live; a kept marker another carry set while racing it
2535
+ // does not belong on it (#1722 r8).
2536
+ if (route === carried.to)
2537
+ await releaseIfLive(c, scope, versionId, carried, carried.dest, carried.to);
2538
+ if (route === carried.to && (await isCarriedAway(carried.dest, scope))) {
2539
+ const { tables: dump, loadStamp: stamp, revision } = await retryTransient(() => carried.source.exportScopeStamped(scope.id));
2540
+ if (dumpMetaValue(dump, CARRIED_AWAY_KEY) !== null) {
2541
+ throw new ControlPlaneError(500, `both copies of scope ${scope.id} hold the carried_away tombstone; nothing was wiped`);
2542
+ }
2543
+ await restoreCarryingSwitches(actor, carried.dest, scope.tenantId, scope.id, dump, {
2544
+ scopeId: scope.id,
2545
+ exact: true,
2546
+ loadStamp: ulid(),
2547
+ });
2548
+ if (await isCarriedAway(carried.dest, scope)) {
2549
+ throw new ControlPlaneError(500, `scope ${scope.id}'s store in '${carried.to}' was wiped again; its source copy stays`);
2550
+ }
2551
+ // Re-carried, so the fence expects what the re-export read.
2552
+ await keepOrWipeSource(c, scope, versionId, carried, { loadStamp: stamp, revision });
2553
+ return;
2554
+ }
2555
+ await keepOrWipeSource(c, scope, versionId, carried, { loadStamp: carried.sourceStamp, revision: carried.sourceRevision });
2556
+ }
2557
+ catch (e) {
2558
+ recordCarryCleanup(c, scope, versionId, 'source-copy', e);
2559
+ }
2126
2560
  };
2127
2561
  app.get('/tenants/:tenantId/scopes/:scopeId/tables', async (c) => {
2128
2562
  const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
@@ -2733,7 +3167,7 @@ export function createControlPlaneApi(options) {
2733
3167
  return c.json({ error: `scope ${scopeId} has no hostname bound — a claim link needs an address to open on` }, 409);
2734
3168
  }
2735
3169
  try {
2736
- return c.json(await vertical.mintOwnerClaim({ tenantId, scopeId, origin: `https://${host.hostname}` }), 201);
3170
+ return c.json(await vertical.mintOwnerClaim({ tenantId, scopeId, origin: `https://${host.hostname}`, actor }), 201);
2737
3171
  }
2738
3172
  catch (e) {
2739
3173
  if (e instanceof ControlPlaneError)
@@ -2847,10 +3281,49 @@ export function createControlPlaneApi(options) {
2847
3281
  archive: (admin, a, t, s) => admin.archiveScope(a, t, s),
2848
3282
  unarchive: (admin, a, t, s) => admin.unarchiveScope(a, t, s),
2849
3283
  };
3284
+ /** #2005: the transitions that bring a scope (back) to life — each marks a copy first. */
3285
+ const REACTIVATIONS = new Set(['activate', 'unsuspend', 'unarchive']);
2850
3286
  for (const [action, run] of Object.entries(transitions)) {
2851
3287
  app.post(`/tenants/:tenantId/scopes/:scopeId/${action}`, async (c) => {
2852
3288
  const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
2853
3289
  const scopeId = scopeIdSchema.parse(c.req.param('scopeId'));
3290
+ // #2005: a copy made before the copy marker existed holds none, and a CP-less host that
3291
+ // finds none runs its effects. So a copy is marked BEFORE it comes back to life, and a
3292
+ // hosted copy whose marker cannot be written is refused here rather than reactivated.
3293
+ if (REACTIVATIONS.has(action)) {
3294
+ const rec = await c.var.admin.getScopeRecord(c.get('actor'), tenantId, scopeId);
3295
+ if (rec && !isPrimaryScope(rec)) {
3296
+ // ANY failed marker write keeps the scope parked: a 503 that says what to do, and an
3297
+ // ops-failure row, whether the step reported it (no deployment resolves) or the vertical
3298
+ // refused (an older one has no /internal/mark-copy and answers 404). Passing a 404 or
3299
+ // a vertical's error through would read as "no such scope" and leave no trace.
3300
+ let failure;
3301
+ try {
3302
+ const step = await markScopeIfCopy(c, rec);
3303
+ if ('fail' in step)
3304
+ failure = { error: step.fail.error, cause: NO_CAUSE };
3305
+ }
3306
+ catch (e) {
3307
+ failure = { error: e instanceof Error ? e.message : String(e), cause: e };
3308
+ }
3309
+ if (failure) {
3310
+ const error = `${action} refused: this copy could not be marked as one (${failure.error}); ` +
3311
+ 'redeploy the vertical so it can mark it, then retry — the scope stays as it was';
3312
+ recordFailure({
3313
+ actor: c.get('actor'),
3314
+ operation: `scope.${action}`,
3315
+ stage: 'mark-copy',
3316
+ tenantId,
3317
+ scopeId,
3318
+ vertical: rec.vertical,
3319
+ version: rec.verticalVersionId,
3320
+ status: 503,
3321
+ message: error,
3322
+ }, failure.cause);
3323
+ return c.json({ error }, 503);
3324
+ }
3325
+ }
3326
+ }
2854
3327
  await run(c.var.admin, c.get('actor'), tenantId, scopeId);
2855
3328
  return c.json(await c.var.admin.getScopeRecord(c.get('actor'), tenantId, scopeId));
2856
3329
  });
@@ -2881,7 +3354,7 @@ export function createControlPlaneApi(options) {
2881
3354
  forkedAt: new Date().toISOString(),
2882
3355
  expiresAt: opts.expiresAt,
2883
3356
  });
2884
- await retryTransient(() => vertical.snapshotScope({ sourceScopeId: scope.id, newScopeId: snapId }));
3357
+ await retryTransient(() => vertical.snapshotScope({ sourceScopeId: scope.id, newScopeId: snapId, tenantId }));
2885
3358
  await c.var.admin.activateScope(actor, tenantId, snapId);
2886
3359
  // Bound to the SOURCE's current version: source and fork share a deployment, so
2887
3360
  // the fork resolves to the DO namespace its bytes actually live in.
@@ -3160,6 +3633,235 @@ export function createControlPlaneApi(options) {
3160
3633
  await c.var.admin.restoreDirectory(actor, dump);
3161
3634
  return c.json({ capturedAt: dump.capturedAt, tables: dump.tables.length });
3162
3635
  });
3636
+ // -- one-time repair of previews that kept a production serving pin (#1724) --------------
3637
+ // A preview adopted onto its vertical's serving script before #1731 still routes by
3638
+ // `COALESCE(servingRef, deploymentRef)`, so it serves production code while its binding
3639
+ // names the version it was pushed from. #1962 heals one on its next `scope bind`; this is
3640
+ // the pass over the previews nobody binds again. It is the SAME act a bind runs, with the
3641
+ // version the preview is already bound to as its target: carry the data off the serving
3642
+ // script into that version's own script, bind it, then clear the pin. A pin that fails to
3643
+ // clear leaves the route where the data still is, and the next pass carries again.
3644
+ //
3645
+ // Staff only, by default-deny: the path is on neither `BUILDER_ROUTES` nor
3646
+ // `TENANT_ROUTES`, so a builder's push token and a tenant credential both 403 before the
3647
+ // handler runs. (A fleet-wide walk is the wrong shape for either, even a tenant's own.)
3648
+ //
3649
+ // Paged and resumable like every fleet walk here: a pass reads one page of ACTIVE
3650
+ // directory rows after `cursor` and repairs at most `limit` previews from it, answering
3651
+ // `nextCursor` until the fleet is exhausted. Bounded by `limit` because each repair is a
3652
+ // data move between two Durable Objects. A scope that fails is reported and the pass
3653
+ // moves on, so one bad preview cannot hold the rest; it is still pinned, so the next walk
3654
+ // from the start visits it again. A healed preview is no longer a candidate, which is
3655
+ // what makes a re-run a no-op. `dryRun` lists the candidates and touches nothing.
3656
+ const REPAIR_SCAN_PAGE = 500;
3657
+ const repairServingPinsBody = z
3658
+ .object({
3659
+ cursor: scopeIdSchema.optional(),
3660
+ limit: z.number().int().min(1).max(100).default(25),
3661
+ dryRun: z.boolean().default(false),
3662
+ })
3663
+ .strict();
3664
+ const repairPreviewServingPin = async (c, scope) => {
3665
+ const actor = c.get('actor');
3666
+ const versionId = scope.verticalVersionId;
3667
+ if (!scope.vertical || !versionId)
3668
+ return { skipped: 'the preview is bound to no version' };
3669
+ // Without its own script the version has nowhere to receive the data, and clearing the pin
3670
+ // would strand it on the serving script.
3671
+ const bound = await c.var.admin.getVersion(actor, versionId, scope.vertical);
3672
+ if (!bound?.deploymentRef)
3673
+ return { skipped: `version ${versionId} has no script of its own` };
3674
+ // The carry and the tail a legacy preview's `scope bind` runs too. The bind expects the
3675
+ // version this pass read, so a push that re-pointed the preview since it was listed wins.
3676
+ const recarry = () => carryOntoVersion(c, scope, versionId, { dropServingRef: true });
3677
+ const carried = await recarry();
3678
+ await bindAfterCarry(c, scope, versionId, carried, { clearServingRef: true, recarry });
3679
+ return { repaired: { from: carried?.from ?? scope.servingRef, to: bound.deploymentRef, tables: carried?.tables ?? 0 } };
3680
+ };
3681
+ app.post('/previews/repair-serving-pins', async (c) => {
3682
+ if (!options.resolveVerticalVersion || !options.resolveVerticalRef) {
3683
+ return c.json({ error: 'repairing serving pins needs dispatch resolution for both ends' }, 501);
3684
+ }
3685
+ const { cursor, limit, dryRun } = await readJsonBody(c, repairServingPinsBody);
3686
+ const actor = c.get('actor');
3687
+ const page = await c.var.admin.listScopes(actor, { status: ['active'], cursor, limit: REPAIR_SCAN_PAGE });
3688
+ const repaired = [];
3689
+ const candidates = [];
3690
+ const skipped = [];
3691
+ const failed = [];
3692
+ let last = cursor;
3693
+ let handled = 0;
3694
+ let stopped = false;
3695
+ for (const scope of page) {
3696
+ if (scope.kind !== 'preview' || !scope.servingRef) {
3697
+ last = scope.id;
3698
+ continue;
3699
+ }
3700
+ if (handled >= limit) {
3701
+ stopped = true;
3702
+ break;
3703
+ }
3704
+ handled += 1;
3705
+ last = scope.id;
3706
+ const at = { tenantId: scope.tenantId, scopeId: scope.id };
3707
+ if (dryRun) {
3708
+ candidates.push({ ...at, servingRef: scope.servingRef });
3709
+ continue;
3710
+ }
3711
+ try {
3712
+ const outcome = await repairPreviewServingPin(c, scope);
3713
+ if ('skipped' in outcome)
3714
+ skipped.push({ ...at, reason: outcome.skipped });
3715
+ else
3716
+ repaired.push({ ...at, ...outcome.repaired });
3717
+ }
3718
+ catch (e) {
3719
+ // The status the app's error boundary would have answered, so a host refusal (a push
3720
+ // re-pointed the preview since it was listed) reads as a 412 and not as a platform fault.
3721
+ const { status } = mapError(e);
3722
+ const message = e instanceof Error ? e.message : String(e);
3723
+ if (status >= 500 && status !== 501) {
3724
+ recordFailure({
3725
+ actor,
3726
+ operation: 'preview.repair-serving-pin',
3727
+ stage: 'repair',
3728
+ tenantId: scope.tenantId,
3729
+ scopeId: scope.id,
3730
+ vertical: scope.vertical,
3731
+ version: scope.verticalVersionId,
3732
+ status,
3733
+ message,
3734
+ }, e);
3735
+ }
3736
+ failed.push({ ...at, status, error: message });
3737
+ }
3738
+ }
3739
+ // Exhausted only when the page ran short: a full page may be followed by more rows.
3740
+ const nextCursor = stopped || page.length === REPAIR_SCAN_PAGE ? (last ?? null) : null;
3741
+ return c.json({ dryRun, repaired, candidates, skipped, failed, nextCursor });
3742
+ });
3743
+ const markScopeIfCopy = async (c, scope) => {
3744
+ const actor = c.get('actor');
3745
+ if (isPrimaryScope(scope))
3746
+ return { skip: 'primary: an install is never marked' };
3747
+ if (!scope.vertical)
3748
+ return { skip: 'bound to no vertical: no module code runs, so nothing reads the marker' };
3749
+ const vertical = await verticalForScope(c, scope);
3750
+ if (!vertical) {
3751
+ const hosted = scope.servingRef !== null && scope.servingRef !== undefined
3752
+ ? true
3753
+ : scope.verticalVersionId
3754
+ ? Boolean((await c.var.admin.getVersion(actor, scope.verticalVersionId, scope.vertical))?.deploymentRef)
3755
+ : false;
3756
+ return hosted
3757
+ ? { fail: { status: 503, error: 'no deployment resolves for this hosted scope, so its copy marker was not written' } }
3758
+ : { skip: 'co-located: the host that runs this scope reads the directory, so it needs no marker' };
3759
+ }
3760
+ const { marked } = await vertical.markCopy(scope.id, { kind: scope.kind, forkedFrom: scope.forkedFrom });
3761
+ await c.var.admin.recordCopyMark(actor, {
3762
+ action: 'mark',
3763
+ tenantId: scope.tenantId,
3764
+ scopeId: scope.id,
3765
+ outcome: marked ? 'marked' : 'already',
3766
+ });
3767
+ return { done: marked ? 'marked' : 'already' };
3768
+ };
3769
+ // -- one-time repair: mark every existing copy as one, in its own storage (#2005) --------
3770
+ // A CP-less hosted vertical has no directory, so it reads a scope's own copy-origin row to
3771
+ // hold a preview's or a fork's executors inert. Every copy carries that row since #2005, and
3772
+ // every carry onto a non-primary scope stamps it (`restoreCarryingSwitches`) — but a copy
3773
+ // made before either, never carried since, holds none and would read as an install there.
3774
+ // This is the pass over those: for each scope the directory says is not primary, ask the
3775
+ // vertical that holds it to stamp the marker (idempotent there), and log the outcome. Active,
3776
+ // suspended and archived scopes alike: a suspended or archived copy can be reactivated, and a
3777
+ // reactivation stamps too (`markScopeIfCopy` in the lifecycle routes), so neither path brings
3778
+ // an unmarked copy back to life. A hosted scope that does not resolve is FAILED, not skipped.
3779
+ //
3780
+ // Staff only, by default-deny, like the serving-pin repair: the path is on neither
3781
+ // `BUILDER_ROUTES` nor `TENANT_ROUTES`. Paged and resumable the same way — one page of
3782
+ // ACTIVE directory rows after `cursor`, at most `limit` scopes marked from it, `nextCursor`
3783
+ // until the fleet is exhausted. A scope that fails is reported and the pass moves on; it is
3784
+ // still unmarked, so the next walk visits it again, and a marked one answers `already`.
3785
+ // `dryRun` lists the candidates and touches nothing. An install is never a candidate.
3786
+ const markCopiesBody = z
3787
+ .object({
3788
+ cursor: scopeIdSchema.optional(),
3789
+ limit: z.number().int().min(1).max(200).default(50),
3790
+ dryRun: z.boolean().default(false),
3791
+ })
3792
+ .strict();
3793
+ app.post('/scopes/mark-copies', async (c) => {
3794
+ const { cursor, limit, dryRun } = await readJsonBody(c, markCopiesBody);
3795
+ const actor = c.get('actor');
3796
+ const page = await c.var.admin.listScopes(actor, {
3797
+ status: ['active', 'suspended', 'archived'],
3798
+ cursor,
3799
+ limit: REPAIR_SCAN_PAGE,
3800
+ });
3801
+ const marked = [];
3802
+ const already = [];
3803
+ const candidates = [];
3804
+ const skipped = [];
3805
+ const failed = [];
3806
+ let last = cursor;
3807
+ let handled = 0;
3808
+ let stopped = false;
3809
+ for (const scope of page) {
3810
+ if (isPrimaryScope(scope)) {
3811
+ last = scope.id;
3812
+ continue;
3813
+ }
3814
+ if (handled >= limit) {
3815
+ stopped = true;
3816
+ break;
3817
+ }
3818
+ handled += 1;
3819
+ last = scope.id;
3820
+ const at = { tenantId: scope.tenantId, scopeId: scope.id };
3821
+ if (dryRun) {
3822
+ candidates.push(at);
3823
+ continue;
3824
+ }
3825
+ try {
3826
+ const step = await markScopeIfCopy(c, scope);
3827
+ if ('skip' in step)
3828
+ skipped.push({ ...at, reason: step.skip });
3829
+ else if ('fail' in step)
3830
+ failed.push({ ...at, ...step.fail });
3831
+ else
3832
+ (step.done === 'marked' ? marked : already).push(at);
3833
+ }
3834
+ catch (e) {
3835
+ const { status } = mapError(e);
3836
+ failed.push({ ...at, status, error: e instanceof Error ? e.message : String(e) });
3837
+ }
3838
+ }
3839
+ const nextCursor = stopped || page.length === REPAIR_SCAN_PAGE ? (last ?? null) : null;
3840
+ return c.json({ dryRun, marked, already, candidates, skipped, failed, nextCursor });
3841
+ });
3842
+ // The correction for a MISTAKEN copy marker (#2005): a scope the directory says IS primary,
3843
+ // marked by a misclassification, a race or an operator, or restored from another scope's backup
3844
+ // before #2009 split the two facts, whose executors a CP-less host would otherwise hold inert
3845
+ // for good. One scope at a time, staff only by default-deny (the path is on neither allowlist),
3846
+ // logged as `clearScopeCopyMark`. Refused for a scope the directory calls a copy. Clears the
3847
+ // classification only: the events a load brought in stay marked, so they never run here.
3848
+ app.post('/tenants/:tenantId/scopes/:scopeId/clear-copy-mark', async (c) => {
3849
+ const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
3850
+ const scopeId = scopeIdSchema.parse(c.req.param('scopeId'));
3851
+ const actor = c.get('actor');
3852
+ const scope = await c.var.admin.getScopeRecord(actor, tenantId, scopeId);
3853
+ if (!scope)
3854
+ return c.json({ error: `unknown scope for tenant: (${tenantId}, ${scopeId})` }, 404);
3855
+ if (!isPrimaryScope(scope)) {
3856
+ return c.json({ error: 'clear-copy-mark refused: the directory says this scope is a copy (a preview or a fork)' }, 409);
3857
+ }
3858
+ const vertical = await verticalForScope(c, scope);
3859
+ if (!vertical)
3860
+ return c.json({ error: await diagnoseUnboundScope(actor, scope) }, 501);
3861
+ const { cleared } = await vertical.clearCopyMark(scopeId, { kind: scope.kind, forkedFrom: scope.forkedFrom });
3862
+ await c.var.admin.recordCopyMark(actor, { action: 'clear', tenantId, scopeId, outcome: cleared ? 'cleared' : 'absent' });
3863
+ return c.json({ cleared });
3864
+ });
3163
3865
  // Reap an ARCHIVED primary scope (control-plane.md §4.4): free its DO storage —
3164
3866
  // Cloudflare never garbage-collects a Durable Object, so a deleted app's bytes persist
3165
3867
  // forever otherwise — while keeping the directory row as a tombstone. A POST verb, not
@@ -3260,7 +3962,7 @@ export function createControlPlaneApi(options) {
3260
3962
  const actor = c.get('actor');
3261
3963
  // Body is optional so the bare `POST …/reap` every existing caller sends still
3262
3964
  // parses; `backup` tri-states on purpose (see the ordering comment below).
3263
- const { backup: wantsBackup } = reapScopeBody.parse(await c.req.json().catch(() => ({})));
3965
+ const { backup: wantsBackup } = await readJsonBody(c, reapScopeBody);
3264
3966
  const scope = await c.var.admin.getScopeRecord(actor, tenantId, scopeId);
3265
3967
  if (!scope)
3266
3968
  return c.json({ error: `unknown scope for tenant: (${tenantId}, ${scopeId})` }, 404);
@@ -3617,7 +4319,8 @@ export function createControlPlaneApi(options) {
3617
4319
  // through, makes the next sweep reconcile the scope, and that reconcile re-asserts
3618
4320
  // what the directory records as off. Left to the sweep rather than done here: the
3619
4321
  // scope's store restarts to finish the restore, and a re-assert now would race it.
3620
- // Meanwhile, the deployment that rewound it holds those modules off (#1819). A hold is
4322
+ // Meanwhile, the deployment that rewound it holds those modules off at its system door, so
4323
+ // their schedules and job runs both wait (#1819, #1834). A hold is
3621
4324
  // released by that reconcile's re-assert, or earlier by a switch move known to land on the
3622
4325
  // rewound storage, such as an operator's ON.
3623
4326
  await c.var.admin.markScopeProvisioned(actor, tenantId, scopeId, null);
@@ -3672,7 +4375,7 @@ export function createControlPlaneApi(options) {
3672
4375
  return c.json({ error: 'adopt-serving is a private-vertical operation' }, 409);
3673
4376
  }
3674
4377
  const owned = (await c.var.admin.listScopes(actor, { tenantId: v.ownerTenant, vertical: slug })).filter((s) => !s.forkedFrom && s.kind !== 'preview' && s.status === 'active');
3675
- const { acknowledge } = adoptServingBody.parse(await c.req.json().catch(() => ({})));
4378
+ const { acknowledge } = await readJsonBody(c, adoptVerticalServingBody);
3676
4379
  const adopted = [];
3677
4380
  const alreadyAdopted = [];
3678
4381
  try {
@@ -3732,6 +4435,226 @@ export function createControlPlaneApi(options) {
3732
4435
  const versionId = z.string().min(1).parse(c.req.query('versionId'));
3733
4436
  return c.json({ affected: await c.var.admin.bindingImpact(c.get('actor'), tenantId, scopeId, versionId) });
3734
4437
  });
4438
+ // -- kept copies (#1722, Codex #2008 r7) ----------------------------------------------------
4439
+ // A kept copy is one a carry's fenced wipe refused because it took a write the carry never
4440
+ // copied (a request still routed to the old script before the bind). The store protects it:
4441
+ // every load into it answers 409 until it is resolved here. The two resolutions both lose
4442
+ // something, so each is named and acknowledged, never defaulted:
4443
+ // - `discard` wipes the kept copy: what reached it after the carry's export is gone.
4444
+ // - `restore-forward` restores the kept copy over the store the scope now routes to, then
4445
+ // wipes the kept copy: what the live store took since the copy was kept is gone.
4446
+ // There is no merge of two scope dumps. To reconcile by hand, pull the kept copy with
4447
+ // `GET …/kept-copy/export` and the live store with the governed export (`GET …/export`, which
4448
+ // reads the scope's route), re-apply what matters, then discard.
4449
+ //
4450
+ // Staff only, by default-deny: neither path is on BUILDER_ROUTES or TENANT_ROUTES. One scope
4451
+ // at a time, refused unless the copy really is kept, and admin-logged with the write revision
4452
+ // before and after (`recordKeptCopyResolution`).
4453
+ const keptScriptQuery = z.object({ script: z.string().min(1) }).strict();
4454
+ const resolveKeptBody = z.discriminatedUnion('action', [
4455
+ z
4456
+ .object({
4457
+ script: z.string().min(1),
4458
+ action: z.literal('discard'),
4459
+ acknowledge: z.object({ discard: z.literal(true) }).strict(),
4460
+ })
4461
+ .strict(),
4462
+ z
4463
+ .object({
4464
+ script: z.string().min(1),
4465
+ action: z.literal('release'),
4466
+ acknowledge: z.object({ release: z.literal(true) }).strict(),
4467
+ })
4468
+ .strict(),
4469
+ z
4470
+ .object({
4471
+ script: z.string().min(1),
4472
+ action: z.literal('restore-forward'),
4473
+ /** The moment the copy was kept, as its marker says: the live store's writes since are replaced. */
4474
+ acknowledge: z.object({ replacesLiveWritesSince: z.string().min(1) }).strict(),
4475
+ })
4476
+ .strict(),
4477
+ ]);
4478
+ const keptHolder = async (script) => {
4479
+ if (!options.resolveVerticalRef)
4480
+ throw new ControlPlaneError(501, 'kept copies need dispatch resolution by script');
4481
+ const holder = await options.resolveVerticalRef(script);
4482
+ if (!holder)
4483
+ throw new ControlPlaneError(404, `no deployment resolves for script '${script}'`);
4484
+ return holder;
4485
+ };
4486
+ app.get('/tenants/:tenantId/scopes/:scopeId/kept-copy', async (c) => {
4487
+ const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
4488
+ const scopeId = scopeIdSchema.parse(c.req.param('scopeId'));
4489
+ const { script } = keptScriptQuery.parse(c.req.query());
4490
+ const scope = await c.var.admin.getScopeRecord(c.get('actor'), tenantId, scopeId);
4491
+ if (!scope)
4492
+ return c.json({ error: `unknown scope for tenant: (${tenantId}, ${scopeId})` }, 404);
4493
+ const kept = await (await keptHolder(script)).keptCopy(scopeId);
4494
+ return c.json({ scopeId, script, kept, routesTo: await routeOf(c, scope) });
4495
+ });
4496
+ // The kept copy's own dump, for reconciling by hand before a discard: the governed export's
4497
+ // rules exactly (`GET …/export` above). Refused outside `global` residency, masked unless
4498
+ // `full=true`, and K-24 access-logged through the same canonical `exportScope`. It reads the
4499
+ // script named, refused unless the copy there is kept, so it never becomes a way to pull a
4500
+ // live store past that route.
4501
+ const keptExportQuery = z
4502
+ .object({ script: z.string().min(1), full: z.enum(['true', 'false']).optional() })
4503
+ .strict();
4504
+ app.get('/tenants/:tenantId/scopes/:scopeId/kept-copy/export', async (c) => {
4505
+ const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
4506
+ const scopeId = scopeIdSchema.parse(c.req.param('scopeId'));
4507
+ const { script, full } = keptExportQuery.parse(c.req.query());
4508
+ const actor = c.get('actor');
4509
+ const scope = await c.var.admin.getScopeRecord(actor, tenantId, scopeId);
4510
+ if (!scope)
4511
+ return c.json({ error: `unknown scope for tenant: (${tenantId}, ${scopeId})` }, 404);
4512
+ if (scope.jurisdiction !== 'global') {
4513
+ return c.json({
4514
+ error: `scope ${scopeId} is pinned to '${scope.jurisdiction}' — a local pull would ` +
4515
+ `move its data outside that jurisdiction; refused (K-32, preview-and-snapshots.md §6)`,
4516
+ }, 403);
4517
+ }
4518
+ const holder = await keptHolder(script);
4519
+ const kept = await holder.keptCopy(scopeId);
4520
+ if (!kept)
4521
+ return c.json({ error: `scope ${scopeId} has no kept copy in '${script}'` }, 409);
4522
+ const dump = await c.var.admin.exportScope(actor, tenantId, scopeId);
4523
+ const tables = await holder.exportScope(scopeId);
4524
+ if (full === 'true')
4525
+ return c.json({ ...dump, tables, masked: false, script, kept });
4526
+ const mask = await createPseudonymizer(options.maskSalt ?? crypto.randomUUID());
4527
+ return c.json({ ...dump, tables: await maskDump(tables, mask), masked: true, script, kept });
4528
+ });
4529
+ app.post('/tenants/:tenantId/scopes/:scopeId/kept-copy/resolve', async (c) => {
4530
+ const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
4531
+ const scopeId = scopeIdSchema.parse(c.req.param('scopeId'));
4532
+ const body = await readJsonBody(c, resolveKeptBody);
4533
+ const actor = c.get('actor');
4534
+ const scope = await c.var.admin.getScopeRecord(actor, tenantId, scopeId);
4535
+ if (!scope)
4536
+ return c.json({ error: `unknown scope for tenant: (${tenantId}, ${scopeId})` }, 404);
4537
+ const holder = await keptHolder(body.script);
4538
+ const kept = await holder.keptCopy(scopeId);
4539
+ if (!kept)
4540
+ return c.json({ error: `scope ${scopeId} has no kept copy in '${body.script}'` }, 409);
4541
+ const route = await routeOf(c, scope);
4542
+ // `release` is for a kept copy the scope routes to (a rollback bound it after a carry kept
4543
+ // it); the other two act on a kept copy the scope has left, against the store it runs on.
4544
+ if (body.action === 'release' ? route !== body.script : !route || route === body.script) {
4545
+ return c.json({
4546
+ error: body.action === 'release'
4547
+ ? `scope ${scopeId} routes to '${route ?? '(nothing)'}', not to the kept copy in '${body.script}'; release is for a kept copy that is live`
4548
+ : `scope ${scopeId} routes to '${route ?? '(nothing)'}', so there is no live copy to resolve against`,
4549
+ }, 409);
4550
+ }
4551
+ const at = new Date().toISOString();
4552
+ const revisionOf = async (client) => {
4553
+ const marker = await client.loadMarker(scopeId);
4554
+ return marker === 'unfenced' ? null : marker.revision;
4555
+ };
4556
+ // The kept copy as it stands now, which is what this resolution acts on and is fenced on: a
4557
+ // write that reaches it after this read refuses the wipe. (The marker's own `revision` is the
4558
+ // one it was kept at; writing the marker, and any later write, moved it on.)
4559
+ const before = await holder.loadMarker(scopeId);
4560
+ const revisionBefore = before === 'unfenced' ? null : before.revision;
4561
+ // Codex #2008 r13: and the load stamp with it, so the release or discard refuses a store a load
4562
+ // has replaced since this read (a kept copy refuses every load but its own resolution, so this
4563
+ // is the same fence stated where the act is, not a new one).
4564
+ const stampFence = before === 'unfenced' ? {} : { loadStamp: before.loadStamp };
4565
+ // #2005 (Codex #2008 r10): whatever the resolution leaves in that store, it is a copy when
4566
+ // the directory says the scope is not primary, marker or no marker.
4567
+ const lineage = copyLineageOf(scope);
4568
+ const markCopy = lineage ? { markCopy: lineage } : {};
4569
+ const record = (action, revisionAfter) => c.var.admin.recordKeptCopyResolution(actor, tenantId, scopeId, {
4570
+ action,
4571
+ script: body.script,
4572
+ liveScript: action === 'restore-forward' ? route : null,
4573
+ keptAt: kept.keptAt,
4574
+ revisionBefore,
4575
+ revisionAfter,
4576
+ });
4577
+ if (body.action === 'release') {
4578
+ const out = await holder.releaseKeptCopy({ scopeId, revision: revisionBefore, ...markCopy, ...stampFence });
4579
+ if ('refused' in out) {
4580
+ return c.json({ error: `the kept copy of scope ${scopeId} ${out.refused === 'changed' ? 'changed since it was read' : 'is no longer kept'}; nothing was released` }, 412);
4581
+ }
4582
+ await record('release', revisionBefore);
4583
+ return c.json({ scopeId, script: body.script, action: 'release', revisionBefore });
4584
+ }
4585
+ if (body.action === 'discard') {
4586
+ const out = await holder.discardKeptCopy({ scopeId, revision: revisionBefore, carriedTo: route, at, ...markCopy, ...stampFence });
4587
+ if ('refused' in out) {
4588
+ return c.json({ error: `the kept copy of scope ${scopeId} ${out.refused === 'changed' ? 'changed since it was read' : 'is no longer kept'}; nothing was discarded` }, 412);
4589
+ }
4590
+ const revisionAfter = await revisionOf(holder);
4591
+ await record('discard', revisionAfter);
4592
+ return c.json({ scopeId, script: body.script, action: 'discard', revisionBefore, revisionAfter });
4593
+ }
4594
+ // restore-forward: the acknowledgement names what is replaced, as the marker says it.
4595
+ if (body.acknowledge.replacesLiveWritesSince !== kept.keptAt) {
4596
+ return c.json({
4597
+ error: `a restore forward replaces what the live store took since ${kept.keptAt}; acknowledge exactly that ` +
4598
+ `(acknowledge.replacesLiveWritesSince: '${kept.keptAt}')`,
4599
+ }, 409);
4600
+ }
4601
+ // Fenced on the binding through the whole resolution (Codex #2008 r8): the kept copy goes only
4602
+ // once its data is in the store the scope still routes to. Every refusal below leaves it kept.
4603
+ const liveScript = route;
4604
+ const live = await keptHolder(liveScript);
4605
+ const bindingHeld = async () => {
4606
+ const now = await c.var.admin.getScopeRecord(actor, tenantId, scopeId);
4607
+ return (Boolean(now) &&
4608
+ (now.verticalVersionId ?? null) === (scope.verticalVersionId ?? null) &&
4609
+ (now.servingRef ?? null) === (scope.servingRef ?? null));
4610
+ };
4611
+ const moved = () => c.json({
4612
+ error: `scope ${scopeId} was re-pointed while the kept copy was being restored forward, so nothing was resolved ` +
4613
+ `and the kept copy in '${body.script}' stays kept; read it again and resolve it again`,
4614
+ }, 409);
4615
+ // The live store as read now, and nothing else, is what gets replaced. The binding read again
4616
+ // after the marker ties the marker to the live store: a move that landed before this read
4617
+ // (its wipe minting a tombstone here) changed the binding, and is refused.
4618
+ const liveMarker = await live.loadMarker(scopeId);
4619
+ if (liveMarker === 'unfenced') {
4620
+ return c.json({ error: `the deployment serving scope ${scopeId} ('${liveScript}') predates the fence (#1722); re-push it, then resolve` }, 409);
4621
+ }
4622
+ if (!(await bindingHeld()))
4623
+ return moved();
4624
+ if ((await isCarriedAway(live, scope)) || (await live.keptCopy(scopeId))) {
4625
+ return c.json({ error: `the store scope ${scopeId} routes to in '${liveScript}' is not a live store (wiped or kept); nothing was resolved` }, 409);
4626
+ }
4627
+ const dump = await retryTransient(() => holder.exportScope(scopeId));
4628
+ const restored = await restoreCarryingSwitches(actor, live, tenantId, scopeId, dump, {
4629
+ scopeId,
4630
+ exact: true,
4631
+ loadStamp: ulid(),
4632
+ expect: liveMarker,
4633
+ });
4634
+ await c.var.admin.reassertSystemSwitches(actor, { tenantId, scopeId }, { appliedInUnit: restored.switchedOff });
4635
+ // Still routed there once the data landed? If not, the move that re-pointed the scope keeps
4636
+ // what this restore put in that store (its source wipe protects a copy changed since its
4637
+ // export), and the kept copy here stays kept too: nothing is lost, nothing is discarded.
4638
+ if (!(await bindingHeld()))
4639
+ return moved();
4640
+ const revisionAfter = await revisionOf(live);
4641
+ await record('restore-forward', revisionAfter);
4642
+ const wiped = await holder.discardKeptCopy({
4643
+ scopeId,
4644
+ revision: revisionBefore,
4645
+ carriedTo: liveScript,
4646
+ at,
4647
+ ...markCopy,
4648
+ ...stampFence,
4649
+ });
4650
+ if ('refused' in wiped) {
4651
+ return c.json({
4652
+ error: `restored forward into '${route}', but the kept copy in '${body.script}' changed since it was read, so it ` +
4653
+ `stays kept: read it again and resolve it again`,
4654
+ }, 409);
4655
+ }
4656
+ return c.json({ scopeId, script: body.script, action: 'restore-forward', liveScript, revisionBefore, revisionAfter });
4657
+ });
3735
4658
  // Pin a scope to a vertical version (#31; orchestration.md §4). Refuses a
3736
4659
  // non-admitted version below the seam — that refusal is the registry's reason to
3737
4660
  // exist. A scope operation, so it keeps the scope route shape. `snapshot: true`
@@ -3768,23 +4691,21 @@ export function createControlPlaneApi(options) {
3768
4691
  if (scope)
3769
4692
  carried = await carryOntoVersion(c, scope, versionId, { dropServingRef: repairPreviewPin });
3770
4693
  };
3771
- // #1674: after the bind, the scope may route to a different store (the carry's
3772
- // destination); put the directory's recorded OFF positions back there. #1742: the carry
3773
- // already did so in the restore's own event; what it reports is audited here.
3774
- const reassert = async () => {
3775
- if (scope) {
3776
- await c.var.admin.reassertSystemSwitches(actor, { tenantId, scopeId }, { appliedInUnit: carried?.switchedOff });
3777
- }
3778
- };
3779
4694
  // The host's own refusal, should the impact have changed since it was read above (a
3780
- // promote landing in between): relayed as the same 409, never as a 500.
4695
+ // promote landing in between): relayed as the same 409, never as a 500. A known scope
4696
+ // binds through the carry tail (#1722): compare-and-set on the binding read above, the pin
4697
+ // cleared after it, the OFF positions re-asserted where it now routes (#1674), and the copy
4698
+ // the carry left in the old script wiped.
3781
4699
  const bind = async (opts) => {
3782
- await c.var.admin.bindScopeVersion(actor, tenantId, scopeId, versionId, opts).catch(relayHostRefusal);
3783
- // Keep the old route until the bind succeeds. If clearing the pin fails, a retry
3784
- // carries again from the still-serving script before it changes the route.
3785
- if (repairPreviewPin) {
3786
- await c.var.admin.setScopeServingRef(actor, tenantId, scopeId, null).catch(relayHostRefusal);
4700
+ if (!scope) {
4701
+ await c.var.admin.bindScopeVersion(actor, tenantId, scopeId, versionId, opts).catch(relayHostRefusal);
4702
+ return;
3787
4703
  }
4704
+ await bindAfterCarry(c, scope, versionId, carried, {
4705
+ ...opts,
4706
+ clearServingRef: repairPreviewPin,
4707
+ recarry: () => carryOntoVersion(c, scope, versionId, { dropServingRef: repairPreviewPin }),
4708
+ });
3788
4709
  };
3789
4710
  if (snapshot) {
3790
4711
  if (!scope) {
@@ -3804,10 +4725,18 @@ export function createControlPlaneApi(options) {
3804
4725
  }
3805
4726
  await carry();
3806
4727
  if (migrationCrossing) {
3807
- // Asked again after the carry: a promote landing since the first question would
3808
- // otherwise leave an archive behind for a bind the host then refuses.
3809
- await refuseOnBreaks(actor, tenantId, scopeId, versionId, { acknowledge });
3810
- await orchestratedSnapshot(c, tenantId, scope, {});
4728
+ try {
4729
+ // Asked again after the carry: a promote landing since the first question would
4730
+ // otherwise leave an archive behind for a bind the host then refuses.
4731
+ await refuseOnBreaks(actor, tenantId, scopeId, versionId, { acknowledge });
4732
+ await orchestratedSnapshot(c, tenantId, scope, {});
4733
+ }
4734
+ catch (e) {
4735
+ // #1722: no bind follows, so the copy the carry restored is reached by nothing.
4736
+ if (carried)
4737
+ await dropUnboundCopy(c, scope, versionId, carried);
4738
+ throw e;
4739
+ }
3811
4740
  }
3812
4741
  await bind({ acknowledge });
3813
4742
  }
@@ -3815,12 +4744,10 @@ export function createControlPlaneApi(options) {
3815
4744
  await carry();
3816
4745
  await bind({ acknowledge, snapshot: true });
3817
4746
  }
3818
- await reassert();
3819
4747
  return c.json(await c.var.admin.getScopeRecord(actor, tenantId, scopeId));
3820
4748
  }
3821
4749
  await carry();
3822
4750
  await bind({ acknowledge });
3823
- await reassert();
3824
4751
  return c.json(await c.var.admin.getScopeRecord(actor, tenantId, scopeId));
3825
4752
  });
3826
4753
  // -- instances (K-31) -------------------------------------------------------
@@ -4514,8 +5441,17 @@ export function createControlPlaneApi(options) {
4514
5441
  throw new ControlPlaneError(502, `version ${versionId} retained no manifest — pushed pre-#286; push it again to serve in place`);
4515
5442
  }
4516
5443
  const manifest = storedDeployManifest.parse(JSON.parse(manifestJson));
5444
+ // #1902: the classes this upload DEPLOYS — the manifest's, plus the platform's sweeper when
5445
+ // the version's recorded decision supplies it. The upload, the migration delta and the
5446
+ // serving record below all read this one set; the manifest's own `doClasses` would leave
5447
+ // the record a class short of the script, and the next serve re-declaring a live class.
5448
+ const doClasses = deployedDoClasses(manifest.doClasses, manifest.platformSweeper);
4517
5449
  const serving = await host.admin.verticalServing(actor, slug);
4518
5450
  const ref = serving?.ref ?? stableDeploymentRefFor(slug);
5451
+ // #1923: the serving upload is the other door a script name is minted through.
5452
+ const collision = platformScriptCollision(slug) ?? (PLATFORM_SCRIPT_NAMES.has(ref) ? ref : null);
5453
+ if (collision)
5454
+ throw new ControlPlaneError(422, platformScriptRefusal(slug, collision));
4519
5455
  const modules = await options.fetchVerticalModules(version.deploymentRef);
4520
5456
  // #301 PR-2: every serving upload re-derives the per-tenant store D1 bindings from
4521
5457
  // the ledger and sends them WITH the bundle's own bindings — an upload's `bindings`
@@ -4537,7 +5473,7 @@ export function createControlPlaneApi(options) {
4537
5473
  compatibilityDate: manifest.compatibilityDate,
4538
5474
  compatibilityFlags: manifest.compatibilityFlags,
4539
5475
  modules,
4540
- doClasses: manifest.doClasses,
5476
+ doClasses,
4541
5477
  bindings: [...manifest.bindings, ...storeBindings],
4542
5478
  // #1242: the version now being served, injected as `SUBSTRAT_VERSION_ID` so the
4543
5479
  // scope host stamps the signals `version` dimension. The serving script is one
@@ -4547,6 +5483,10 @@ export function createControlPlaneApi(options) {
4547
5483
  // #1054: the model runtime is bound only for a version that ASKED for it, so the
4548
5484
  // capability is visible in the manifest diff at admit rather than fleet-wide.
4549
5485
  ...(manifest.usesModels ? { usesModels: true } : {}),
5486
+ // #1902: the sweeper decision the PUSH made and recorded, reused — never re-made, so a
5487
+ // promote or a backout cannot refuse a version that was accepted. A version pushed
5488
+ // before the record existed has none and keeps what it had: no platform sweeper.
5489
+ ...(manifest.platformSweeper ? { supplySweeper: true } : {}),
4550
5490
  // #340/#578: the version's static files travel with it onto the serving script —
4551
5491
  // from the RETAINED manifest, with no bytes. The runtime's asset store dedupes
4552
5492
  // per SCRIPT (not namespace-wide — the #578 finding), so the serving script only
@@ -4570,18 +5510,14 @@ export function createControlPlaneApi(options) {
4570
5510
  }, serving
4571
5511
  ? { priorDoClasses: serving.doClasses, priorMigrationTag: serving.migrationTag }
4572
5512
  : undefined);
4573
- const addedClasses = serving
4574
- ? manifest.doClasses.some((cls) => !serving.doClasses.includes(cls))
4575
- : false;
5513
+ const addedClasses = serving ? doClasses.some((cls) => !serving.doClasses.includes(cls)) : false;
4576
5514
  await admin.setVerticalServing(actor, slug, {
4577
5515
  ref,
4578
5516
  versionId,
4579
5517
  // The serving script's class set only ever GROWS (DO classes cannot be deleted
4580
5518
  // while their storage lives), so record the union, and the tag only moves when
4581
5519
  // a migration actually rode the upload.
4582
- doClasses: serving
4583
- ? [...new Set([...serving.doClasses, ...manifest.doClasses])]
4584
- : manifest.doClasses,
5520
+ doClasses: serving ? [...new Set([...serving.doClasses, ...doClasses])] : doClasses,
4585
5521
  migrationTag: serving
4586
5522
  ? addedClasses
4587
5523
  ? nextMigrationTag(serving.migrationTag)
@@ -4929,6 +5865,12 @@ export function createControlPlaneApi(options) {
4929
5865
  slug = bare;
4930
5866
  ownerTenant = (await ownerOf(c.get('actor'), slug)) ?? null;
4931
5867
  }
5868
+ // #1923: a slug that would deploy under one of the platform's own script names is refused
5869
+ // before anything is uploaded. Registration refuses it too; this is the deploy half, for a
5870
+ // registry id that predates the rule, so it can never put a script under a platform name.
5871
+ const collision = platformScriptCollision(slug);
5872
+ if (collision)
5873
+ return c.json({ error: platformScriptRefusal(slug, collision) }, 422);
4932
5874
  // #388: refuse the SILENT lineage fork. A first push of a registry id that does not
4933
5875
  // exist yet, whose product name (the slug's tail) matches an existing lineage this
4934
5876
  // push could CONFUSE ITSELF WITH, is almost always a mis-identified project — a
@@ -5038,6 +5980,13 @@ export function createControlPlaneApi(options) {
5038
5980
  error: `${assetParts.size} uploaded asset part(s) are absent from the manifest (${extra}${assetParts.size > 5 ? ', …' : ''})`,
5039
5981
  }, 400);
5040
5982
  }
5983
+ // #1902: whether this version's uploads carry the platform's scope sweeper — decided here,
5984
+ // ONCE, before anything is uploaded, and recorded with the version below so every later
5985
+ // upload of it reuses the answer. A declaration that contradicts itself is refused now,
5986
+ // while it is still a push; it can never become a promote or a backout that refuses.
5987
+ const sweeper = platformSweeperDecision(manifest, { entry: manifest.entry, modules });
5988
+ if ('refuse' in sweeper)
5989
+ return c.json({ error: sweeper.refuse }, 422);
5041
5990
  // Mint the version id first: the deploymentRef (the dispatch script name) is keyed
5042
5991
  // on it, so it is CF-valid and unique per version.
5043
5992
  const id = ulid();
@@ -5053,6 +6002,8 @@ export function createControlPlaneApi(options) {
5053
6002
  // #1242: the freshly minted version id — this archive script IS that version,
5054
6003
  // and a preview routed at it stamps the signals `version` dimension honestly.
5055
6004
  versionId: id,
6005
+ // #1902: the platform's scope sweeper, as decided above and recorded with the version.
6006
+ ...(sweeper.supply ? { supplySweeper: true } : {}),
5056
6007
  // #340: the verified bytes go up with the bundle. The manifest's routing config
5057
6008
  // rides along untouched — it decides what the RUNTIME does with paths, and carries
5058
6009
  // no reach, so there is nothing in it for the sandbox contract to refuse.
@@ -5132,7 +6083,8 @@ export function createControlPlaneApi(options) {
5132
6083
  ...(origin ? { origin } : {}),
5133
6084
  // Retained for the serving upload (#286): the archive script keeps the module
5134
6085
  // bytes, this keeps their shape (entry, compat, doClasses, bindings).
5135
- manifestJson: JSON.stringify(manifest),
6086
+ // #1902: with the sweeper decision the push made, which no later upload re-makes.
6087
+ manifestJson: JSON.stringify({ ...manifest, platformSweeper: sweeper.supply }),
5136
6088
  });
5137
6089
  // Same spirit as the permission-surface gate, advisory tier: when the push DECLARES
5138
6090
  // surfaces, name any surface that hostnames are still bound to but the declaration
@@ -6080,13 +7032,15 @@ export function createControlPlaneApi(options) {
6080
7032
  }
6081
7033
  throw e;
6082
7034
  }
6083
- // Heal a preview provisioned before #527: clear any inherited serving_ref so routing
6084
- // follows the bound version (its per-version script), not the prod serving script.
6085
- if (existing.servingRef)
6086
- await c.var.admin.setScopeServingRef(actor, tenantId, existing.id, null);
6087
- await c.var.admin.bindScopeVersion(actor, tenantId, existing.id, opts.versionId);
6088
- // #1674: re-assert any recorded OFF in the script the preview now routes to.
6089
- await c.var.admin.reassertSystemSwitches(actor, { tenantId, scopeId: existing.id }, { appliedInUnit: carried?.switchedOff });
7035
+ // The carry tail (#1722): the bind expects the version the carry exported from, so of two
7036
+ // pushes that read the same binding only one lands. Then a preview provisioned before
7037
+ // #527 has its inherited serving_ref cleared, so routing follows the bound version (its
7038
+ // per-version script) and not the prod serving script; the OFF positions are re-asserted
7039
+ // there (#1674); and the copy left in the old script is wiped.
7040
+ await bindAfterCarry(c, existing, opts.versionId, carried, {
7041
+ clearServingRef: true,
7042
+ recarry: () => carryOntoVersion(c, existing, opts.versionId, { dropServingRef: true }),
7043
+ });
6090
7044
  // Renew (or clear) the preview's GC deadline so a reused preview does not silently die.
6091
7045
  await c.var.admin.setScopeExpiresAt(actor, tenantId, existing.id, expiresAt);
6092
7046
  const hostname = await bindPreviewHostname(c.var.admin, actor, baseHostname, tenantId, existing.id, opts.tag, surface);
@@ -6158,6 +7112,11 @@ export function createControlPlaneApi(options) {
6158
7112
  };
6159
7113
  let row;
6160
7114
  let restore;
7115
+ /** #2009: the directory's classification of the row below, which every restore of it carries. */
7116
+ const markCopyOfRow = () => {
7117
+ const lineage = copyLineageOf({ kind: row.kind ?? '', forkedFrom: row.forkedFrom ?? null });
7118
+ return lineage ? { markCopy: lineage } : {};
7119
+ };
6161
7120
  if (source) {
6162
7121
  // A fresh fork. Export from where the prod data lives TODAY. The canonical
6163
7122
  // `admin.exportScope` first — it writes the K-24 audit entry (and the co-located
@@ -6188,7 +7147,9 @@ export function createControlPlaneApi(options) {
6188
7147
  // there; restore re-projects the vertical's roles from the dump's tuples). A one-shot
6189
7148
  // DO storage blip heals on the in-request retry WITHOUT burning a CI attempt (which
6190
7149
  // pushes a fresh version per try) — #559 (2).
6191
- restore = () => target.restoreScope(tenantId, previewId, tables, { sourceScopeId: source.id, exact: true });
7150
+ // #2009: a load classifies nothing by itself, so the restore carries the directory's
7151
+ // classification of the row below, as every restore onto a non-primary scope does.
7152
+ restore = () => target.restoreScope(tenantId, previewId, tables, { sourceScopeId: source.id, exact: true, ...markCopyOfRow() });
6192
7153
  }
6193
7154
  else {
6194
7155
  // A clean-room preview (#509 (b)): an EMPTY scope, no source to export. No `forkedFrom`
@@ -6206,7 +7167,8 @@ export function createControlPlaneApi(options) {
6206
7167
  jurisdiction: 'global',
6207
7168
  expiresAt: expiresAt ?? undefined,
6208
7169
  };
6209
- restore = target ? () => target.restoreScope(tenantId, previewId, []) : null;
7170
+ // #2009: marked a copy on the directory's word, as the fork above.
7171
+ restore = target ? () => target.restoreScope(tenantId, previewId, [], markCopyOfRow()) : null;
6210
7172
  }
6211
7173
  try {
6212
7174
  // Inside the try: a host writes the directory row FIRST and then migrates, projects and