@substrat-run/control-plane-api 0.135.0 → 0.136.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/aggregate-reads.d.ts.map +1 -1
- package/dist/aggregate-reads.js +4 -1
- package/dist/aggregate-reads.js.map +1 -1
- package/dist/aggregate-source.d.ts +17 -1
- package/dist/aggregate-source.d.ts.map +1 -1
- package/dist/aggregate-source.js +16 -10
- package/dist/aggregate-source.js.map +1 -1
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +1016 -65
- package/dist/api.js.map +1 -1
- package/dist/cf-observability.d.ts +6 -0
- package/dist/cf-observability.d.ts.map +1 -1
- package/dist/cf-observability.js +125 -33
- package/dist/cf-observability.js.map +1 -1
- package/dist/connect-url.d.ts +2 -2
- package/dist/connect-url.d.ts.map +1 -1
- package/dist/connect-url.js +7 -1
- package/dist/connect-url.js.map +1 -1
- package/dist/connection-relay.d.ts +4 -2
- package/dist/connection-relay.d.ts.map +1 -1
- package/dist/connection-relay.js +10 -1
- package/dist/connection-relay.js.map +1 -1
- package/dist/deploy.d.ts +3 -0
- package/dist/deploy.d.ts.map +1 -1
- package/dist/deploy.js.map +1 -1
- package/dist/failure-attribution.js +1 -1
- package/dist/failure-attribution.js.map +1 -1
- package/dist/index.d.ts +6 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -3
- package/dist/index.js.map +1 -1
- package/dist/observability.d.ts +20 -1
- package/dist/observability.d.ts.map +1 -1
- package/dist/observability.js +11 -1
- package/dist/observability.js.map +1 -1
- package/dist/platform-drain.d.ts +24 -2
- package/dist/platform-drain.d.ts.map +1 -1
- package/dist/platform-drain.js +36 -3
- package/dist/platform-drain.js.map +1 -1
- package/dist/platform-entry.d.ts +62 -6
- package/dist/platform-entry.d.ts.map +1 -1
- package/dist/platform-entry.generated.d.ts +8 -1
- package/dist/platform-entry.generated.d.ts.map +1 -1
- package/dist/platform-entry.generated.js +10 -3
- package/dist/platform-entry.generated.js.map +1 -1
- package/dist/platform-entry.js +151 -9
- package/dist/platform-entry.js.map +1 -1
- package/dist/provider-error.d.ts +9 -0
- package/dist/provider-error.d.ts.map +1 -0
- package/dist/provider-error.js +9 -0
- package/dist/provider-error.js.map +1 -0
- package/dist/tenant-request-query.d.ts.map +1 -1
- package/dist/tenant-request-query.js +3 -1
- package/dist/tenant-request-query.js.map +1 -1
- package/dist/vertical-client.d.ts +177 -40
- package/dist/vertical-client.d.ts.map +1 -1
- package/dist/vertical-client.js +350 -171
- package/dist/vertical-client.js.map +1 -1
- package/package.json +7 -7
package/dist/api.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { resolveObservabilityWindow } from './observability-window.js';
|
|
2
2
|
import { Hono } from 'hono';
|
|
3
|
-
import { parseHostname, withLabel, RESERVED_LABEL_SEPARATOR, adminAction, ownerTransferInput, OWNER_TRANSFER_AUDIT_ERROR_MAX, ASSET_PART_PREFIX, assetHash, channelName, connectionActivity, connectionActivitySource, connectionCredential, connectionFilter, connectionHealthState, connectionProvider, connectorDispatchKind, PROVISION_SIBLING_KIND, ARCHIVE_SCOPE_KIND, PROVISION_TENANT_KIND, SET_ENTITLEMENTS_KIND, MODEL_USAGE_KIND, SWEEP_RUNS_KIND, toConnectionHealthEntry, CONNECTION_EXPIRY_WARNING_DAYS, CONNECTION_STALE_DAYS, connectionProbe, createTenantInput, entitlementGrantInput, hostname as hostnameSchema, hostnameRegion, hostnameStatus, identityLink, errorCode, issueStatus, issueStatusInput, listPageQuery, matchesOutboundHost, pageOf, platformRequestFilter, principalId as principalIdSchema, impersonationStamp, promotionAcknowledgement, bindAcknowledgement, provisionableJurisdiction, publishVersionInput, queryScopeInput, readScopeTableInput, entityHistoryInput, eventFacetInput, eventCauseInput, eventEffectsInput, invocationEventsInput, deadLettersInput, lifecycleFlowInput, delegatedReadParams, createOrgInput, roleKey as roleKeySchema, peerSwitch, systemSwitch, DEFAULT_DENIAL_LIMIT, DENIAL_LIMIT_MAX, denialGroupBy, registerVerticalInput, serviceDimensions, sweepRunKind, sweepRunOutcome, scopeDump, dataSubjectId as dataSubjectIdSchema, scopeId as scopeIdSchema, scopeStatus, storageShape, surfaceName, tenantId as tenantIdSchema, tenantStatus, versionOrigin, z, PROBLEM_CONTENT_TYPE, toProblem, redrainEventsInput, importCursorMove, importCursorAcknowledgementMissing, REDRAIN_BATCH, migrationsOnTop, errorCodeOf, capabilityFilterQuery, } from '@substrat-run/contracts';
|
|
3
|
+
import { parseHostname, withLabel, RESERVED_LABEL_SEPARATOR, adminAction, ownerTransferInput, OWNER_TRANSFER_AUDIT_ERROR_MAX, ASSET_PART_PREFIX, assetHash, channelName, connectionActivity, connectionActivitySource, connectionCredential, connectionFilter, connectionHealthState, connectionProvider, connectorDispatchKind, PROVISION_SIBLING_KIND, ARCHIVE_SCOPE_KIND, PROVISION_TENANT_KIND, SET_ENTITLEMENTS_KIND, MODEL_USAGE_KIND, SWEEP_RUNS_KIND, toConnectionHealthEntry, CONNECTION_EXPIRY_WARNING_DAYS, CONNECTION_STALE_DAYS, connectionProbe, createTenantInput, entitlementGrantInput, hostname as hostnameSchema, hostnameRegion, hostnameStatus, identityLink, errorCode, issueStatus, issueStatusInput, listPageQuery, matchesOutboundHost, pageOf, platformRequestFilter, principalId as principalIdSchema, impersonationStamp, promotionAcknowledgement, bindAcknowledgement, provisionableJurisdiction, publishVersionInput, queryScopeInput, readScopeTableInput, entityHistoryInput, eventFacetInput, eventCauseInput, eventEffectsInput, invocationEventsInput, deadLettersInput, lifecycleFlowInput, delegatedReadParams, createOrgInput, roleKey as roleKeySchema, peerSwitch, systemSwitch, DEFAULT_DENIAL_LIMIT, DENIAL_LIMIT_MAX, denialGroupBy, registerVerticalInput, serviceDimensions, sweepRunKind, sweepRunOutcome, scopeDump, dataSubjectId as dataSubjectIdSchema, scopeId as scopeIdSchema, scopeStatus, storageShape, surfaceName, tenantId as tenantIdSchema, tenantStatus, versionOrigin, z, PROBLEM_CONTENT_TYPE, toProblem, redrainEventsInput, importCursorMove, importCursorAcknowledgementMissing, REDRAIN_BATCH, migrationsOnTop, errorCodeOf, capabilityFilterQuery, PLATFORM_FEATURES_HEADER, PLATFORM_FEATURE_SCOPE_SWEEPER, } from '@substrat-run/contracts';
|
|
4
4
|
import { attributeFailure } from './failure-attribution.js';
|
|
5
|
-
import { BIND_EXPORT_BREAK_REFUSAL, bindExportBreakRefusal, crossVerticalHealth, exportBreakRefusal, isBindExportBreakRefusal, isExportBreakRefusal, migrationProgress, ulid, } from '@substrat-run/kernel';
|
|
5
|
+
import { BIND_EXPORT_BREAK_REFUSAL, CARRIED_AWAY_KEY, bindExportBreakRefusal, carriedAwayDump, dumpMetaValue, metaValueIn, crossVerticalHealth, exportBreakRefusal, isBindExportBreakRefusal, isExportBreakRefusal, isPrimaryScope, migrationProgress, ulid, } from '@substrat-run/kernel';
|
|
6
6
|
import { TENANT_HEADER, confinedTenant } from './auth.js';
|
|
7
7
|
import { mintTenantToken } from './tenant-token.js';
|
|
8
8
|
import { connectionGrantsForScope } from './vertical-client.js';
|
|
@@ -37,6 +37,7 @@ import { TENANT_SERIES_SCOPE_CAP, REQUEST_FACET_KEYS } from './observability.js'
|
|
|
37
37
|
import { parseTenantRequestQuery, parseTenantWindowQuery, refineTenantWindow, scriptFamiliesOfScopes } from './tenant-request-query.js';
|
|
38
38
|
import { namespacesForScript } from './do-namespaces.js';
|
|
39
39
|
import { backupDirectoryIfDue } from './directory-backup.js';
|
|
40
|
+
import { deployedDoClasses, platformSweeperDecision } from './platform-entry.js';
|
|
40
41
|
import { STORAGE_PAGE_DEFAULT, STORAGE_PAGE_MAX, readStoragePage } from './storage-meter.js';
|
|
41
42
|
import { isCustomHostname, validateBindableHostname, } from './custom-hostnames.js';
|
|
42
43
|
/**
|
|
@@ -46,6 +47,13 @@ import { isCustomHostname, validateBindableHostname, } from './custom-hostnames.
|
|
|
46
47
|
*/
|
|
47
48
|
export const CLI_MIN_VERSION_HEADER = 'x-substrat-cli-min-version';
|
|
48
49
|
export const CLI_LATEST_VERSION_HEADER = 'x-substrat-cli-latest-version';
|
|
50
|
+
/**
|
|
51
|
+
* The directory's classification of a scope that is not primary (#2005), which a verb sends so
|
|
52
|
+
* the store it writes is marked a copy whatever its own marker says; undefined for a primary one.
|
|
53
|
+
*/
|
|
54
|
+
function copyLineageOf(scope) {
|
|
55
|
+
return isPrimaryScope(scope) ? undefined : { kind: scope.kind, forkedFrom: scope.forkedFrom };
|
|
56
|
+
}
|
|
49
57
|
/**
|
|
50
58
|
* "This principal is confined to a tenant, and this is not that tenant."
|
|
51
59
|
*
|
|
@@ -70,6 +78,27 @@ class ExportBreakRefused extends ControlPlaneError {
|
|
|
70
78
|
this.breaks = breaks;
|
|
71
79
|
}
|
|
72
80
|
}
|
|
81
|
+
/**
|
|
82
|
+
* A carry whose source store kept changing between its export and its bind (#1722, Codex #2008
|
|
83
|
+
* r12): every re-carry from a fresh export found the source changed again before it could bind.
|
|
84
|
+
* Nothing was bound and the destination's copy was dropped, so the caller retries the bind. Typed,
|
|
85
|
+
* so a client tells it from the other 409s: `carrySourceChanged` names the two scripts and how many
|
|
86
|
+
* exports were tried.
|
|
87
|
+
*/
|
|
88
|
+
class CarrySourceChanged extends ControlPlaneError {
|
|
89
|
+
scopeId;
|
|
90
|
+
from;
|
|
91
|
+
to;
|
|
92
|
+
attempts;
|
|
93
|
+
constructor(scopeId, from, to, attempts) {
|
|
94
|
+
super(409, `scope ${scopeId}'s store in '${from}' changed while it was being carried to '${to}', on each of ` +
|
|
95
|
+
`${attempts} exports, so nothing was bound (#1722). Retry the bind.`);
|
|
96
|
+
this.scopeId = scopeId;
|
|
97
|
+
this.from = from;
|
|
98
|
+
this.to = to;
|
|
99
|
+
this.attempts = attempts;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
73
102
|
/** The body a route answers an `ExportBreakRefused` with: the sentence and the listing. */
|
|
74
103
|
function exportBreakBody(e) {
|
|
75
104
|
return { error: e.message, ...(e instanceof ExportBreakRefused ? { exportBreaks: { affected: e.breaks } } : {}) };
|
|
@@ -253,6 +282,9 @@ const rebindScopeVerticalBody = z.object({
|
|
|
253
282
|
// #1756: an adopt moves the scope onto what its vertical serves, and is refused when that breaks
|
|
254
283
|
// an app in the tenant. The body is optional, as it always was.
|
|
255
284
|
const adoptServingBody = z.object({ acknowledge: bindAcknowledgement.optional() });
|
|
285
|
+
// The BULK form starts a run over every still-legacy scope of a vertical, so it reads its body
|
|
286
|
+
// strictly (`readJsonBody`); the single-scope form above keeps the lenient parse it always had.
|
|
287
|
+
const adoptVerticalServingBody = adoptServingBody.strict();
|
|
256
288
|
// A snapshot request (preview-and-snapshots.md §3/§9). `expiresAt` opts into the GC
|
|
257
289
|
// sweep; absent = pinned until deliberately deleted. `kind` defaults to 'archive'.
|
|
258
290
|
const snapshotScopeBody = z.object({
|
|
@@ -265,9 +297,43 @@ const snapshotScopeBody = z.object({
|
|
|
265
297
|
// false — the explicit "I accept an unrecoverable wipe"
|
|
266
298
|
// undefined — back up if a store is configured, proceed without one if not, which is
|
|
267
299
|
// what keeps every pre-#493 caller (and self-host) working unchanged
|
|
268
|
-
const reapScopeBody = z
|
|
300
|
+
const reapScopeBody = z
|
|
301
|
+
.object({
|
|
269
302
|
backup: z.boolean().optional(),
|
|
270
|
-
})
|
|
303
|
+
})
|
|
304
|
+
.strict();
|
|
305
|
+
/**
|
|
306
|
+
* Read the JSON body of a route whose default is an ACT (a reap, a bulk adoption, a fleet
|
|
307
|
+
* repair), under one grammar. The old `await c.req.json().catch(() => ({}))` turned a body
|
|
308
|
+
* that did not parse into the defaults, so a body of `{"backup": true` (cut short on the wire)
|
|
309
|
+
* ran the action it was cut short of describing.
|
|
310
|
+
*
|
|
311
|
+
* **Only a ZERO-LENGTH body takes the defaults.** Any other body must hold a JSON value, or it
|
|
312
|
+
* is a 400. That is deliberately not "an empty-looking body": a body of whitespace, or of a
|
|
313
|
+
* byte-order mark and nothing else, is a request that did not arrive intact, and reading it as
|
|
314
|
+
* "no body" would run the default act on it (a tenant reap of ` \t\n ` reaped with no backup).
|
|
315
|
+
* So the check is on the BYTES, before any decoding or trimming, and a leading BOM is stripped
|
|
316
|
+
* only so that JSON following it can parse. Do not "simplify" this to `text.trim() === ''`.
|
|
317
|
+
*
|
|
318
|
+
* The schema decides the rest. One that guards an act is `.strict()`, so a misspelt key is
|
|
319
|
+
* refused rather than dropped.
|
|
320
|
+
*/
|
|
321
|
+
async function readJsonBody(c, schema) {
|
|
322
|
+
const bytes = await c.req.arrayBuffer();
|
|
323
|
+
let raw = {};
|
|
324
|
+
if (bytes.byteLength > 0) {
|
|
325
|
+
// `ignoreBOM: true` KEEPS the mark in the string; the default decoder would drop it and
|
|
326
|
+
// let a BOM-only body decode to '' and look empty.
|
|
327
|
+
const text = new TextDecoder('utf-8', { ignoreBOM: true }).decode(bytes).replace(/^\uFEFF/, '');
|
|
328
|
+
try {
|
|
329
|
+
raw = JSON.parse(text);
|
|
330
|
+
}
|
|
331
|
+
catch {
|
|
332
|
+
throw new ControlPlaneError(400, 'the body is not valid JSON');
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
return schema.parse(raw);
|
|
336
|
+
}
|
|
271
337
|
// A directory-restore request (#40). `capturedAt` addresses the copy — there is one
|
|
272
338
|
// directory, so that is its whole address. `overwrite` is the guard against the
|
|
273
339
|
// dangerous case: replaying a restore onto a control plane that already recovered.
|
|
@@ -695,17 +761,22 @@ export function createControlPlaneApi(options) {
|
|
|
695
761
|
// The CLI version advisory (#971), stamped on EVERY response — including the 401 the
|
|
696
762
|
// auth middleware below answers with, since a CLI too old to authenticate is exactly
|
|
697
763
|
// the one that needs to hear it. Registered first so nothing can return around it.
|
|
698
|
-
// Nothing configured ⇒ no header
|
|
764
|
+
// Nothing configured ⇒ no advisory header.
|
|
765
|
+
//
|
|
766
|
+
// Beside it, the capability half of the same handshake (#1902): what this plane provides
|
|
767
|
+
// that a CLI may leave out on its strength. Stamped unconfigured, because it is a fact about
|
|
768
|
+
// this code rather than about a deployment — the uploader this package ships supplies the
|
|
769
|
+
// scope sweeper. The CLI asks before omitting one and refuses on silence, which is what an
|
|
770
|
+
// older plane answers.
|
|
699
771
|
const advisory = options.cliAdvisory;
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
}
|
|
772
|
+
app.use('*', async (c, next) => {
|
|
773
|
+
await next();
|
|
774
|
+
c.header(PLATFORM_FEATURES_HEADER, PLATFORM_FEATURE_SCOPE_SWEEPER);
|
|
775
|
+
if (advisory?.minVersion)
|
|
776
|
+
c.header(CLI_MIN_VERSION_HEADER, advisory.minVersion);
|
|
777
|
+
if (advisory?.latestVersion)
|
|
778
|
+
c.header(CLI_LATEST_VERSION_HEADER, advisory.latestVersion);
|
|
779
|
+
});
|
|
709
780
|
// Fail closed, before any route runs: no principal, no reach.
|
|
710
781
|
//
|
|
711
782
|
// The TENANT-SCOPED credential is read FIRST (#977), ahead of staff. Not for
|
|
@@ -962,6 +1033,9 @@ export function createControlPlaneApi(options) {
|
|
|
962
1033
|
}
|
|
963
1034
|
if (err instanceof ExportBreakRefused)
|
|
964
1035
|
return c.json(exportBreakBody(err), 409);
|
|
1036
|
+
if (err instanceof CarrySourceChanged) {
|
|
1037
|
+
return c.json({ error: err.message, carrySourceChanged: { scopeId: err.scopeId, from: err.from, to: err.to, attempts: err.attempts } }, 409);
|
|
1038
|
+
}
|
|
965
1039
|
const { status, body } = mapError(err);
|
|
966
1040
|
// A 5xx is the PLATFORM failing — an unmapped throw, or a downstream vertical's
|
|
967
1041
|
// own 5xx passing through (a DO storage fault during a preview restore is the
|
|
@@ -1055,7 +1129,7 @@ export function createControlPlaneApi(options) {
|
|
|
1055
1129
|
app.post('/tenants/:tenantId/reap', async (c) => {
|
|
1056
1130
|
const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
|
|
1057
1131
|
const actor = c.get('actor');
|
|
1058
|
-
const { backup: wantsBackup } =
|
|
1132
|
+
const { backup: wantsBackup } = await readJsonBody(c, reapScopeBody);
|
|
1059
1133
|
const tenant = await c.var.admin.getTenant(actor, tenantId);
|
|
1060
1134
|
if (!tenant)
|
|
1061
1135
|
return c.json({ error: `unknown tenant: ${tenantId}` }, 404);
|
|
@@ -1843,12 +1917,23 @@ export function createControlPlaneApi(options) {
|
|
|
1843
1917
|
*/
|
|
1844
1918
|
const restoreCarryingSwitches = async (actor, dest, tenantId, scopeId, tables,
|
|
1845
1919
|
/** #1869: the scope `tables` were captured from — `scopeId` itself unless the copy moves —
|
|
1846
|
-
* and `exact` when the platform exported them itself rather than a caller supplying them.
|
|
1920
|
+
* and `exact` when the platform exported them itself rather than a caller supplying them.
|
|
1921
|
+
* #1722: `loadStamp`, the stamp a carry leaves on the copy it lands, and `expect`, the marker
|
|
1922
|
+
* the carry read from `dest`, so a store that moved since is never overwritten. */
|
|
1847
1923
|
source) => {
|
|
1924
|
+
// #2005: every carry, reuse, adopt and governed restore lands here. When the directory says
|
|
1925
|
+
// the destination is not primary, the vertical marks it a copy in its own storage — a carry
|
|
1926
|
+
// of a copy made before every copy carried the marker brings none, and a CP-less host reads
|
|
1927
|
+
// the marker for primacy. Idempotent there; an install is never marked.
|
|
1928
|
+
const rec = await host.admin.getScopeRecord(actor, tenantId, scopeId);
|
|
1929
|
+
const markCopy = rec ? copyLineageOf(rec) : undefined;
|
|
1848
1930
|
return retryTransient(async () => dest.restoreScope(tenantId, scopeId, tables, {
|
|
1849
1931
|
...(await switchCarryFor(host.admin, actor, { tenantId, scopeId })),
|
|
1850
1932
|
sourceScopeId: source.scopeId,
|
|
1851
1933
|
exact: source.exact,
|
|
1934
|
+
loadStamp: source.loadStamp,
|
|
1935
|
+
expect: source.expect,
|
|
1936
|
+
...(markCopy ? { markCopy } : {}),
|
|
1852
1937
|
}));
|
|
1853
1938
|
};
|
|
1854
1939
|
const adoptScopeOntoServing = async (c, tenantId, scopeId, opts = {}) => {
|
|
@@ -2042,6 +2127,16 @@ export function createControlPlaneApi(options) {
|
|
|
2042
2127
|
await c.var.admin.reassertSystemSwitches(actor, { tenantId, scopeId }, { appliedInUnit: restored.switchedOff });
|
|
2043
2128
|
return { servingRef: serving.ref, versionId: serving.versionId, tables: restored.tables };
|
|
2044
2129
|
};
|
|
2130
|
+
/** The script a scope's requests reach: its serving pin, else its bound version's own script. */
|
|
2131
|
+
const routeOf = async (c, scope) => {
|
|
2132
|
+
if (scope.servingRef)
|
|
2133
|
+
return scope.servingRef;
|
|
2134
|
+
if (!scope.vertical || !scope.verticalVersionId)
|
|
2135
|
+
return null;
|
|
2136
|
+
return (await c.var.admin.getVersion(c.get('actor'), scope.verticalVersionId, scope.vertical))?.deploymentRef ?? null;
|
|
2137
|
+
};
|
|
2138
|
+
/** How many exports a carry tries before its source holds still long enough to bind (#1722 r12). */
|
|
2139
|
+
const CARRY_EXPORT_ATTEMPTS = 3;
|
|
2045
2140
|
/**
|
|
2046
2141
|
* Carry a scope's data onto the script a version bind is about to route it to (#1710).
|
|
2047
2142
|
* The caller binds only after this returns.
|
|
@@ -2069,11 +2164,23 @@ export function createControlPlaneApi(options) {
|
|
|
2069
2164
|
* over any partial copy. A retry after a bind that landed finds the same script on both
|
|
2070
2165
|
* ends and carries nothing.
|
|
2071
2166
|
*
|
|
2167
|
+
* #1722: the export is checked before anything is restored. The scope is read again, and a
|
|
2168
|
+
* binding or pin that moved since the caller read it is refused (412): another carry bound
|
|
2169
|
+
* away from this script, which may already have wiped it, and an export from a wiped DO comes
|
|
2170
|
+
* back as an empty store. A dump that carries the `carried_away` tombstone is refused the same
|
|
2171
|
+
* way. The caller binds with `bindAfterCarry`, which wipes the source copy once the bind
|
|
2172
|
+
* lands, so neither check can pass on an export from a copy that is about to go.
|
|
2173
|
+
*
|
|
2174
|
+
* The restore is conditional too. Before the check above, the carry reads the destination's
|
|
2175
|
+
* `LoadMarker` (load stamp and write revision), and the restore is refused inside its own
|
|
2176
|
+
* transaction if the store moved since. So a carry held before its restore cannot overwrite
|
|
2177
|
+
* a store that another carry into the same script has loaded, or that went live under that
|
|
2178
|
+
* carry's bind and took a write: a CI retry of the same version. A deployment built before
|
|
2179
|
+
* #1722 cannot read a marker, and there the binding re-read is all that guards the restore.
|
|
2180
|
+
* Its window: a bind that lands between that re-read and the restore, followed by a write.
|
|
2181
|
+
*
|
|
2072
2182
|
* What it does not do:
|
|
2073
2183
|
* - Carry writes that land on the source between the export and the bind. Those are lost.
|
|
2074
|
-
* - Delete the source copy. That copy is unreachable once the caller binds, and it stays
|
|
2075
|
-
* in the old script, because nothing reaps it yet (#1722). Deleting it here would race
|
|
2076
|
-
* a concurrent push that is still exporting from it.
|
|
2077
2184
|
* - Cross lineages. A version of another vertical reads as absent here, and moving a scope
|
|
2078
2185
|
* between lineages is `rebindScopeOntoVertical`'s job, behind its migration gate.
|
|
2079
2186
|
*/
|
|
@@ -2095,10 +2202,7 @@ export function createControlPlaneApi(options) {
|
|
|
2095
2202
|
if (!resolveVersion || (scope.servingRef && !opts.dropServingRef))
|
|
2096
2203
|
return null;
|
|
2097
2204
|
const to = incoming.deploymentRef ?? null;
|
|
2098
|
-
const
|
|
2099
|
-
? await c.var.admin.getVersion(actor, scope.verticalVersionId, scope.vertical)
|
|
2100
|
-
: undefined;
|
|
2101
|
-
const from = scope.servingRef ?? bound?.deploymentRef ?? null;
|
|
2205
|
+
const from = await routeOf(c, scope);
|
|
2102
2206
|
if (!to || !from || from === to)
|
|
2103
2207
|
return null;
|
|
2104
2208
|
// Resolve each end at exactly the script named above. No fallback to the prod channel:
|
|
@@ -2116,13 +2220,343 @@ export function createControlPlaneApi(options) {
|
|
|
2116
2220
|
throw new ControlPlaneError(502, `cannot move scope ${scope.id} from '${from}' to '${to}': ${missing} could not be reached, ` +
|
|
2117
2221
|
`so the version was not bound (#1710)`);
|
|
2118
2222
|
}
|
|
2119
|
-
const dump = await retryTransient(() => source.
|
|
2120
|
-
//
|
|
2223
|
+
const { tables: dump, loadStamp: sourceStamp, revision: sourceRevision } = await retryTransient(() => source.exportScopeStamped(scope.id));
|
|
2224
|
+
// What the destination holds now, read before the binding is checked again. The restore
|
|
2225
|
+
// sends it back and is refused if the store moved since: another carry into the same script
|
|
2226
|
+
// loaded it, or it went live under another carry's bind and took a write. That is a CI retry
|
|
2227
|
+
// of the same version, held before its restore while the first run bound and served.
|
|
2228
|
+
const destMarker = await retryTransient(() => dest.loadMarker(scope.id));
|
|
2229
|
+
const now = await c.var.admin.getScopeRecord(actor, scope.tenantId, scope.id);
|
|
2230
|
+
if (!now ||
|
|
2231
|
+
(now.verticalVersionId ?? null) !== (scope.verticalVersionId ?? null) ||
|
|
2232
|
+
(now.servingRef ?? null) !== (scope.servingRef ?? null)) {
|
|
2233
|
+
throw new ControlPlaneError(412, `scope ${scope.id} was re-pointed while its data was being copied out of '${from}', so nothing was ` +
|
|
2234
|
+
`moved or bound (#1722). Reload the scope and retry.`);
|
|
2235
|
+
}
|
|
2236
|
+
if (dumpMetaValue(dump, CARRIED_AWAY_KEY) !== null) {
|
|
2237
|
+
throw new ControlPlaneError(412, `the copy of scope ${scope.id} in '${from}' was already carried to another script and wiped, so nothing ` +
|
|
2238
|
+
`was moved or bound (#1722). Reload the scope and retry.`);
|
|
2239
|
+
}
|
|
2240
|
+
// #1742: the recorded OFF positions ride the restore, as on adopt and rebind. #1722: so does
|
|
2241
|
+
// a stamp of this carry's own, which a refused bind's wipe of the copy expects.
|
|
2242
|
+
const restoredStamp = ulid();
|
|
2121
2243
|
const restored = await restoreCarryingSwitches(actor, dest, scope.tenantId, scope.id, dump, {
|
|
2122
2244
|
scopeId: scope.id,
|
|
2123
2245
|
exact: true,
|
|
2246
|
+
loadStamp: restoredStamp,
|
|
2247
|
+
...(destMarker === 'unfenced' ? {} : { expect: destMarker }),
|
|
2248
|
+
});
|
|
2249
|
+
return {
|
|
2250
|
+
from,
|
|
2251
|
+
to,
|
|
2252
|
+
tables: restored.tables,
|
|
2253
|
+
switchedOff: restored.switchedOff,
|
|
2254
|
+
source,
|
|
2255
|
+
dest,
|
|
2256
|
+
sourceStamp,
|
|
2257
|
+
sourceRevision,
|
|
2258
|
+
restoredStamp,
|
|
2259
|
+
};
|
|
2260
|
+
};
|
|
2261
|
+
/**
|
|
2262
|
+
* Wipe one copy of a scope that no route reaches (#1722), without reaping it. Fenced where the
|
|
2263
|
+
* deployment can fence it: the wipe runs only if nothing was loaded into that DO since
|
|
2264
|
+
* `expectLoadStamp`, compared inside the wipe's own transaction, and `false` means something
|
|
2265
|
+
* was (a rollback restored there), so the copy stays. A deployment built before the fence
|
|
2266
|
+
* answers `'unfenced'`, and the wipe is then an unconditional load of the tombstone through
|
|
2267
|
+
* the restore verb every deployment has. The caller re-reads the route first; that re-read is
|
|
2268
|
+
* all that guards an unfenced wipe.
|
|
2269
|
+
*/
|
|
2270
|
+
const wipeCarriedCopy = async (holder, scope, expect, carriedTo,
|
|
2271
|
+
/** The scope was read routing elsewhere: a copy changed in any way since is kept (#1722 r8). */
|
|
2272
|
+
protectIfChanged = false) => {
|
|
2273
|
+
const at = new Date().toISOString();
|
|
2274
|
+
// #2005 (Codex #2008 r10): the directory's word that the scope is not primary goes with the
|
|
2275
|
+
// wipe, on both paths, so a copy made before the marker leaves it marked — tombstoned or kept.
|
|
2276
|
+
const markCopy = copyLineageOf(scope);
|
|
2277
|
+
const fenced = await retryTransient(() => holder.wipeCarriedCopy({
|
|
2278
|
+
scopeId: scope.id,
|
|
2279
|
+
expectLoadStamp: expect.loadStamp,
|
|
2280
|
+
...(expect.revision !== undefined ? { expectRevision: expect.revision } : {}),
|
|
2281
|
+
...(protectIfChanged ? { protectIfChanged: true } : {}),
|
|
2282
|
+
carriedTo,
|
|
2283
|
+
at,
|
|
2284
|
+
...(markCopy ? { markCopy } : {}),
|
|
2285
|
+
}));
|
|
2286
|
+
if (fenced !== 'unfenced')
|
|
2287
|
+
return fenced.wiped;
|
|
2288
|
+
await retryTransient(() => holder.restoreScope(scope.tenantId, scope.id, carriedAwayDump({ to: carriedTo, at }), {
|
|
2289
|
+
sourceScopeId: scope.id,
|
|
2290
|
+
exact: true,
|
|
2291
|
+
// #2005: as every restore onto a non-primary scope, so the wiped copy stays a copy.
|
|
2292
|
+
...(markCopy ? { markCopy } : {}),
|
|
2293
|
+
}));
|
|
2294
|
+
return true;
|
|
2295
|
+
};
|
|
2296
|
+
/**
|
|
2297
|
+
* Where the scope routes now, read again after a carry's bind (null for a scope gone from the
|
|
2298
|
+
* directory). Bound to the carry's own version and unpinned, that is the carry's destination,
|
|
2299
|
+
* which needs no version read.
|
|
2300
|
+
*/
|
|
2301
|
+
const currentRoute = async (c, scope, versionId, carried) => {
|
|
2302
|
+
const now = await c.var.admin.getScopeRecord(c.get('actor'), scope.tenantId, scope.id);
|
|
2303
|
+
if (!now)
|
|
2304
|
+
return null;
|
|
2305
|
+
if (!now.servingRef && now.verticalVersionId === versionId)
|
|
2306
|
+
return carried.to;
|
|
2307
|
+
return routeOf(c, now);
|
|
2308
|
+
};
|
|
2309
|
+
/** Whether the scope's store in `holder` is a wiped copy: it holds the `carried_away` tombstone. */
|
|
2310
|
+
const isCarriedAway = async (holder, scope) => {
|
|
2311
|
+
const page = await holder.readScopeTable(scope.id, { table: '_substrat_meta', limit: 100, offset: 0 });
|
|
2312
|
+
return metaValueIn(page.rows, page.columns, CARRIED_AWAY_KEY) !== null;
|
|
2313
|
+
};
|
|
2314
|
+
const recordCarryCleanup = (c, scope, versionId, stage, e) => {
|
|
2315
|
+
const { status } = mapError(e);
|
|
2316
|
+
recordFailure({
|
|
2317
|
+
actor: c.get('actor'),
|
|
2318
|
+
operation: 'scope.carry',
|
|
2319
|
+
stage,
|
|
2320
|
+
tenantId: scope.tenantId,
|
|
2321
|
+
scopeId: scope.id,
|
|
2322
|
+
vertical: scope.vertical ?? null,
|
|
2323
|
+
version: versionId,
|
|
2324
|
+
status,
|
|
2325
|
+
message: e instanceof Error ? e.message : String(e),
|
|
2326
|
+
}, e);
|
|
2327
|
+
};
|
|
2328
|
+
/**
|
|
2329
|
+
* Bind a scope onto `versionId` after `carryOntoVersion` (#1722), and settle the copies. The
|
|
2330
|
+
* one tail every carry caller runs: a preview push, `scope bind`, and the serving-pin repair.
|
|
2331
|
+
*
|
|
2332
|
+
* 1. The bind is compare-and-set on the binding the caller read, which is the version the carry
|
|
2333
|
+
* exported from. Two pushes that both read v1 cannot both bind: the second is refused (412)
|
|
2334
|
+
* and never routes the preview to what it carried, which may be an export of a copy the
|
|
2335
|
+
* first one had already wiped. On any failed bind, the copy this carry restored is wiped, unless
|
|
2336
|
+
* the scope now routes to that same script: a CI retry of the same version restores into
|
|
2337
|
+
* the winner's live store, and wiping it there would lose everything.
|
|
2338
|
+
* 2. The pin is cleared (`clearServingRef`) only after the bind lands, so a failure in between
|
|
2339
|
+
* leaves the route where the data still is, and a retry carries again.
|
|
2340
|
+
* 3. The recorded OFF positions are re-asserted where the scope now routes (#1674).
|
|
2341
|
+
* 4. The copy left in the old script is wiped (`settleCarriedSource`). A failure there is
|
|
2342
|
+
* recorded and never fails the bind, which has landed: the copy then stays, as before #1722.
|
|
2343
|
+
*/
|
|
2344
|
+
const bindAfterCarry = async (c, scope, versionId, carried, opts = {}) => {
|
|
2345
|
+
const actor = c.get('actor');
|
|
2346
|
+
// 0. The source, read again right before the bind (Codex #2008 r12). A store that changed since
|
|
2347
|
+
// the export (a write, a load, or a staff clear of its copy marker) holds something the
|
|
2348
|
+
// destination does not, so binding now would serve the stale export. The carry is run
|
|
2349
|
+
// again from a fresh export, a bounded number of times, and then refused with a typed 409.
|
|
2350
|
+
// What changes after this read is the source wipe's to fence (`keepOrWipeSource`).
|
|
2351
|
+
for (let attempt = 1; carried; attempt++) {
|
|
2352
|
+
const stale = carried;
|
|
2353
|
+
let now;
|
|
2354
|
+
try {
|
|
2355
|
+
now = await retryTransient(() => stale.source.loadMarker(scope.id));
|
|
2356
|
+
}
|
|
2357
|
+
catch (e) {
|
|
2358
|
+
await dropUnboundCopy(c, scope, versionId, stale); // no bind follows
|
|
2359
|
+
throw e;
|
|
2360
|
+
}
|
|
2361
|
+
if (now === 'unfenced' || (now.loadStamp === stale.sourceStamp && now.revision === stale.sourceRevision))
|
|
2362
|
+
break;
|
|
2363
|
+
let next = null;
|
|
2364
|
+
if (attempt < CARRY_EXPORT_ATTEMPTS && opts.recarry) {
|
|
2365
|
+
try {
|
|
2366
|
+
next = await opts.recarry();
|
|
2367
|
+
}
|
|
2368
|
+
catch (e) {
|
|
2369
|
+
await dropUnboundCopy(c, scope, versionId, stale);
|
|
2370
|
+
throw e;
|
|
2371
|
+
}
|
|
2372
|
+
}
|
|
2373
|
+
if (!next) {
|
|
2374
|
+
await dropUnboundCopy(c, scope, versionId, stale);
|
|
2375
|
+
throw new CarrySourceChanged(scope.id, stale.from, stale.to, attempt);
|
|
2376
|
+
}
|
|
2377
|
+
carried = next;
|
|
2378
|
+
}
|
|
2379
|
+
try {
|
|
2380
|
+
await c.var.admin
|
|
2381
|
+
.bindScopeVersion(actor, scope.tenantId, scope.id, versionId, {
|
|
2382
|
+
acknowledge: opts.acknowledge,
|
|
2383
|
+
...(opts.snapshot ? { snapshot: true } : {}),
|
|
2384
|
+
expectedVersionId: scope.verticalVersionId ?? null,
|
|
2385
|
+
})
|
|
2386
|
+
.catch(relayHostRefusal);
|
|
2387
|
+
}
|
|
2388
|
+
catch (e) {
|
|
2389
|
+
if (carried)
|
|
2390
|
+
await dropUnboundCopy(c, scope, versionId, carried);
|
|
2391
|
+
throw e;
|
|
2392
|
+
}
|
|
2393
|
+
if (opts.clearServingRef && scope.servingRef) {
|
|
2394
|
+
await c.var.admin.setScopeServingRef(actor, scope.tenantId, scope.id, null).catch(relayHostRefusal);
|
|
2395
|
+
}
|
|
2396
|
+
try {
|
|
2397
|
+
await c.var.admin.reassertSystemSwitches(actor, { tenantId: scope.tenantId, scopeId: scope.id }, { appliedInUnit: carried?.switchedOff });
|
|
2398
|
+
}
|
|
2399
|
+
finally {
|
|
2400
|
+
// Even when the re-assert fails: the route has moved, and a retry finds the same script
|
|
2401
|
+
// on both ends, carries nothing, and so would never come back for this copy.
|
|
2402
|
+
if (carried)
|
|
2403
|
+
await settleCarriedSource(c, scope, versionId, carried);
|
|
2404
|
+
}
|
|
2405
|
+
};
|
|
2406
|
+
/**
|
|
2407
|
+
* The copy a carry restored for a bind that did not land, wiped unless the scope routes to
|
|
2408
|
+
* that script now. Best effort: it never throws, and a failure is recorded.
|
|
2409
|
+
*/
|
|
2410
|
+
const dropUnboundCopy = async (c, scope, versionId, carried) => {
|
|
2411
|
+
try {
|
|
2412
|
+
const route = await currentRoute(c, scope, versionId, carried);
|
|
2413
|
+
if (route === carried.to)
|
|
2414
|
+
return;
|
|
2415
|
+
await wipeCarriedCopy(carried.dest, scope, { loadStamp: carried.restoredStamp }, route ?? carried.from);
|
|
2416
|
+
}
|
|
2417
|
+
catch (e) {
|
|
2418
|
+
recordCarryCleanup(c, scope, versionId, 'unbound-copy', e);
|
|
2419
|
+
}
|
|
2420
|
+
};
|
|
2421
|
+
/**
|
|
2422
|
+
* Clear the kept-copy marker of a store the scope routes to (#1722, Codex #2008 r8). A carry
|
|
2423
|
+
* keeps a changed source when it reads the route elsewhere; a rollback binding it a moment later
|
|
2424
|
+
* makes that store live, and a live store holds nothing divergent. Fenced on the revision read
|
|
2425
|
+
* now; re-reads the route first, so it never releases a copy the scope has left.
|
|
2426
|
+
*/
|
|
2427
|
+
const releaseIfLive = async (c, scope, versionId, carried, holder, script) => {
|
|
2428
|
+
if (!(await holder.keptCopy(scope.id)))
|
|
2429
|
+
return;
|
|
2430
|
+
if ((await currentRoute(c, scope, versionId, carried)) !== script)
|
|
2431
|
+
return;
|
|
2432
|
+
const marker = await holder.loadMarker(scope.id);
|
|
2433
|
+
if (marker === 'unfenced')
|
|
2434
|
+
return;
|
|
2435
|
+
const markCopy = copyLineageOf(scope);
|
|
2436
|
+
await holder.releaseKeptCopy({
|
|
2437
|
+
scopeId: scope.id,
|
|
2438
|
+
revision: marker.revision,
|
|
2439
|
+
loadStamp: marker.loadStamp, // Codex #2008 r13: the store read, not only its revision
|
|
2440
|
+
...(markCopy ? { markCopy } : {}),
|
|
2441
|
+
});
|
|
2442
|
+
};
|
|
2443
|
+
/**
|
|
2444
|
+
* A source kept ONLY because staff cleared its copy marker after the carry's export (Codex #2008
|
|
2445
|
+
* r12; the store records that itself, `KeptCopy.clearedOnly`). Its data is what the carry
|
|
2446
|
+
* copied; the fresher decision is the clear, so it is brought to the store the scope runs on:
|
|
2447
|
+
* the destination's marker is cleared, and only then is the source discarded. True when both
|
|
2448
|
+
* landed. Anything else (a write since, a move, a refusal) leaves the source kept, and the caller
|
|
2449
|
+
* records it as any kept copy.
|
|
2450
|
+
*
|
|
2451
|
+
* Every step is fenced on THIS carry, not on whatever the stores hold now (Codex #2008 r13). The
|
|
2452
|
+
* destination must still hold this carry's load (its `restoredStamp`): a governed restore that
|
|
2453
|
+
* replaced it since brings a copy marker that is genuine, and clearing it would let another
|
|
2454
|
+
* scope's copy run its executors. That stamp and the revision read are compared inside the clear's
|
|
2455
|
+
* own transaction, so nothing can land between the check and the clear. The binding is read again
|
|
2456
|
+
* right before the discard, and the discard is fenced on the source's stamp and kept revision.
|
|
2457
|
+
*/
|
|
2458
|
+
const bringClearForward = async (c, scope, versionId, carried) => {
|
|
2459
|
+
if (!isPrimaryScope(scope))
|
|
2460
|
+
return false; // only staff clear, and only a primary's marker
|
|
2461
|
+
const kept = await carried.source.keptCopy(scope.id);
|
|
2462
|
+
if (!kept?.clearedOnly)
|
|
2463
|
+
return false;
|
|
2464
|
+
const destMarker = await carried.dest.loadMarker(scope.id);
|
|
2465
|
+
if (destMarker === 'unfenced' || destMarker.loadStamp !== carried.restoredStamp)
|
|
2466
|
+
return false;
|
|
2467
|
+
if ((await currentRoute(c, scope, versionId, carried)) !== carried.to)
|
|
2468
|
+
return false;
|
|
2469
|
+
try {
|
|
2470
|
+
await carried.dest.clearCopyMark(scope.id, { kind: scope.kind, forkedFrom: scope.forkedFrom }, {
|
|
2471
|
+
expect: { loadStamp: carried.restoredStamp, revision: destMarker.revision },
|
|
2472
|
+
});
|
|
2473
|
+
}
|
|
2474
|
+
catch (e) {
|
|
2475
|
+
if (e instanceof ControlPlaneError && (e.status === 412 || e.status === 409))
|
|
2476
|
+
return false;
|
|
2477
|
+
throw e;
|
|
2478
|
+
}
|
|
2479
|
+
if ((await currentRoute(c, scope, versionId, carried)) !== carried.to)
|
|
2480
|
+
return false;
|
|
2481
|
+
const discarded = await carried.source.discardKeptCopy({
|
|
2482
|
+
scopeId: scope.id,
|
|
2483
|
+
revision: kept.clearedOnly.revision,
|
|
2484
|
+
// Clear-only means nothing was loaded there since the export: the stamp the carry read.
|
|
2485
|
+
loadStamp: carried.sourceStamp,
|
|
2486
|
+
carriedTo: carried.to,
|
|
2487
|
+
at: new Date().toISOString(),
|
|
2124
2488
|
});
|
|
2125
|
-
return
|
|
2489
|
+
return 'discarded' in discarded;
|
|
2490
|
+
};
|
|
2491
|
+
/**
|
|
2492
|
+
* The source wipe, fenced on what the export read. Refused means the copy changed after the
|
|
2493
|
+
* export: a write from a request still routed there before the bind, which the carry never
|
|
2494
|
+
* copied. The copy is then kept, holding that write, and recorded so it can be recovered.
|
|
2495
|
+
*/
|
|
2496
|
+
const keepOrWipeSource = async (c, scope, versionId, carried, expect) => {
|
|
2497
|
+
// The caller read the route elsewhere, so a copy that changed in any way since the export (a
|
|
2498
|
+
// load as much as a write: a staff restore-forward into it racing this carry, Codex #2008 r8)
|
|
2499
|
+
// is kept in the store rather than left an unmarked orphan.
|
|
2500
|
+
if (await wipeCarriedCopy(carried.source, scope, expect, carried.to, true))
|
|
2501
|
+
return;
|
|
2502
|
+
// A rollback that restored into it since made it the live store again: it is no kept copy, and
|
|
2503
|
+
// a marker this refusal set while the rollback had not yet bound comes off.
|
|
2504
|
+
const route = await currentRoute(c, scope, versionId, carried);
|
|
2505
|
+
if (route === carried.from) {
|
|
2506
|
+
await releaseIfLive(c, scope, versionId, carried, carried.source, carried.from);
|
|
2507
|
+
return;
|
|
2508
|
+
}
|
|
2509
|
+
if (route === carried.to && (await bringClearForward(c, scope, versionId, carried)))
|
|
2510
|
+
return;
|
|
2511
|
+
recordCarryCleanup(c, scope, versionId, 'source-copy-kept', new ControlPlaneError(409, `scope ${scope.id}'s copy in '${carried.from}' changed after it was carried to '${carried.to}', so it was ` +
|
|
2512
|
+
`kept: it holds what reached it in between, which the carry did not copy (#1722)`));
|
|
2513
|
+
};
|
|
2514
|
+
/**
|
|
2515
|
+
* Wipe the copy a landed carry left in its source script (#1722), once nothing routes there.
|
|
2516
|
+
*
|
|
2517
|
+
* The route is read again first. A scope that routes to the source again was bound back to it
|
|
2518
|
+
* (a rollback that has landed), and its copy stays. A scope that routes to this carry's
|
|
2519
|
+
* destination has its store checked for the `carried_away` tombstone first: an unfenced wipe
|
|
2520
|
+
* from an earlier carry out of this script can overtake this carry's restore when this carry
|
|
2521
|
+
* IS the rollback, and then the source, which nothing has wiped yet, is carried again before
|
|
2522
|
+
* it goes. The fenced wipe in a deployment that has it makes the overtaking impossible.
|
|
2523
|
+
*
|
|
2524
|
+
* Left open, for a script built before the fence only: an unfenced wipe whose request reaches
|
|
2525
|
+
* the old script after a rollback's bind and this check, both, still destroys the rollback's
|
|
2526
|
+
* restore. Narrow, and recorded here rather than claimed closed.
|
|
2527
|
+
*/
|
|
2528
|
+
const settleCarriedSource = async (c, scope, versionId, carried) => {
|
|
2529
|
+
const actor = c.get('actor');
|
|
2530
|
+
try {
|
|
2531
|
+
const route = await currentRoute(c, scope, versionId, carried);
|
|
2532
|
+
if (route === carried.from)
|
|
2533
|
+
return;
|
|
2534
|
+
// The store this carry landed in is live; a kept marker another carry set while racing it
|
|
2535
|
+
// does not belong on it (#1722 r8).
|
|
2536
|
+
if (route === carried.to)
|
|
2537
|
+
await releaseIfLive(c, scope, versionId, carried, carried.dest, carried.to);
|
|
2538
|
+
if (route === carried.to && (await isCarriedAway(carried.dest, scope))) {
|
|
2539
|
+
const { tables: dump, loadStamp: stamp, revision } = await retryTransient(() => carried.source.exportScopeStamped(scope.id));
|
|
2540
|
+
if (dumpMetaValue(dump, CARRIED_AWAY_KEY) !== null) {
|
|
2541
|
+
throw new ControlPlaneError(500, `both copies of scope ${scope.id} hold the carried_away tombstone; nothing was wiped`);
|
|
2542
|
+
}
|
|
2543
|
+
await restoreCarryingSwitches(actor, carried.dest, scope.tenantId, scope.id, dump, {
|
|
2544
|
+
scopeId: scope.id,
|
|
2545
|
+
exact: true,
|
|
2546
|
+
loadStamp: ulid(),
|
|
2547
|
+
});
|
|
2548
|
+
if (await isCarriedAway(carried.dest, scope)) {
|
|
2549
|
+
throw new ControlPlaneError(500, `scope ${scope.id}'s store in '${carried.to}' was wiped again; its source copy stays`);
|
|
2550
|
+
}
|
|
2551
|
+
// Re-carried, so the fence expects what the re-export read.
|
|
2552
|
+
await keepOrWipeSource(c, scope, versionId, carried, { loadStamp: stamp, revision });
|
|
2553
|
+
return;
|
|
2554
|
+
}
|
|
2555
|
+
await keepOrWipeSource(c, scope, versionId, carried, { loadStamp: carried.sourceStamp, revision: carried.sourceRevision });
|
|
2556
|
+
}
|
|
2557
|
+
catch (e) {
|
|
2558
|
+
recordCarryCleanup(c, scope, versionId, 'source-copy', e);
|
|
2559
|
+
}
|
|
2126
2560
|
};
|
|
2127
2561
|
app.get('/tenants/:tenantId/scopes/:scopeId/tables', async (c) => {
|
|
2128
2562
|
const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
|
|
@@ -2847,10 +3281,49 @@ export function createControlPlaneApi(options) {
|
|
|
2847
3281
|
archive: (admin, a, t, s) => admin.archiveScope(a, t, s),
|
|
2848
3282
|
unarchive: (admin, a, t, s) => admin.unarchiveScope(a, t, s),
|
|
2849
3283
|
};
|
|
3284
|
+
/** #2005: the transitions that bring a scope (back) to life — each marks a copy first. */
|
|
3285
|
+
const REACTIVATIONS = new Set(['activate', 'unsuspend', 'unarchive']);
|
|
2850
3286
|
for (const [action, run] of Object.entries(transitions)) {
|
|
2851
3287
|
app.post(`/tenants/:tenantId/scopes/:scopeId/${action}`, async (c) => {
|
|
2852
3288
|
const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
|
|
2853
3289
|
const scopeId = scopeIdSchema.parse(c.req.param('scopeId'));
|
|
3290
|
+
// #2005: a copy made before the copy marker existed holds none, and a CP-less host that
|
|
3291
|
+
// finds none runs its effects. So a copy is marked BEFORE it comes back to life, and a
|
|
3292
|
+
// hosted copy whose marker cannot be written is refused here rather than reactivated.
|
|
3293
|
+
if (REACTIVATIONS.has(action)) {
|
|
3294
|
+
const rec = await c.var.admin.getScopeRecord(c.get('actor'), tenantId, scopeId);
|
|
3295
|
+
if (rec && !isPrimaryScope(rec)) {
|
|
3296
|
+
// ANY failed marker write keeps the scope parked: a 503 that says what to do, and an
|
|
3297
|
+
// ops-failure row, whether the step reported it (no deployment resolves) or the vertical
|
|
3298
|
+
// refused (an older one has no /internal/mark-copy and answers 404). Passing a 404 or
|
|
3299
|
+
// a vertical's error through would read as "no such scope" and leave no trace.
|
|
3300
|
+
let failure;
|
|
3301
|
+
try {
|
|
3302
|
+
const step = await markScopeIfCopy(c, rec);
|
|
3303
|
+
if ('fail' in step)
|
|
3304
|
+
failure = { error: step.fail.error, cause: NO_CAUSE };
|
|
3305
|
+
}
|
|
3306
|
+
catch (e) {
|
|
3307
|
+
failure = { error: e instanceof Error ? e.message : String(e), cause: e };
|
|
3308
|
+
}
|
|
3309
|
+
if (failure) {
|
|
3310
|
+
const error = `${action} refused: this copy could not be marked as one (${failure.error}); ` +
|
|
3311
|
+
'redeploy the vertical so it can mark it, then retry — the scope stays as it was';
|
|
3312
|
+
recordFailure({
|
|
3313
|
+
actor: c.get('actor'),
|
|
3314
|
+
operation: `scope.${action}`,
|
|
3315
|
+
stage: 'mark-copy',
|
|
3316
|
+
tenantId,
|
|
3317
|
+
scopeId,
|
|
3318
|
+
vertical: rec.vertical,
|
|
3319
|
+
version: rec.verticalVersionId,
|
|
3320
|
+
status: 503,
|
|
3321
|
+
message: error,
|
|
3322
|
+
}, failure.cause);
|
|
3323
|
+
return c.json({ error }, 503);
|
|
3324
|
+
}
|
|
3325
|
+
}
|
|
3326
|
+
}
|
|
2854
3327
|
await run(c.var.admin, c.get('actor'), tenantId, scopeId);
|
|
2855
3328
|
return c.json(await c.var.admin.getScopeRecord(c.get('actor'), tenantId, scopeId));
|
|
2856
3329
|
});
|
|
@@ -3160,6 +3633,235 @@ export function createControlPlaneApi(options) {
|
|
|
3160
3633
|
await c.var.admin.restoreDirectory(actor, dump);
|
|
3161
3634
|
return c.json({ capturedAt: dump.capturedAt, tables: dump.tables.length });
|
|
3162
3635
|
});
|
|
3636
|
+
// -- one-time repair of previews that kept a production serving pin (#1724) --------------
|
|
3637
|
+
// A preview adopted onto its vertical's serving script before #1731 still routes by
|
|
3638
|
+
// `COALESCE(servingRef, deploymentRef)`, so it serves production code while its binding
|
|
3639
|
+
// names the version it was pushed from. #1962 heals one on its next `scope bind`; this is
|
|
3640
|
+
// the pass over the previews nobody binds again. It is the SAME act a bind runs, with the
|
|
3641
|
+
// version the preview is already bound to as its target: carry the data off the serving
|
|
3642
|
+
// script into that version's own script, bind it, then clear the pin. A pin that fails to
|
|
3643
|
+
// clear leaves the route where the data still is, and the next pass carries again.
|
|
3644
|
+
//
|
|
3645
|
+
// Staff only, by default-deny: the path is on neither `BUILDER_ROUTES` nor
|
|
3646
|
+
// `TENANT_ROUTES`, so a builder's push token and a tenant credential both 403 before the
|
|
3647
|
+
// handler runs. (A fleet-wide walk is the wrong shape for either, even a tenant's own.)
|
|
3648
|
+
//
|
|
3649
|
+
// Paged and resumable like every fleet walk here: a pass reads one page of ACTIVE
|
|
3650
|
+
// directory rows after `cursor` and repairs at most `limit` previews from it, answering
|
|
3651
|
+
// `nextCursor` until the fleet is exhausted. Bounded by `limit` because each repair is a
|
|
3652
|
+
// data move between two Durable Objects. A scope that fails is reported and the pass
|
|
3653
|
+
// moves on, so one bad preview cannot hold the rest; it is still pinned, so the next walk
|
|
3654
|
+
// from the start visits it again. A healed preview is no longer a candidate, which is
|
|
3655
|
+
// what makes a re-run a no-op. `dryRun` lists the candidates and touches nothing.
|
|
3656
|
+
const REPAIR_SCAN_PAGE = 500;
|
|
3657
|
+
const repairServingPinsBody = z
|
|
3658
|
+
.object({
|
|
3659
|
+
cursor: scopeIdSchema.optional(),
|
|
3660
|
+
limit: z.number().int().min(1).max(100).default(25),
|
|
3661
|
+
dryRun: z.boolean().default(false),
|
|
3662
|
+
})
|
|
3663
|
+
.strict();
|
|
3664
|
+
const repairPreviewServingPin = async (c, scope) => {
|
|
3665
|
+
const actor = c.get('actor');
|
|
3666
|
+
const versionId = scope.verticalVersionId;
|
|
3667
|
+
if (!scope.vertical || !versionId)
|
|
3668
|
+
return { skipped: 'the preview is bound to no version' };
|
|
3669
|
+
// Without its own script the version has nowhere to receive the data, and clearing the pin
|
|
3670
|
+
// would strand it on the serving script.
|
|
3671
|
+
const bound = await c.var.admin.getVersion(actor, versionId, scope.vertical);
|
|
3672
|
+
if (!bound?.deploymentRef)
|
|
3673
|
+
return { skipped: `version ${versionId} has no script of its own` };
|
|
3674
|
+
// The carry and the tail a legacy preview's `scope bind` runs too. The bind expects the
|
|
3675
|
+
// version this pass read, so a push that re-pointed the preview since it was listed wins.
|
|
3676
|
+
const recarry = () => carryOntoVersion(c, scope, versionId, { dropServingRef: true });
|
|
3677
|
+
const carried = await recarry();
|
|
3678
|
+
await bindAfterCarry(c, scope, versionId, carried, { clearServingRef: true, recarry });
|
|
3679
|
+
return { repaired: { from: carried?.from ?? scope.servingRef, to: bound.deploymentRef, tables: carried?.tables ?? 0 } };
|
|
3680
|
+
};
|
|
3681
|
+
app.post('/previews/repair-serving-pins', async (c) => {
|
|
3682
|
+
if (!options.resolveVerticalVersion || !options.resolveVerticalRef) {
|
|
3683
|
+
return c.json({ error: 'repairing serving pins needs dispatch resolution for both ends' }, 501);
|
|
3684
|
+
}
|
|
3685
|
+
const { cursor, limit, dryRun } = await readJsonBody(c, repairServingPinsBody);
|
|
3686
|
+
const actor = c.get('actor');
|
|
3687
|
+
const page = await c.var.admin.listScopes(actor, { status: ['active'], cursor, limit: REPAIR_SCAN_PAGE });
|
|
3688
|
+
const repaired = [];
|
|
3689
|
+
const candidates = [];
|
|
3690
|
+
const skipped = [];
|
|
3691
|
+
const failed = [];
|
|
3692
|
+
let last = cursor;
|
|
3693
|
+
let handled = 0;
|
|
3694
|
+
let stopped = false;
|
|
3695
|
+
for (const scope of page) {
|
|
3696
|
+
if (scope.kind !== 'preview' || !scope.servingRef) {
|
|
3697
|
+
last = scope.id;
|
|
3698
|
+
continue;
|
|
3699
|
+
}
|
|
3700
|
+
if (handled >= limit) {
|
|
3701
|
+
stopped = true;
|
|
3702
|
+
break;
|
|
3703
|
+
}
|
|
3704
|
+
handled += 1;
|
|
3705
|
+
last = scope.id;
|
|
3706
|
+
const at = { tenantId: scope.tenantId, scopeId: scope.id };
|
|
3707
|
+
if (dryRun) {
|
|
3708
|
+
candidates.push({ ...at, servingRef: scope.servingRef });
|
|
3709
|
+
continue;
|
|
3710
|
+
}
|
|
3711
|
+
try {
|
|
3712
|
+
const outcome = await repairPreviewServingPin(c, scope);
|
|
3713
|
+
if ('skipped' in outcome)
|
|
3714
|
+
skipped.push({ ...at, reason: outcome.skipped });
|
|
3715
|
+
else
|
|
3716
|
+
repaired.push({ ...at, ...outcome.repaired });
|
|
3717
|
+
}
|
|
3718
|
+
catch (e) {
|
|
3719
|
+
// The status the app's error boundary would have answered, so a host refusal (a push
|
|
3720
|
+
// re-pointed the preview since it was listed) reads as a 412 and not as a platform fault.
|
|
3721
|
+
const { status } = mapError(e);
|
|
3722
|
+
const message = e instanceof Error ? e.message : String(e);
|
|
3723
|
+
if (status >= 500 && status !== 501) {
|
|
3724
|
+
recordFailure({
|
|
3725
|
+
actor,
|
|
3726
|
+
operation: 'preview.repair-serving-pin',
|
|
3727
|
+
stage: 'repair',
|
|
3728
|
+
tenantId: scope.tenantId,
|
|
3729
|
+
scopeId: scope.id,
|
|
3730
|
+
vertical: scope.vertical,
|
|
3731
|
+
version: scope.verticalVersionId,
|
|
3732
|
+
status,
|
|
3733
|
+
message,
|
|
3734
|
+
}, e);
|
|
3735
|
+
}
|
|
3736
|
+
failed.push({ ...at, status, error: message });
|
|
3737
|
+
}
|
|
3738
|
+
}
|
|
3739
|
+
// Exhausted only when the page ran short: a full page may be followed by more rows.
|
|
3740
|
+
const nextCursor = stopped || page.length === REPAIR_SCAN_PAGE ? (last ?? null) : null;
|
|
3741
|
+
return c.json({ dryRun, repaired, candidates, skipped, failed, nextCursor });
|
|
3742
|
+
});
|
|
3743
|
+
const markScopeIfCopy = async (c, scope) => {
|
|
3744
|
+
const actor = c.get('actor');
|
|
3745
|
+
if (isPrimaryScope(scope))
|
|
3746
|
+
return { skip: 'primary: an install is never marked' };
|
|
3747
|
+
if (!scope.vertical)
|
|
3748
|
+
return { skip: 'bound to no vertical: no module code runs, so nothing reads the marker' };
|
|
3749
|
+
const vertical = await verticalForScope(c, scope);
|
|
3750
|
+
if (!vertical) {
|
|
3751
|
+
const hosted = scope.servingRef !== null && scope.servingRef !== undefined
|
|
3752
|
+
? true
|
|
3753
|
+
: scope.verticalVersionId
|
|
3754
|
+
? Boolean((await c.var.admin.getVersion(actor, scope.verticalVersionId, scope.vertical))?.deploymentRef)
|
|
3755
|
+
: false;
|
|
3756
|
+
return hosted
|
|
3757
|
+
? { fail: { status: 503, error: 'no deployment resolves for this hosted scope, so its copy marker was not written' } }
|
|
3758
|
+
: { skip: 'co-located: the host that runs this scope reads the directory, so it needs no marker' };
|
|
3759
|
+
}
|
|
3760
|
+
const { marked } = await vertical.markCopy(scope.id, { kind: scope.kind, forkedFrom: scope.forkedFrom });
|
|
3761
|
+
await c.var.admin.recordCopyMark(actor, {
|
|
3762
|
+
action: 'mark',
|
|
3763
|
+
tenantId: scope.tenantId,
|
|
3764
|
+
scopeId: scope.id,
|
|
3765
|
+
outcome: marked ? 'marked' : 'already',
|
|
3766
|
+
});
|
|
3767
|
+
return { done: marked ? 'marked' : 'already' };
|
|
3768
|
+
};
|
|
3769
|
+
// -- one-time repair: mark every existing copy as one, in its own storage (#2005) --------
|
|
3770
|
+
// A CP-less hosted vertical has no directory, so it reads a scope's own copy-origin row to
|
|
3771
|
+
// hold a preview's or a fork's executors inert. Every copy carries that row since #2005, and
|
|
3772
|
+
// every carry onto a non-primary scope stamps it (`restoreCarryingSwitches`) — but a copy
|
|
3773
|
+
// made before either, never carried since, holds none and would read as an install there.
|
|
3774
|
+
// This is the pass over those: for each scope the directory says is not primary, ask the
|
|
3775
|
+
// vertical that holds it to stamp the marker (idempotent there), and log the outcome. Active,
|
|
3776
|
+
// suspended and archived scopes alike: a suspended or archived copy can be reactivated, and a
|
|
3777
|
+
// reactivation stamps too (`markScopeIfCopy` in the lifecycle routes), so neither path brings
|
|
3778
|
+
// an unmarked copy back to life. A hosted scope that does not resolve is FAILED, not skipped.
|
|
3779
|
+
//
|
|
3780
|
+
// Staff only, by default-deny, like the serving-pin repair: the path is on neither
|
|
3781
|
+
// `BUILDER_ROUTES` nor `TENANT_ROUTES`. Paged and resumable the same way — one page of
|
|
3782
|
+
// ACTIVE directory rows after `cursor`, at most `limit` scopes marked from it, `nextCursor`
|
|
3783
|
+
// until the fleet is exhausted. A scope that fails is reported and the pass moves on; it is
|
|
3784
|
+
// still unmarked, so the next walk visits it again, and a marked one answers `already`.
|
|
3785
|
+
// `dryRun` lists the candidates and touches nothing. An install is never a candidate.
|
|
3786
|
+
const markCopiesBody = z
|
|
3787
|
+
.object({
|
|
3788
|
+
cursor: scopeIdSchema.optional(),
|
|
3789
|
+
limit: z.number().int().min(1).max(200).default(50),
|
|
3790
|
+
dryRun: z.boolean().default(false),
|
|
3791
|
+
})
|
|
3792
|
+
.strict();
|
|
3793
|
+
app.post('/scopes/mark-copies', async (c) => {
|
|
3794
|
+
const { cursor, limit, dryRun } = await readJsonBody(c, markCopiesBody);
|
|
3795
|
+
const actor = c.get('actor');
|
|
3796
|
+
const page = await c.var.admin.listScopes(actor, {
|
|
3797
|
+
status: ['active', 'suspended', 'archived'],
|
|
3798
|
+
cursor,
|
|
3799
|
+
limit: REPAIR_SCAN_PAGE,
|
|
3800
|
+
});
|
|
3801
|
+
const marked = [];
|
|
3802
|
+
const already = [];
|
|
3803
|
+
const candidates = [];
|
|
3804
|
+
const skipped = [];
|
|
3805
|
+
const failed = [];
|
|
3806
|
+
let last = cursor;
|
|
3807
|
+
let handled = 0;
|
|
3808
|
+
let stopped = false;
|
|
3809
|
+
for (const scope of page) {
|
|
3810
|
+
if (isPrimaryScope(scope)) {
|
|
3811
|
+
last = scope.id;
|
|
3812
|
+
continue;
|
|
3813
|
+
}
|
|
3814
|
+
if (handled >= limit) {
|
|
3815
|
+
stopped = true;
|
|
3816
|
+
break;
|
|
3817
|
+
}
|
|
3818
|
+
handled += 1;
|
|
3819
|
+
last = scope.id;
|
|
3820
|
+
const at = { tenantId: scope.tenantId, scopeId: scope.id };
|
|
3821
|
+
if (dryRun) {
|
|
3822
|
+
candidates.push(at);
|
|
3823
|
+
continue;
|
|
3824
|
+
}
|
|
3825
|
+
try {
|
|
3826
|
+
const step = await markScopeIfCopy(c, scope);
|
|
3827
|
+
if ('skip' in step)
|
|
3828
|
+
skipped.push({ ...at, reason: step.skip });
|
|
3829
|
+
else if ('fail' in step)
|
|
3830
|
+
failed.push({ ...at, ...step.fail });
|
|
3831
|
+
else
|
|
3832
|
+
(step.done === 'marked' ? marked : already).push(at);
|
|
3833
|
+
}
|
|
3834
|
+
catch (e) {
|
|
3835
|
+
const { status } = mapError(e);
|
|
3836
|
+
failed.push({ ...at, status, error: e instanceof Error ? e.message : String(e) });
|
|
3837
|
+
}
|
|
3838
|
+
}
|
|
3839
|
+
const nextCursor = stopped || page.length === REPAIR_SCAN_PAGE ? (last ?? null) : null;
|
|
3840
|
+
return c.json({ dryRun, marked, already, candidates, skipped, failed, nextCursor });
|
|
3841
|
+
});
|
|
3842
|
+
// The correction for a MISTAKEN copy marker (#2005): a scope the directory says IS primary,
|
|
3843
|
+
// marked by a misclassification, a race or an operator, or restored from another scope's backup
|
|
3844
|
+
// before #2009 split the two facts, whose executors a CP-less host would otherwise hold inert
|
|
3845
|
+
// for good. One scope at a time, staff only by default-deny (the path is on neither allowlist),
|
|
3846
|
+
// logged as `clearScopeCopyMark`. Refused for a scope the directory calls a copy. Clears the
|
|
3847
|
+
// classification only: the events a load brought in stay marked, so they never run here.
|
|
3848
|
+
app.post('/tenants/:tenantId/scopes/:scopeId/clear-copy-mark', async (c) => {
|
|
3849
|
+
const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
|
|
3850
|
+
const scopeId = scopeIdSchema.parse(c.req.param('scopeId'));
|
|
3851
|
+
const actor = c.get('actor');
|
|
3852
|
+
const scope = await c.var.admin.getScopeRecord(actor, tenantId, scopeId);
|
|
3853
|
+
if (!scope)
|
|
3854
|
+
return c.json({ error: `unknown scope for tenant: (${tenantId}, ${scopeId})` }, 404);
|
|
3855
|
+
if (!isPrimaryScope(scope)) {
|
|
3856
|
+
return c.json({ error: 'clear-copy-mark refused: the directory says this scope is a copy (a preview or a fork)' }, 409);
|
|
3857
|
+
}
|
|
3858
|
+
const vertical = await verticalForScope(c, scope);
|
|
3859
|
+
if (!vertical)
|
|
3860
|
+
return c.json({ error: await diagnoseUnboundScope(actor, scope) }, 501);
|
|
3861
|
+
const { cleared } = await vertical.clearCopyMark(scopeId, { kind: scope.kind, forkedFrom: scope.forkedFrom });
|
|
3862
|
+
await c.var.admin.recordCopyMark(actor, { action: 'clear', tenantId, scopeId, outcome: cleared ? 'cleared' : 'absent' });
|
|
3863
|
+
return c.json({ cleared });
|
|
3864
|
+
});
|
|
3163
3865
|
// Reap an ARCHIVED primary scope (control-plane.md §4.4): free its DO storage —
|
|
3164
3866
|
// Cloudflare never garbage-collects a Durable Object, so a deleted app's bytes persist
|
|
3165
3867
|
// forever otherwise — while keeping the directory row as a tombstone. A POST verb, not
|
|
@@ -3260,7 +3962,7 @@ export function createControlPlaneApi(options) {
|
|
|
3260
3962
|
const actor = c.get('actor');
|
|
3261
3963
|
// Body is optional so the bare `POST …/reap` every existing caller sends still
|
|
3262
3964
|
// parses; `backup` tri-states on purpose (see the ordering comment below).
|
|
3263
|
-
const { backup: wantsBackup } =
|
|
3965
|
+
const { backup: wantsBackup } = await readJsonBody(c, reapScopeBody);
|
|
3264
3966
|
const scope = await c.var.admin.getScopeRecord(actor, tenantId, scopeId);
|
|
3265
3967
|
if (!scope)
|
|
3266
3968
|
return c.json({ error: `unknown scope for tenant: (${tenantId}, ${scopeId})` }, 404);
|
|
@@ -3672,7 +4374,7 @@ export function createControlPlaneApi(options) {
|
|
|
3672
4374
|
return c.json({ error: 'adopt-serving is a private-vertical operation' }, 409);
|
|
3673
4375
|
}
|
|
3674
4376
|
const owned = (await c.var.admin.listScopes(actor, { tenantId: v.ownerTenant, vertical: slug })).filter((s) => !s.forkedFrom && s.kind !== 'preview' && s.status === 'active');
|
|
3675
|
-
const { acknowledge } =
|
|
4377
|
+
const { acknowledge } = await readJsonBody(c, adoptVerticalServingBody);
|
|
3676
4378
|
const adopted = [];
|
|
3677
4379
|
const alreadyAdopted = [];
|
|
3678
4380
|
try {
|
|
@@ -3732,6 +4434,226 @@ export function createControlPlaneApi(options) {
|
|
|
3732
4434
|
const versionId = z.string().min(1).parse(c.req.query('versionId'));
|
|
3733
4435
|
return c.json({ affected: await c.var.admin.bindingImpact(c.get('actor'), tenantId, scopeId, versionId) });
|
|
3734
4436
|
});
|
|
4437
|
+
// -- kept copies (#1722, Codex #2008 r7) ----------------------------------------------------
|
|
4438
|
+
// A kept copy is one a carry's fenced wipe refused because it took a write the carry never
|
|
4439
|
+
// copied (a request still routed to the old script before the bind). The store protects it:
|
|
4440
|
+
// every load into it answers 409 until it is resolved here. The two resolutions both lose
|
|
4441
|
+
// something, so each is named and acknowledged, never defaulted:
|
|
4442
|
+
// - `discard` wipes the kept copy: what reached it after the carry's export is gone.
|
|
4443
|
+
// - `restore-forward` restores the kept copy over the store the scope now routes to, then
|
|
4444
|
+
// wipes the kept copy: what the live store took since the copy was kept is gone.
|
|
4445
|
+
// There is no merge of two scope dumps. To reconcile by hand, pull the kept copy with
|
|
4446
|
+
// `GET …/kept-copy/export` and the live store with the governed export (`GET …/export`, which
|
|
4447
|
+
// reads the scope's route), re-apply what matters, then discard.
|
|
4448
|
+
//
|
|
4449
|
+
// Staff only, by default-deny: neither path is on BUILDER_ROUTES or TENANT_ROUTES. One scope
|
|
4450
|
+
// at a time, refused unless the copy really is kept, and admin-logged with the write revision
|
|
4451
|
+
// before and after (`recordKeptCopyResolution`).
|
|
4452
|
+
const keptScriptQuery = z.object({ script: z.string().min(1) }).strict();
|
|
4453
|
+
const resolveKeptBody = z.discriminatedUnion('action', [
|
|
4454
|
+
z
|
|
4455
|
+
.object({
|
|
4456
|
+
script: z.string().min(1),
|
|
4457
|
+
action: z.literal('discard'),
|
|
4458
|
+
acknowledge: z.object({ discard: z.literal(true) }).strict(),
|
|
4459
|
+
})
|
|
4460
|
+
.strict(),
|
|
4461
|
+
z
|
|
4462
|
+
.object({
|
|
4463
|
+
script: z.string().min(1),
|
|
4464
|
+
action: z.literal('release'),
|
|
4465
|
+
acknowledge: z.object({ release: z.literal(true) }).strict(),
|
|
4466
|
+
})
|
|
4467
|
+
.strict(),
|
|
4468
|
+
z
|
|
4469
|
+
.object({
|
|
4470
|
+
script: z.string().min(1),
|
|
4471
|
+
action: z.literal('restore-forward'),
|
|
4472
|
+
/** The moment the copy was kept, as its marker says: the live store's writes since are replaced. */
|
|
4473
|
+
acknowledge: z.object({ replacesLiveWritesSince: z.string().min(1) }).strict(),
|
|
4474
|
+
})
|
|
4475
|
+
.strict(),
|
|
4476
|
+
]);
|
|
4477
|
+
const keptHolder = async (script) => {
|
|
4478
|
+
if (!options.resolveVerticalRef)
|
|
4479
|
+
throw new ControlPlaneError(501, 'kept copies need dispatch resolution by script');
|
|
4480
|
+
const holder = await options.resolveVerticalRef(script);
|
|
4481
|
+
if (!holder)
|
|
4482
|
+
throw new ControlPlaneError(404, `no deployment resolves for script '${script}'`);
|
|
4483
|
+
return holder;
|
|
4484
|
+
};
|
|
4485
|
+
app.get('/tenants/:tenantId/scopes/:scopeId/kept-copy', async (c) => {
|
|
4486
|
+
const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
|
|
4487
|
+
const scopeId = scopeIdSchema.parse(c.req.param('scopeId'));
|
|
4488
|
+
const { script } = keptScriptQuery.parse(c.req.query());
|
|
4489
|
+
const scope = await c.var.admin.getScopeRecord(c.get('actor'), tenantId, scopeId);
|
|
4490
|
+
if (!scope)
|
|
4491
|
+
return c.json({ error: `unknown scope for tenant: (${tenantId}, ${scopeId})` }, 404);
|
|
4492
|
+
const kept = await (await keptHolder(script)).keptCopy(scopeId);
|
|
4493
|
+
return c.json({ scopeId, script, kept, routesTo: await routeOf(c, scope) });
|
|
4494
|
+
});
|
|
4495
|
+
// The kept copy's own dump, for reconciling by hand before a discard: the governed export's
|
|
4496
|
+
// rules exactly (`GET …/export` above). Refused outside `global` residency, masked unless
|
|
4497
|
+
// `full=true`, and K-24 access-logged through the same canonical `exportScope`. It reads the
|
|
4498
|
+
// script named, refused unless the copy there is kept, so it never becomes a way to pull a
|
|
4499
|
+
// live store past that route.
|
|
4500
|
+
const keptExportQuery = z
|
|
4501
|
+
.object({ script: z.string().min(1), full: z.enum(['true', 'false']).optional() })
|
|
4502
|
+
.strict();
|
|
4503
|
+
app.get('/tenants/:tenantId/scopes/:scopeId/kept-copy/export', async (c) => {
|
|
4504
|
+
const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
|
|
4505
|
+
const scopeId = scopeIdSchema.parse(c.req.param('scopeId'));
|
|
4506
|
+
const { script, full } = keptExportQuery.parse(c.req.query());
|
|
4507
|
+
const actor = c.get('actor');
|
|
4508
|
+
const scope = await c.var.admin.getScopeRecord(actor, tenantId, scopeId);
|
|
4509
|
+
if (!scope)
|
|
4510
|
+
return c.json({ error: `unknown scope for tenant: (${tenantId}, ${scopeId})` }, 404);
|
|
4511
|
+
if (scope.jurisdiction !== 'global') {
|
|
4512
|
+
return c.json({
|
|
4513
|
+
error: `scope ${scopeId} is pinned to '${scope.jurisdiction}' — a local pull would ` +
|
|
4514
|
+
`move its data outside that jurisdiction; refused (K-32, preview-and-snapshots.md §6)`,
|
|
4515
|
+
}, 403);
|
|
4516
|
+
}
|
|
4517
|
+
const holder = await keptHolder(script);
|
|
4518
|
+
const kept = await holder.keptCopy(scopeId);
|
|
4519
|
+
if (!kept)
|
|
4520
|
+
return c.json({ error: `scope ${scopeId} has no kept copy in '${script}'` }, 409);
|
|
4521
|
+
const dump = await c.var.admin.exportScope(actor, tenantId, scopeId);
|
|
4522
|
+
const tables = await holder.exportScope(scopeId);
|
|
4523
|
+
if (full === 'true')
|
|
4524
|
+
return c.json({ ...dump, tables, masked: false, script, kept });
|
|
4525
|
+
const mask = await createPseudonymizer(options.maskSalt ?? crypto.randomUUID());
|
|
4526
|
+
return c.json({ ...dump, tables: await maskDump(tables, mask), masked: true, script, kept });
|
|
4527
|
+
});
|
|
4528
|
+
app.post('/tenants/:tenantId/scopes/:scopeId/kept-copy/resolve', async (c) => {
|
|
4529
|
+
const tenantId = tenantIdSchema.parse(c.req.param('tenantId'));
|
|
4530
|
+
const scopeId = scopeIdSchema.parse(c.req.param('scopeId'));
|
|
4531
|
+
const body = await readJsonBody(c, resolveKeptBody);
|
|
4532
|
+
const actor = c.get('actor');
|
|
4533
|
+
const scope = await c.var.admin.getScopeRecord(actor, tenantId, scopeId);
|
|
4534
|
+
if (!scope)
|
|
4535
|
+
return c.json({ error: `unknown scope for tenant: (${tenantId}, ${scopeId})` }, 404);
|
|
4536
|
+
const holder = await keptHolder(body.script);
|
|
4537
|
+
const kept = await holder.keptCopy(scopeId);
|
|
4538
|
+
if (!kept)
|
|
4539
|
+
return c.json({ error: `scope ${scopeId} has no kept copy in '${body.script}'` }, 409);
|
|
4540
|
+
const route = await routeOf(c, scope);
|
|
4541
|
+
// `release` is for a kept copy the scope routes to (a rollback bound it after a carry kept
|
|
4542
|
+
// it); the other two act on a kept copy the scope has left, against the store it runs on.
|
|
4543
|
+
if (body.action === 'release' ? route !== body.script : !route || route === body.script) {
|
|
4544
|
+
return c.json({
|
|
4545
|
+
error: body.action === 'release'
|
|
4546
|
+
? `scope ${scopeId} routes to '${route ?? '(nothing)'}', not to the kept copy in '${body.script}'; release is for a kept copy that is live`
|
|
4547
|
+
: `scope ${scopeId} routes to '${route ?? '(nothing)'}', so there is no live copy to resolve against`,
|
|
4548
|
+
}, 409);
|
|
4549
|
+
}
|
|
4550
|
+
const at = new Date().toISOString();
|
|
4551
|
+
const revisionOf = async (client) => {
|
|
4552
|
+
const marker = await client.loadMarker(scopeId);
|
|
4553
|
+
return marker === 'unfenced' ? null : marker.revision;
|
|
4554
|
+
};
|
|
4555
|
+
// The kept copy as it stands now, which is what this resolution acts on and is fenced on: a
|
|
4556
|
+
// write that reaches it after this read refuses the wipe. (The marker's own `revision` is the
|
|
4557
|
+
// one it was kept at; writing the marker, and any later write, moved it on.)
|
|
4558
|
+
const before = await holder.loadMarker(scopeId);
|
|
4559
|
+
const revisionBefore = before === 'unfenced' ? null : before.revision;
|
|
4560
|
+
// Codex #2008 r13: and the load stamp with it, so the release or discard refuses a store a load
|
|
4561
|
+
// has replaced since this read (a kept copy refuses every load but its own resolution, so this
|
|
4562
|
+
// is the same fence stated where the act is, not a new one).
|
|
4563
|
+
const stampFence = before === 'unfenced' ? {} : { loadStamp: before.loadStamp };
|
|
4564
|
+
// #2005 (Codex #2008 r10): whatever the resolution leaves in that store, it is a copy when
|
|
4565
|
+
// the directory says the scope is not primary, marker or no marker.
|
|
4566
|
+
const lineage = copyLineageOf(scope);
|
|
4567
|
+
const markCopy = lineage ? { markCopy: lineage } : {};
|
|
4568
|
+
const record = (action, revisionAfter) => c.var.admin.recordKeptCopyResolution(actor, tenantId, scopeId, {
|
|
4569
|
+
action,
|
|
4570
|
+
script: body.script,
|
|
4571
|
+
liveScript: action === 'restore-forward' ? route : null,
|
|
4572
|
+
keptAt: kept.keptAt,
|
|
4573
|
+
revisionBefore,
|
|
4574
|
+
revisionAfter,
|
|
4575
|
+
});
|
|
4576
|
+
if (body.action === 'release') {
|
|
4577
|
+
const out = await holder.releaseKeptCopy({ scopeId, revision: revisionBefore, ...markCopy, ...stampFence });
|
|
4578
|
+
if ('refused' in out) {
|
|
4579
|
+
return c.json({ error: `the kept copy of scope ${scopeId} ${out.refused === 'changed' ? 'changed since it was read' : 'is no longer kept'}; nothing was released` }, 412);
|
|
4580
|
+
}
|
|
4581
|
+
await record('release', revisionBefore);
|
|
4582
|
+
return c.json({ scopeId, script: body.script, action: 'release', revisionBefore });
|
|
4583
|
+
}
|
|
4584
|
+
if (body.action === 'discard') {
|
|
4585
|
+
const out = await holder.discardKeptCopy({ scopeId, revision: revisionBefore, carriedTo: route, at, ...markCopy, ...stampFence });
|
|
4586
|
+
if ('refused' in out) {
|
|
4587
|
+
return c.json({ error: `the kept copy of scope ${scopeId} ${out.refused === 'changed' ? 'changed since it was read' : 'is no longer kept'}; nothing was discarded` }, 412);
|
|
4588
|
+
}
|
|
4589
|
+
const revisionAfter = await revisionOf(holder);
|
|
4590
|
+
await record('discard', revisionAfter);
|
|
4591
|
+
return c.json({ scopeId, script: body.script, action: 'discard', revisionBefore, revisionAfter });
|
|
4592
|
+
}
|
|
4593
|
+
// restore-forward: the acknowledgement names what is replaced, as the marker says it.
|
|
4594
|
+
if (body.acknowledge.replacesLiveWritesSince !== kept.keptAt) {
|
|
4595
|
+
return c.json({
|
|
4596
|
+
error: `a restore forward replaces what the live store took since ${kept.keptAt}; acknowledge exactly that ` +
|
|
4597
|
+
`(acknowledge.replacesLiveWritesSince: '${kept.keptAt}')`,
|
|
4598
|
+
}, 409);
|
|
4599
|
+
}
|
|
4600
|
+
// Fenced on the binding through the whole resolution (Codex #2008 r8): the kept copy goes only
|
|
4601
|
+
// once its data is in the store the scope still routes to. Every refusal below leaves it kept.
|
|
4602
|
+
const liveScript = route;
|
|
4603
|
+
const live = await keptHolder(liveScript);
|
|
4604
|
+
const bindingHeld = async () => {
|
|
4605
|
+
const now = await c.var.admin.getScopeRecord(actor, tenantId, scopeId);
|
|
4606
|
+
return (Boolean(now) &&
|
|
4607
|
+
(now.verticalVersionId ?? null) === (scope.verticalVersionId ?? null) &&
|
|
4608
|
+
(now.servingRef ?? null) === (scope.servingRef ?? null));
|
|
4609
|
+
};
|
|
4610
|
+
const moved = () => c.json({
|
|
4611
|
+
error: `scope ${scopeId} was re-pointed while the kept copy was being restored forward, so nothing was resolved ` +
|
|
4612
|
+
`and the kept copy in '${body.script}' stays kept; read it again and resolve it again`,
|
|
4613
|
+
}, 409);
|
|
4614
|
+
// The live store as read now, and nothing else, is what gets replaced. The binding read again
|
|
4615
|
+
// after the marker ties the marker to the live store: a move that landed before this read
|
|
4616
|
+
// (its wipe minting a tombstone here) changed the binding, and is refused.
|
|
4617
|
+
const liveMarker = await live.loadMarker(scopeId);
|
|
4618
|
+
if (liveMarker === 'unfenced') {
|
|
4619
|
+
return c.json({ error: `the deployment serving scope ${scopeId} ('${liveScript}') predates the fence (#1722); re-push it, then resolve` }, 409);
|
|
4620
|
+
}
|
|
4621
|
+
if (!(await bindingHeld()))
|
|
4622
|
+
return moved();
|
|
4623
|
+
if ((await isCarriedAway(live, scope)) || (await live.keptCopy(scopeId))) {
|
|
4624
|
+
return c.json({ error: `the store scope ${scopeId} routes to in '${liveScript}' is not a live store (wiped or kept); nothing was resolved` }, 409);
|
|
4625
|
+
}
|
|
4626
|
+
const dump = await retryTransient(() => holder.exportScope(scopeId));
|
|
4627
|
+
const restored = await restoreCarryingSwitches(actor, live, tenantId, scopeId, dump, {
|
|
4628
|
+
scopeId,
|
|
4629
|
+
exact: true,
|
|
4630
|
+
loadStamp: ulid(),
|
|
4631
|
+
expect: liveMarker,
|
|
4632
|
+
});
|
|
4633
|
+
await c.var.admin.reassertSystemSwitches(actor, { tenantId, scopeId }, { appliedInUnit: restored.switchedOff });
|
|
4634
|
+
// Still routed there once the data landed? If not, the move that re-pointed the scope keeps
|
|
4635
|
+
// what this restore put in that store (its source wipe protects a copy changed since its
|
|
4636
|
+
// export), and the kept copy here stays kept too: nothing is lost, nothing is discarded.
|
|
4637
|
+
if (!(await bindingHeld()))
|
|
4638
|
+
return moved();
|
|
4639
|
+
const revisionAfter = await revisionOf(live);
|
|
4640
|
+
await record('restore-forward', revisionAfter);
|
|
4641
|
+
const wiped = await holder.discardKeptCopy({
|
|
4642
|
+
scopeId,
|
|
4643
|
+
revision: revisionBefore,
|
|
4644
|
+
carriedTo: liveScript,
|
|
4645
|
+
at,
|
|
4646
|
+
...markCopy,
|
|
4647
|
+
...stampFence,
|
|
4648
|
+
});
|
|
4649
|
+
if ('refused' in wiped) {
|
|
4650
|
+
return c.json({
|
|
4651
|
+
error: `restored forward into '${route}', but the kept copy in '${body.script}' changed since it was read, so it ` +
|
|
4652
|
+
`stays kept: read it again and resolve it again`,
|
|
4653
|
+
}, 409);
|
|
4654
|
+
}
|
|
4655
|
+
return c.json({ scopeId, script: body.script, action: 'restore-forward', liveScript, revisionBefore, revisionAfter });
|
|
4656
|
+
});
|
|
3735
4657
|
// Pin a scope to a vertical version (#31; orchestration.md §4). Refuses a
|
|
3736
4658
|
// non-admitted version below the seam — that refusal is the registry's reason to
|
|
3737
4659
|
// exist. A scope operation, so it keeps the scope route shape. `snapshot: true`
|
|
@@ -3768,23 +4690,21 @@ export function createControlPlaneApi(options) {
|
|
|
3768
4690
|
if (scope)
|
|
3769
4691
|
carried = await carryOntoVersion(c, scope, versionId, { dropServingRef: repairPreviewPin });
|
|
3770
4692
|
};
|
|
3771
|
-
// #1674: after the bind, the scope may route to a different store (the carry's
|
|
3772
|
-
// destination); put the directory's recorded OFF positions back there. #1742: the carry
|
|
3773
|
-
// already did so in the restore's own event; what it reports is audited here.
|
|
3774
|
-
const reassert = async () => {
|
|
3775
|
-
if (scope) {
|
|
3776
|
-
await c.var.admin.reassertSystemSwitches(actor, { tenantId, scopeId }, { appliedInUnit: carried?.switchedOff });
|
|
3777
|
-
}
|
|
3778
|
-
};
|
|
3779
4693
|
// The host's own refusal, should the impact have changed since it was read above (a
|
|
3780
|
-
// promote landing in between): relayed as the same 409, never as a 500.
|
|
4694
|
+
// promote landing in between): relayed as the same 409, never as a 500. A known scope
|
|
4695
|
+
// binds through the carry tail (#1722): compare-and-set on the binding read above, the pin
|
|
4696
|
+
// cleared after it, the OFF positions re-asserted where it now routes (#1674), and the copy
|
|
4697
|
+
// the carry left in the old script wiped.
|
|
3781
4698
|
const bind = async (opts) => {
|
|
3782
|
-
|
|
3783
|
-
|
|
3784
|
-
|
|
3785
|
-
if (repairPreviewPin) {
|
|
3786
|
-
await c.var.admin.setScopeServingRef(actor, tenantId, scopeId, null).catch(relayHostRefusal);
|
|
4699
|
+
if (!scope) {
|
|
4700
|
+
await c.var.admin.bindScopeVersion(actor, tenantId, scopeId, versionId, opts).catch(relayHostRefusal);
|
|
4701
|
+
return;
|
|
3787
4702
|
}
|
|
4703
|
+
await bindAfterCarry(c, scope, versionId, carried, {
|
|
4704
|
+
...opts,
|
|
4705
|
+
clearServingRef: repairPreviewPin,
|
|
4706
|
+
recarry: () => carryOntoVersion(c, scope, versionId, { dropServingRef: repairPreviewPin }),
|
|
4707
|
+
});
|
|
3788
4708
|
};
|
|
3789
4709
|
if (snapshot) {
|
|
3790
4710
|
if (!scope) {
|
|
@@ -3804,10 +4724,18 @@ export function createControlPlaneApi(options) {
|
|
|
3804
4724
|
}
|
|
3805
4725
|
await carry();
|
|
3806
4726
|
if (migrationCrossing) {
|
|
3807
|
-
|
|
3808
|
-
|
|
3809
|
-
|
|
3810
|
-
|
|
4727
|
+
try {
|
|
4728
|
+
// Asked again after the carry: a promote landing since the first question would
|
|
4729
|
+
// otherwise leave an archive behind for a bind the host then refuses.
|
|
4730
|
+
await refuseOnBreaks(actor, tenantId, scopeId, versionId, { acknowledge });
|
|
4731
|
+
await orchestratedSnapshot(c, tenantId, scope, {});
|
|
4732
|
+
}
|
|
4733
|
+
catch (e) {
|
|
4734
|
+
// #1722: no bind follows, so the copy the carry restored is reached by nothing.
|
|
4735
|
+
if (carried)
|
|
4736
|
+
await dropUnboundCopy(c, scope, versionId, carried);
|
|
4737
|
+
throw e;
|
|
4738
|
+
}
|
|
3811
4739
|
}
|
|
3812
4740
|
await bind({ acknowledge });
|
|
3813
4741
|
}
|
|
@@ -3815,12 +4743,10 @@ export function createControlPlaneApi(options) {
|
|
|
3815
4743
|
await carry();
|
|
3816
4744
|
await bind({ acknowledge, snapshot: true });
|
|
3817
4745
|
}
|
|
3818
|
-
await reassert();
|
|
3819
4746
|
return c.json(await c.var.admin.getScopeRecord(actor, tenantId, scopeId));
|
|
3820
4747
|
}
|
|
3821
4748
|
await carry();
|
|
3822
4749
|
await bind({ acknowledge });
|
|
3823
|
-
await reassert();
|
|
3824
4750
|
return c.json(await c.var.admin.getScopeRecord(actor, tenantId, scopeId));
|
|
3825
4751
|
});
|
|
3826
4752
|
// -- instances (K-31) -------------------------------------------------------
|
|
@@ -4514,6 +5440,11 @@ export function createControlPlaneApi(options) {
|
|
|
4514
5440
|
throw new ControlPlaneError(502, `version ${versionId} retained no manifest — pushed pre-#286; push it again to serve in place`);
|
|
4515
5441
|
}
|
|
4516
5442
|
const manifest = storedDeployManifest.parse(JSON.parse(manifestJson));
|
|
5443
|
+
// #1902: the classes this upload DEPLOYS — the manifest's, plus the platform's sweeper when
|
|
5444
|
+
// the version's recorded decision supplies it. The upload, the migration delta and the
|
|
5445
|
+
// serving record below all read this one set; the manifest's own `doClasses` would leave
|
|
5446
|
+
// the record a class short of the script, and the next serve re-declaring a live class.
|
|
5447
|
+
const doClasses = deployedDoClasses(manifest.doClasses, manifest.platformSweeper);
|
|
4517
5448
|
const serving = await host.admin.verticalServing(actor, slug);
|
|
4518
5449
|
const ref = serving?.ref ?? stableDeploymentRefFor(slug);
|
|
4519
5450
|
const modules = await options.fetchVerticalModules(version.deploymentRef);
|
|
@@ -4537,7 +5468,7 @@ export function createControlPlaneApi(options) {
|
|
|
4537
5468
|
compatibilityDate: manifest.compatibilityDate,
|
|
4538
5469
|
compatibilityFlags: manifest.compatibilityFlags,
|
|
4539
5470
|
modules,
|
|
4540
|
-
doClasses
|
|
5471
|
+
doClasses,
|
|
4541
5472
|
bindings: [...manifest.bindings, ...storeBindings],
|
|
4542
5473
|
// #1242: the version now being served, injected as `SUBSTRAT_VERSION_ID` so the
|
|
4543
5474
|
// scope host stamps the signals `version` dimension. The serving script is one
|
|
@@ -4547,6 +5478,10 @@ export function createControlPlaneApi(options) {
|
|
|
4547
5478
|
// #1054: the model runtime is bound only for a version that ASKED for it, so the
|
|
4548
5479
|
// capability is visible in the manifest diff at admit rather than fleet-wide.
|
|
4549
5480
|
...(manifest.usesModels ? { usesModels: true } : {}),
|
|
5481
|
+
// #1902: the sweeper decision the PUSH made and recorded, reused — never re-made, so a
|
|
5482
|
+
// promote or a backout cannot refuse a version that was accepted. A version pushed
|
|
5483
|
+
// before the record existed has none and keeps what it had: no platform sweeper.
|
|
5484
|
+
...(manifest.platformSweeper ? { supplySweeper: true } : {}),
|
|
4550
5485
|
// #340/#578: the version's static files travel with it onto the serving script —
|
|
4551
5486
|
// from the RETAINED manifest, with no bytes. The runtime's asset store dedupes
|
|
4552
5487
|
// per SCRIPT (not namespace-wide — the #578 finding), so the serving script only
|
|
@@ -4570,18 +5505,14 @@ export function createControlPlaneApi(options) {
|
|
|
4570
5505
|
}, serving
|
|
4571
5506
|
? { priorDoClasses: serving.doClasses, priorMigrationTag: serving.migrationTag }
|
|
4572
5507
|
: undefined);
|
|
4573
|
-
const addedClasses = serving
|
|
4574
|
-
? manifest.doClasses.some((cls) => !serving.doClasses.includes(cls))
|
|
4575
|
-
: false;
|
|
5508
|
+
const addedClasses = serving ? doClasses.some((cls) => !serving.doClasses.includes(cls)) : false;
|
|
4576
5509
|
await admin.setVerticalServing(actor, slug, {
|
|
4577
5510
|
ref,
|
|
4578
5511
|
versionId,
|
|
4579
5512
|
// The serving script's class set only ever GROWS (DO classes cannot be deleted
|
|
4580
5513
|
// while their storage lives), so record the union, and the tag only moves when
|
|
4581
5514
|
// a migration actually rode the upload.
|
|
4582
|
-
doClasses: serving
|
|
4583
|
-
? [...new Set([...serving.doClasses, ...manifest.doClasses])]
|
|
4584
|
-
: manifest.doClasses,
|
|
5515
|
+
doClasses: serving ? [...new Set([...serving.doClasses, ...doClasses])] : doClasses,
|
|
4585
5516
|
migrationTag: serving
|
|
4586
5517
|
? addedClasses
|
|
4587
5518
|
? nextMigrationTag(serving.migrationTag)
|
|
@@ -5038,6 +5969,13 @@ export function createControlPlaneApi(options) {
|
|
|
5038
5969
|
error: `${assetParts.size} uploaded asset part(s) are absent from the manifest (${extra}${assetParts.size > 5 ? ', …' : ''})`,
|
|
5039
5970
|
}, 400);
|
|
5040
5971
|
}
|
|
5972
|
+
// #1902: whether this version's uploads carry the platform's scope sweeper — decided here,
|
|
5973
|
+
// ONCE, before anything is uploaded, and recorded with the version below so every later
|
|
5974
|
+
// upload of it reuses the answer. A declaration that contradicts itself is refused now,
|
|
5975
|
+
// while it is still a push; it can never become a promote or a backout that refuses.
|
|
5976
|
+
const sweeper = platformSweeperDecision(manifest, { entry: manifest.entry, modules });
|
|
5977
|
+
if ('refuse' in sweeper)
|
|
5978
|
+
return c.json({ error: sweeper.refuse }, 422);
|
|
5041
5979
|
// Mint the version id first: the deploymentRef (the dispatch script name) is keyed
|
|
5042
5980
|
// on it, so it is CF-valid and unique per version.
|
|
5043
5981
|
const id = ulid();
|
|
@@ -5053,6 +5991,8 @@ export function createControlPlaneApi(options) {
|
|
|
5053
5991
|
// #1242: the freshly minted version id — this archive script IS that version,
|
|
5054
5992
|
// and a preview routed at it stamps the signals `version` dimension honestly.
|
|
5055
5993
|
versionId: id,
|
|
5994
|
+
// #1902: the platform's scope sweeper, as decided above and recorded with the version.
|
|
5995
|
+
...(sweeper.supply ? { supplySweeper: true } : {}),
|
|
5056
5996
|
// #340: the verified bytes go up with the bundle. The manifest's routing config
|
|
5057
5997
|
// rides along untouched — it decides what the RUNTIME does with paths, and carries
|
|
5058
5998
|
// no reach, so there is nothing in it for the sandbox contract to refuse.
|
|
@@ -5132,7 +6072,8 @@ export function createControlPlaneApi(options) {
|
|
|
5132
6072
|
...(origin ? { origin } : {}),
|
|
5133
6073
|
// Retained for the serving upload (#286): the archive script keeps the module
|
|
5134
6074
|
// bytes, this keeps their shape (entry, compat, doClasses, bindings).
|
|
5135
|
-
|
|
6075
|
+
// #1902: with the sweeper decision the push made, which no later upload re-makes.
|
|
6076
|
+
manifestJson: JSON.stringify({ ...manifest, platformSweeper: sweeper.supply }),
|
|
5136
6077
|
});
|
|
5137
6078
|
// Same spirit as the permission-surface gate, advisory tier: when the push DECLARES
|
|
5138
6079
|
// surfaces, name any surface that hostnames are still bound to but the declaration
|
|
@@ -6080,13 +7021,15 @@ export function createControlPlaneApi(options) {
|
|
|
6080
7021
|
}
|
|
6081
7022
|
throw e;
|
|
6082
7023
|
}
|
|
6083
|
-
//
|
|
6084
|
-
//
|
|
6085
|
-
|
|
6086
|
-
|
|
6087
|
-
|
|
6088
|
-
|
|
6089
|
-
|
|
7024
|
+
// The carry tail (#1722): the bind expects the version the carry exported from, so of two
|
|
7025
|
+
// pushes that read the same binding only one lands. Then a preview provisioned before
|
|
7026
|
+
// #527 has its inherited serving_ref cleared, so routing follows the bound version (its
|
|
7027
|
+
// per-version script) and not the prod serving script; the OFF positions are re-asserted
|
|
7028
|
+
// there (#1674); and the copy left in the old script is wiped.
|
|
7029
|
+
await bindAfterCarry(c, existing, opts.versionId, carried, {
|
|
7030
|
+
clearServingRef: true,
|
|
7031
|
+
recarry: () => carryOntoVersion(c, existing, opts.versionId, { dropServingRef: true }),
|
|
7032
|
+
});
|
|
6090
7033
|
// Renew (or clear) the preview's GC deadline so a reused preview does not silently die.
|
|
6091
7034
|
await c.var.admin.setScopeExpiresAt(actor, tenantId, existing.id, expiresAt);
|
|
6092
7035
|
const hostname = await bindPreviewHostname(c.var.admin, actor, baseHostname, tenantId, existing.id, opts.tag, surface);
|
|
@@ -6158,6 +7101,11 @@ export function createControlPlaneApi(options) {
|
|
|
6158
7101
|
};
|
|
6159
7102
|
let row;
|
|
6160
7103
|
let restore;
|
|
7104
|
+
/** #2009: the directory's classification of the row below, which every restore of it carries. */
|
|
7105
|
+
const markCopyOfRow = () => {
|
|
7106
|
+
const lineage = copyLineageOf({ kind: row.kind ?? '', forkedFrom: row.forkedFrom ?? null });
|
|
7107
|
+
return lineage ? { markCopy: lineage } : {};
|
|
7108
|
+
};
|
|
6161
7109
|
if (source) {
|
|
6162
7110
|
// A fresh fork. Export from where the prod data lives TODAY. The canonical
|
|
6163
7111
|
// `admin.exportScope` first — it writes the K-24 audit entry (and the co-located
|
|
@@ -6188,7 +7136,9 @@ export function createControlPlaneApi(options) {
|
|
|
6188
7136
|
// there; restore re-projects the vertical's roles from the dump's tuples). A one-shot
|
|
6189
7137
|
// DO storage blip heals on the in-request retry WITHOUT burning a CI attempt (which
|
|
6190
7138
|
// pushes a fresh version per try) — #559 (2).
|
|
6191
|
-
|
|
7139
|
+
// #2009: a load classifies nothing by itself, so the restore carries the directory's
|
|
7140
|
+
// classification of the row below, as every restore onto a non-primary scope does.
|
|
7141
|
+
restore = () => target.restoreScope(tenantId, previewId, tables, { sourceScopeId: source.id, exact: true, ...markCopyOfRow() });
|
|
6192
7142
|
}
|
|
6193
7143
|
else {
|
|
6194
7144
|
// A clean-room preview (#509 (b)): an EMPTY scope, no source to export. No `forkedFrom`
|
|
@@ -6206,7 +7156,8 @@ export function createControlPlaneApi(options) {
|
|
|
6206
7156
|
jurisdiction: 'global',
|
|
6207
7157
|
expiresAt: expiresAt ?? undefined,
|
|
6208
7158
|
};
|
|
6209
|
-
|
|
7159
|
+
// #2009: marked a copy on the directory's word, as the fork above.
|
|
7160
|
+
restore = target ? () => target.restoreScope(tenantId, previewId, [], markCopyOfRow()) : null;
|
|
6210
7161
|
}
|
|
6211
7162
|
try {
|
|
6212
7163
|
// Inside the try: a host writes the directory row FIRST and then migrates, projects and
|