@substrat-run/adapter-cloudflare 0.136.0 → 0.138.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/checker.d.ts +7 -1
- package/dist/checker.d.ts.map +1 -1
- package/dist/checker.js +5 -3
- package/dist/checker.js.map +1 -1
- package/dist/control-plane-do.d.ts +123 -52
- package/dist/control-plane-do.d.ts.map +1 -1
- package/dist/control-plane-do.js +321 -73
- package/dist/control-plane-do.js.map +1 -1
- package/dist/do-reply.d.ts +25 -0
- package/dist/do-reply.d.ts.map +1 -0
- package/dist/do-reply.js +17 -0
- package/dist/do-reply.js.map +1 -0
- package/dist/host.d.ts +221 -16
- package/dist/host.d.ts.map +1 -1
- package/dist/host.js +1073 -401
- package/dist/host.js.map +1 -1
- package/dist/live-reads.d.ts +39 -1
- package/dist/live-reads.d.ts.map +1 -1
- package/dist/live-reads.js +79 -0
- package/dist/live-reads.js.map +1 -1
- package/dist/scope-do.d.ts +4 -7
- package/dist/scope-do.d.ts.map +1 -1
- package/dist/scope-do.js +595 -134
- package/dist/scope-do.js.map +1 -1
- package/dist/system-door.d.ts +21 -0
- package/dist/system-door.d.ts.map +1 -0
- package/dist/system-door.js +13 -0
- package/dist/system-door.js.map +1 -0
- package/package.json +11 -11
package/dist/host.js
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import { env as ambientEnv } from 'cloudflare:workers';
|
|
2
2
|
import { isRewindRefusal, REWIND_REFUSED } from './rewind-refusal.js';
|
|
3
|
-
import {
|
|
3
|
+
import { isSystemDoorMoved, SYSTEM_DOOR_MOVED, SYSTEM_DOOR_REGATES } from './system-door.js';
|
|
4
|
+
import { delegatedReadParams, operationSeriesCount, fromWireFailure, exportReadInput, exportedBatch, importBatch, importResult, importState, importsOfManifestJson, exportsOfManifestJson, importCursorMove, importCursorMoveAt, importCursorMoved, accessLogEntry, adminLogEntry, opsFailureEntry, opsFailureFingerprint, issueEntry, sweepRunEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunsPayload, modelUsageEntry, attachmentRecord, createTenantInput, identityLink, identityPool, createOrgInput, promotionAcknowledgement, bindAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, registerVerticalInput, vertical as verticalSchema, verticalServingState, verticalChannel, verticalVersion, connection, capabilityExchange, mintedCapability, capabilityGrant, principalId, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, systemSwitch, systemSwitchOutcome, systemScheduleEntry, systemSwitchRecord, entitlementGrant, entitlementGrantInput, capabilityFilter, instant, meterReading, subjectRef, createConnectionInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, org as orgSchema, orgMembership, resolvedIdentity, identityMembership, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, delegatedReadRecord, ownerTransferAudit, copyMarkAudit, subjectShredReceipt, connectorDispatchKind, callsOfManifestJson, outboundOfManifestJson, SCOPE_GATE_REASONS, substratError, redrainEventsInput, coverage, peerCoverage, peerGrantsEntry, peerSwitch, peerSwitchOutcome, verticalCaller, verticalResolution, verticalSlug, entityObjectRef, ATTACHMENT_BLOB_BINDING, blobStoreBindingName, lifecycleDelivery, scopeLifecycle, } from '@substrat-run/contracts';
|
|
4
5
|
import { normalizeHostname, toRouteTarget } from './route-resolver.js';
|
|
5
|
-
import { attachmentBlobKey, attachmentSha256, entitlementDenial, requiredEntitlementFor, foldMeterReading, parseValidationRecords, resolveScopeRecord, ulid, KEPT_COPY_REFUSAL, capabilityTokenHash, checkBecomeInput, plausibleSessionToken, foldModelUsage, backoffAt, resolveRetryPolicy, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, openSealed, ImpersonationRefused, assertSessionUsable, impersonationRowValues, mapImpersonationRow, newImpersonationSession, jobRunOf, runDueJobRuns, startJobRun, attachmentTextJob, assertAttachmentExtractors, assertJobRegistrable, isAttachmentTextRun, searchLimit, searchMatchExpression, globalFetch, assertRedrainWindow, platformRequestOf, undrainedEventsOf, inUnitMovesToAudit, staleCarryRevertRow, staleCarryReverts, withRecorded, systemSwitchedOffMessage, tenantSystemSwitchedOffMessage, CrossVerticalRegistry, importCursorSourceOf, exportBreaksOf, exportBreakRefusal, bindExportBreaksOf, bindExportBreakRefusal, collectPeers, peerSeats, connectorCallRecord, noopConnectorCallRecorder, recordConnectorCall, settleConnectionUse, splitManifestMigrations, unknownRoleError, assertRowLimit, assertRowOffset, INERT_SCOPE_REASON, lifecycleReceipt, lifecycleRefusal, isPrimaryScope, isPrimaryScopeRow, asyncInvocationId, asyncLinePass, } from '@substrat-run/kernel';
|
|
6
|
+
import { attachmentBlobKey, attachmentSha256, entitlementDenial, requiredEntitlementFor, foldMeterReading, parseValidationRecords, resolveScopeRecord, ulid, KEPT_COPY_REFUSAL, capabilityTokenHash, checkBecomeInput, plausibleSessionToken, foldModelUsage, backoffAt, resolveRetryPolicy, executorOutcomeOf, isDeliveryRefusal, refusalJournalText, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, openSealed, ImpersonationRefused, assertSessionUsable, impersonationRowValues, mapImpersonationRow, newImpersonationSession, jobRunOf, assertLeaseMs, runDueJobRuns, SYSTEM_DOOR_WAIT, startJobRun, attachmentTextJob, assertAttachmentExtractors, assertJobRegistrable, isAttachmentTextRun, searchLimit, searchMatchExpression, globalFetch, assertRedrainWindow, platformRequestOf, undrainedEventsOf, inUnitMovesToAudit, staleCarryRevertRow, staleCarryReverts, recordedOffFromWire, peerSubjectRef, SWITCH_KINDS, reassertActionOf, reassertEntry, reassertOffRow, reassertOnRow, switchActionOf, switchAuditSubject, switchNotFoundMessage, recordWriteSuperseded, switchSubjectOf, switchSupersededMessage, withRecorded, systemSwitchedOffMessage, tenantSystemSwitchedOffMessage, CrossVerticalRegistry, importCursorSourceOf, exportBreaksOf, exportBreakRefusal, bindExportBreaksOf, bindExportBreakRefusal, collectPeers, peerSeats, connectorCallRecord, noopConnectorCallRecorder, recordConnectorCall, settleConnectionUse, splitManifestMigrations, unknownRoleError, assertRowLimit, assertRowOffset, emptyImportResult, INERT_SCOPE_REASON, lifecycleReceipt, lifecycleRefusal, isPrimaryScope, isPrimaryScopeRow, asyncInvocationId, asyncLinePass, } from '@substrat-run/kernel';
|
|
6
7
|
import { attributedHost } from '@substrat-run/kernel';
|
|
7
|
-
import { isOrangeToOrange, isUpgradeRequest, LIVE_MODE_HEADER, LIVE_PRINCIPAL_HEADER, LIVE_SCOPE_HEADER, LIVE_SUBSCRIBE_PATH, LIVE_TENANT_HEADER, } from './live-reads.js';
|
|
8
|
+
import { isOrangeToOrange, isUpgradeRequest, LIVE_MODE_HEADER, LIVE_PRINCIPAL_HEADER, LIVE_SCOPE_HEADER, LIVE_SUBSCRIBE_PATH, LIVE_TENANT_HEADER, LIVE_WITHIN_HEADER, encodeLiveWithin, liveWithinOf, } from './live-reads.js';
|
|
8
9
|
import { tenantStoreDatabaseName } from './d1.js';
|
|
9
10
|
import { blobStoreBucketName, r2TenantBlobStore } from './r2.js';
|
|
11
|
+
import { unwrapReply } from './do-reply.js';
|
|
10
12
|
/** DO row → contract shape. Never reads the secrets table — that is the split. */
|
|
11
13
|
const toConnection = (r) => connection.parse({
|
|
12
14
|
id: r.id,
|
|
@@ -25,18 +27,34 @@ const toConnection = (r) => connection.parse({
|
|
|
25
27
|
createdAt: r.created_at,
|
|
26
28
|
revokedAt: r.revoked_at,
|
|
27
29
|
});
|
|
28
|
-
/** Rethrow a capability attachment verb's failure, rebuilt with its code; else its value. */
|
|
29
|
-
function unwrapCapabilityReply(reply) {
|
|
30
|
-
if (reply.failure)
|
|
31
|
-
throw fromWireFailure(reply.failure);
|
|
32
|
-
return reply.value;
|
|
33
|
-
}
|
|
34
30
|
/**
|
|
35
31
|
* How far ahead of the clock a held lifecycle epoch may be and still be learned past (#1713). An
|
|
36
32
|
* epoch is the time a directory restore minted it, so a legitimate one is behind its minter's
|
|
37
33
|
* clock; the skew allows for clocks that disagree, and bounds what a forged answer can move.
|
|
38
34
|
*/
|
|
39
35
|
export const LIFECYCLE_EPOCH_SKEW_MS = 24 * 60 * 60 * 1000;
|
|
36
|
+
/**
|
|
37
|
+
* #2016: how many served scopes one heal pass asks for their tenant record, beyond the drift and the
|
|
38
|
+
* holds. Rotated least-recently-asked first, so a deployment that cannot answer yet (a build from
|
|
39
|
+
* before the record) costs at most this many deliveries a pass and starves nothing.
|
|
40
|
+
*/
|
|
41
|
+
export const TENANT_UNRECORDED_PER_PASS = 50;
|
|
42
|
+
/**
|
|
43
|
+
* K-3's refusal of a (tenant, scope) pair the host does not hold — the same answer for a scope that
|
|
44
|
+
* does not exist and for one of another tenant (#1718), in one wording.
|
|
45
|
+
*/
|
|
46
|
+
const unknownScopeForTenant = (tenantId, scopeId) => substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
|
|
47
|
+
/**
|
|
48
|
+
* #2016: the line a CP-less host writes each time it admits a pair to a scope with no tenant record
|
|
49
|
+
* and no role rows — nothing to hold the pair against. Ids only.
|
|
50
|
+
*/
|
|
51
|
+
export const tenantUnrecordedLine = (tenantId, scopeId) => ({
|
|
52
|
+
substrat: 'tenant-unrecorded',
|
|
53
|
+
level: 'warn',
|
|
54
|
+
tenantId,
|
|
55
|
+
scopeId,
|
|
56
|
+
message: 'admitted to a scope whose tenant is not recorded; a reconcile or lifecycle delivery records it',
|
|
57
|
+
});
|
|
40
58
|
/**
|
|
41
59
|
* Refuse to mark a scope a copy unless the request classifies it non-primary (#2005). The
|
|
42
60
|
* classification is the platform directory's; the vertical applies the same `isPrimaryScope` to
|
|
@@ -51,9 +69,10 @@ function assertCopyLineage(lineage) {
|
|
|
51
69
|
* A control-plane stand-in for a CP-less vertical (scope-local-permissions.md Phase 3).
|
|
52
70
|
* The hot path a served scope actually touches becomes trust-the-upstream:
|
|
53
71
|
* - `scopeAccessRefusal` / `setMigrationState` → no-op: the router gates the scope's
|
|
54
|
-
* lifecycle + tenancy from the shared directory, and the vertical
|
|
55
|
-
*
|
|
56
|
-
*
|
|
72
|
+
* lifecycle + tenancy from the shared directory, and the vertical does not re-read a
|
|
73
|
+
* directory it does not have. Its own half of both gates reads the scope's storage
|
|
74
|
+
* instead (`validateScopeAccess`): the tenant the scope was provisioned for (#2016) and the lifecycle
|
|
75
|
+
* the platform delivered (#1713).
|
|
57
76
|
* - `tenantHoldsEntitlement` → true: the SKU was enforced on the shared control plane
|
|
58
77
|
* at provision (before `provisionInstance`), so a scope that EXISTS here was granted
|
|
59
78
|
* it upstream — a single-vertical deployment holds its own entitlements by construction.
|
|
@@ -68,22 +87,31 @@ function nullControlPlane() {
|
|
|
68
87
|
recordAdmin: noop,
|
|
69
88
|
recordAccess: noop,
|
|
70
89
|
tenantHoldsEntitlement: async () => true,
|
|
71
|
-
// #1674: the
|
|
90
|
+
// #1674, #2029: the switches' record is a DIRECTORY store, and a CP-less host has no
|
|
72
91
|
// directory — the shared control plane that delegates here keeps it. Same posture as
|
|
73
92
|
// `recordAdmin`: nothing to write, nothing recorded to read back.
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
93
|
+
recordSwitchedOff: async () => null,
|
|
94
|
+
recordSwitchedOn: async () => null,
|
|
95
|
+
restoreSwitchRecord: noop,
|
|
77
96
|
listSystemSwitches: async () => [],
|
|
78
|
-
|
|
79
|
-
|
|
97
|
+
switchRecordsOf: async () => [],
|
|
98
|
+
switchRecordStatesOf: async () => [],
|
|
99
|
+
switchesOwedOf: async () => [],
|
|
100
|
+
clearSwitchOwed: noop,
|
|
101
|
+
// #1823: no tenant tuples here either — the platform reads them and sends `tenantHeld`.
|
|
102
|
+
tenantHeldOf: async () => [],
|
|
103
|
+
};
|
|
104
|
+
const unavailable = (method) => new Error(`control plane unavailable: '${method}' — this host is scope-local / CP-less ` +
|
|
105
|
+
`(docs/architecture/scope-local-permissions.md, Phase 3)`);
|
|
106
|
+
// A directory write through `reply` (#113) is named for the write it carries, not for `reply`.
|
|
107
|
+
passthrough.reply = async (method) => {
|
|
108
|
+
throw unavailable(String(method));
|
|
80
109
|
};
|
|
81
110
|
return new Proxy({}, {
|
|
82
111
|
get: (_t, prop) => typeof prop === 'string' && prop in passthrough
|
|
83
112
|
? passthrough[prop]
|
|
84
113
|
: async () => {
|
|
85
|
-
throw
|
|
86
|
-
`(docs/architecture/scope-local-permissions.md, Phase 3)`);
|
|
114
|
+
throw unavailable(String(prop));
|
|
87
115
|
},
|
|
88
116
|
});
|
|
89
117
|
}
|
|
@@ -124,6 +152,35 @@ export const SWITCH_HOLD_EXTRA_WAITS = 3;
|
|
|
124
152
|
* clock, against the releasing host's clock: minutes, not seconds.
|
|
125
153
|
*/
|
|
126
154
|
export const SWITCH_HOLD_PENDING_MAX_MS = 5 * 60_000;
|
|
155
|
+
/**
|
|
156
|
+
* A move the deployment serving the scope applied, or may have, without attesting what the
|
|
157
|
+
* platform needs from it (#2045 Codex r3): the switch fence (`fenced`), or for a tenant-held
|
|
158
|
+
* module's OFF the evaluator that denies a tenant-level grant (`deniesTenantGrants`, #1823).
|
|
159
|
+
*
|
|
160
|
+
* The preflight (`switchTarget`'s `attestFence`) tells a deployment built before the fence apart
|
|
161
|
+
* before anything is recorded or moved. On such a deployment only an ON is refused: the race the
|
|
162
|
+
* fence closes is an older OFF landing after a newer ON, and with every ON refused there is no
|
|
163
|
+
* newer ON to land after, so an OFF goes through unfenced (#2045 follow-up). A kill switch that
|
|
164
|
+
* cannot be pulled until every vertical is re-pushed is the outage the fence must never cause.
|
|
165
|
+
* An unfenced OFF keeps its write-ahead owed mark, so the first re-assert after the vertical is
|
|
166
|
+
* redeployed moves the scope again under the record's fence.
|
|
167
|
+
*
|
|
168
|
+
* This error fires when a deployment that attested the fence answers a move without it — a
|
|
169
|
+
* rollback to an old build between the preflight and the move — or when an ON reaches a
|
|
170
|
+
* deployment without the fence. It is NOT compensated with an opposite move: an unfenced
|
|
171
|
+
* compensation can undo a newer call's switch (Codex r3, finding 2). The call's record and its
|
|
172
|
+
* write-ahead owed mark stay, and the call is refused.
|
|
173
|
+
*/
|
|
174
|
+
class UnattestedSwitch extends Error {
|
|
175
|
+
}
|
|
176
|
+
/** A fenced move answered without `fenced`: the deployment predates the fence. */
|
|
177
|
+
const unfencedMessage = (scopeId) => `the deployment serving scope ${scopeId} predates the switch fence (#2045): it cannot refuse an older ` +
|
|
178
|
+
`switch call's OFF landing after this ON, so no switch is turned on there. Switching off still works. ` +
|
|
179
|
+
`Redeploy the vertical, then retry.`;
|
|
180
|
+
/** A tenant-held module's OFF answered without `deniesTenantGrants` (#1823). */
|
|
181
|
+
const undeniedMessage = (scopeId, moduleId) => `the deployment serving scope ${scopeId} predates the kill switch's tenant-grant denial (#1823): ` +
|
|
182
|
+
`module '${moduleId}' holds a tenant-level grant that its OFF would leave authorizing there. ` +
|
|
183
|
+
`Redeploy the vertical, then retry.`;
|
|
127
184
|
/**
|
|
128
185
|
* The attachment bucket a host resolves when it was handed no resolver (#1995): the
|
|
129
186
|
* platform's per-tenant binding, read off this script's own env, named by the one shared
|
|
@@ -132,6 +189,49 @@ export const SWITCH_HOLD_PENDING_MAX_MS = 5 * 60_000;
|
|
|
132
189
|
function ambientAttachmentBucket(tenantId) {
|
|
133
190
|
return ambientEnv[blobStoreBindingName(ATTACHMENT_BLOB_BINDING, tenantId)] ?? null;
|
|
134
191
|
}
|
|
192
|
+
/**
|
|
193
|
+
* #2029: scope instances whose door has read the rewind hold and found NO claim on the scope at
|
|
194
|
+
* all, so their doors skip the hold read for the rest of the instance's life. Isolate-wide, because
|
|
195
|
+
* a host lives for one request: without it, every peer call would read the deployment's one hold
|
|
196
|
+
* object — an extra hop, on a single object, on the hot path of every scope never rewound.
|
|
197
|
+
*
|
|
198
|
+
* Safe because claims are only ever added for storage a LATER instance serves. A rewind writes its
|
|
199
|
+
* claim before it arms, and arming restarts the scope, so its rewound storage is always served by a
|
|
200
|
+
* new instance with a new id, which reads the hold afresh (`SWITCH_HOLD_SETTLE_MS` puts every row of
|
|
201
|
+
* the claim before that instance existed). A join (`switchHoldJoin`) goes into a rewind's claim too,
|
|
202
|
+
* so into that same later instance. An instance that read no claim therefore never gains one; claims
|
|
203
|
+
* only leave it, by a switch move. Only a GOOD read is remembered: a read that failed open is not.
|
|
204
|
+
* Bounded: past `CLEAR_SCOPE_INSTANCES_MAX` it is emptied, which costs a re-read, nothing else.
|
|
205
|
+
*/
|
|
206
|
+
const CLEAR_SCOPE_INSTANCES = new Set();
|
|
207
|
+
const CLEAR_SCOPE_INSTANCES_MAX = 10_000;
|
|
208
|
+
const rememberClearInstance = (key) => {
|
|
209
|
+
if (CLEAR_SCOPE_INSTANCES.size >= CLEAR_SCOPE_INSTANCES_MAX)
|
|
210
|
+
CLEAR_SCOPE_INSTANCES.clear();
|
|
211
|
+
CLEAR_SCOPE_INSTANCES.add(key);
|
|
212
|
+
};
|
|
213
|
+
/**
|
|
214
|
+
* #2045: a carry's fences, by tuple subject (`RecordedOffCarry.fences`): for each recorded-off
|
|
215
|
+
* subject, the operation id of the call the record holds.
|
|
216
|
+
*/
|
|
217
|
+
const switchFencesCarried = (kinds) => {
|
|
218
|
+
const out = {};
|
|
219
|
+
for (const [kind, { keys, fences }] of kinds) {
|
|
220
|
+
for (const key of keys) {
|
|
221
|
+
const fence = fences.get(key);
|
|
222
|
+
if (fence !== undefined)
|
|
223
|
+
out[switchSubjectOf(kind, key)] = fence;
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
return out;
|
|
227
|
+
};
|
|
228
|
+
/** #2029: how a door's refusals and a re-assert's name each kind of subject. */
|
|
229
|
+
const DOOR_WORDS = {
|
|
230
|
+
system: { what: (key) => `module '${key}'`, door: 'system door', switchName: 'schedule', switched: 'schedules' },
|
|
231
|
+
peer: { what: (key) => `peer vertical '${key}'`, door: 'peer door', switchName: 'peer', switched: 'peers' },
|
|
232
|
+
};
|
|
233
|
+
/** #2029: the rewind hold's key for one subject — a module's id, bare as #1819 stored it, or `vertical:<slug>`. */
|
|
234
|
+
const holdKeyOf = (kind, key) => (kind === 'system' ? key : peerSubjectRef(key));
|
|
135
235
|
export class CloudflareScopeHost {
|
|
136
236
|
admin;
|
|
137
237
|
scopeNs;
|
|
@@ -349,6 +449,22 @@ export class CloudflareScopeHost {
|
|
|
349
449
|
},
|
|
350
450
|
};
|
|
351
451
|
}
|
|
452
|
+
/**
|
|
453
|
+
* #1184: the scope reads an executor's handler may make. Plain RPCs to the ScopeDO: the
|
|
454
|
+
* handler runs here on the coordinator, after the DO's own task has returned.
|
|
455
|
+
*/
|
|
456
|
+
executorScope(tenantId, scopeId) {
|
|
457
|
+
const stub = this.scopeStub(scopeId);
|
|
458
|
+
return {
|
|
459
|
+
history: async (entity, page) => stub.entityHistory({ entityType: entity.entityType, entityId: entity.entityId, limit: page?.limit, cursor: page?.cursor }),
|
|
460
|
+
covers: async (principal, roleKey, level) => {
|
|
461
|
+
const bound = await stub.canAssignFor(tenantId, scopeId, principalId.parse(principal), roleKey, level === 'tenant');
|
|
462
|
+
if (!bound)
|
|
463
|
+
throw unknownRoleError(roleKey);
|
|
464
|
+
return coverage.parse(bound);
|
|
465
|
+
},
|
|
466
|
+
};
|
|
467
|
+
}
|
|
352
468
|
/**
|
|
353
469
|
* Drain this scope's outbox into the registered executors (K-22 §4.2).
|
|
354
470
|
*
|
|
@@ -372,7 +488,9 @@ export class CloudflareScopeHost {
|
|
|
372
488
|
* own `invocationId` field reads null — an ambient read there would record "no
|
|
373
489
|
* call" for every attempt an operation's own tail made.
|
|
374
490
|
*/
|
|
375
|
-
invocationId
|
|
491
|
+
invocationId,
|
|
492
|
+
/** #1184: what each attempt did, for the emitting call's `onExecutorOutcomes`. */
|
|
493
|
+
outcomes) {
|
|
376
494
|
const report = {
|
|
377
495
|
attempted: 0,
|
|
378
496
|
delivered: 0,
|
|
@@ -386,27 +504,22 @@ export class CloudflareScopeHost {
|
|
|
386
504
|
// attempt runs — the DO only journals it.
|
|
387
505
|
const lines = asyncLinePass();
|
|
388
506
|
try {
|
|
389
|
-
await this.drainExecutorsPass(tenantId, scopeId, invocationId, report, lines);
|
|
507
|
+
await this.drainExecutorsPass(tenantId, scopeId, invocationId, report, lines, outcomes);
|
|
390
508
|
}
|
|
391
509
|
finally {
|
|
392
510
|
lines.end();
|
|
393
511
|
}
|
|
394
512
|
return report;
|
|
395
513
|
}
|
|
396
|
-
async drainExecutorsPass(tenantId, scopeId, invocationId, report, lines) {
|
|
514
|
+
async drainExecutorsPass(tenantId, scopeId, invocationId, report, lines, outcomes) {
|
|
397
515
|
const stub = this.scopeStub(scopeId);
|
|
516
|
+
const scope = this.executorScope(tenantId, scopeId);
|
|
398
517
|
// #2005: a non-primary scope (a fork, a snapshot, a preview of either kind) causes no
|
|
399
518
|
// outbound effects, so its deliveries are journaled terminal with the reason and no
|
|
400
|
-
// handler runs. Asked
|
|
401
|
-
//
|
|
402
|
-
// `is_copy`, #2009, set on the directory's word) — a preview and a snapshot reach a hosted
|
|
403
|
-
// vertical as restores the platform marks.
|
|
404
|
-
// Asked on the first due event, once per pass: most passes have none, and the directory
|
|
405
|
-
// is one global object.
|
|
519
|
+
// handler runs (`isInertScope`). Asked on the first due event, once per pass: most passes
|
|
520
|
+
// have none, and the directory is one global object.
|
|
406
521
|
let inert;
|
|
407
|
-
const isInert = () => (inert ??= this.
|
|
408
|
-
? stub.isCopy()
|
|
409
|
-
: this.cp.getScopeRecord(tenantId, scopeId).then((row) => !isPrimaryScopeRow(row)));
|
|
522
|
+
const isInert = () => (inert ??= this.isInertScope(tenantId, scopeId));
|
|
410
523
|
// #1713: a CP-less scope its lifecycle holds attempts nothing and journals nothing, so every
|
|
411
524
|
// due delivery stays due for the first pass after it is live again. Asked on the first due
|
|
412
525
|
// event, like `isInert`. Every caller passed `assertLive` already, so this is for the scope
|
|
@@ -437,6 +550,9 @@ export class CloudflareScopeHost {
|
|
|
437
550
|
attempt,
|
|
438
551
|
startedAt: Date.now(),
|
|
439
552
|
});
|
|
553
|
+
const outcomeOf = (event, outcome, error) => {
|
|
554
|
+
outcomes?.push(executorOutcomeOf(id, event, outcome, error));
|
|
555
|
+
};
|
|
440
556
|
for (const bad of undecodable) {
|
|
441
557
|
report.attempted += 1;
|
|
442
558
|
const attempt = await stub.recordExecutorAttempt(bad.eventId, deliveryId, bad.error, null, invocationId);
|
|
@@ -450,6 +566,7 @@ export class CloudflareScopeHost {
|
|
|
450
566
|
// No next attempt, like an undecodable row: a scope does not become primary.
|
|
451
567
|
const attempt = await stub.recordExecutorAttempt(event.id, deliveryId, INERT_SCOPE_REASON, null, invocationId);
|
|
452
568
|
report.inert = (report.inert ?? 0) + 1;
|
|
569
|
+
outcomeOf(event, 'inert', INERT_SCOPE_REASON);
|
|
453
570
|
// Its own outcome, never `delivered`: no handler ran (#2005).
|
|
454
571
|
lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'inert' });
|
|
455
572
|
continue;
|
|
@@ -467,6 +584,7 @@ export class CloudflareScopeHost {
|
|
|
467
584
|
// the at-least-once residue, as it already must in-process.
|
|
468
585
|
await stub.routeExecutorEventToPlatform(event.id, deliveryId, connectorDispatchKind(executor.provider), JSON.stringify({ executorId: id, event }), JSON.stringify({ system: 'connector-dispatch' }), invocationId);
|
|
469
586
|
report.routedToPlatform += 1;
|
|
587
|
+
outcomeOf(event, 'routed');
|
|
470
588
|
// Handed on, not run here: the platform's drain owns its attempts from now on.
|
|
471
589
|
lines.write({ ...unitOf(event.id, 1), startedAt, outcome: 'routed' });
|
|
472
590
|
}
|
|
@@ -474,13 +592,26 @@ export class CloudflareScopeHost {
|
|
|
474
592
|
await executor.handler(await this.connectorContext(tenantId, scopeId, executor.timeoutMs, event.id), event);
|
|
475
593
|
const attempt = await stub.recordExecutorAttempt(event.id, deliveryId, null, null, invocationId);
|
|
476
594
|
report.delivered += 1;
|
|
595
|
+
outcomeOf(event, 'delivered');
|
|
477
596
|
lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'delivered' });
|
|
478
597
|
}
|
|
479
598
|
else {
|
|
480
|
-
await executor.handler(this.admin, event);
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
599
|
+
const result = await executor.handler(this.admin, event, scope);
|
|
600
|
+
// #1184: a refusal is the handler's own terminal decision — journaled with its
|
|
601
|
+
// reason, no next attempt, listed by `executorDeadLetters` beside an exhausted one.
|
|
602
|
+
const refused = isDeliveryRefusal(result) ? result : null;
|
|
603
|
+
const text = refused && refusalJournalText(refused);
|
|
604
|
+
const attempt = await stub.recordExecutorAttempt(event.id, deliveryId, text, null, invocationId);
|
|
605
|
+
if (refused) {
|
|
606
|
+
report.deadLettered += 1;
|
|
607
|
+
outcomeOf(event, 'refused', refused.reason);
|
|
608
|
+
lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'dead-lettered', error: text });
|
|
609
|
+
}
|
|
610
|
+
else {
|
|
611
|
+
report.delivered += 1;
|
|
612
|
+
outcomeOf(event, 'delivered');
|
|
613
|
+
lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'delivered' });
|
|
614
|
+
}
|
|
484
615
|
}
|
|
485
616
|
}
|
|
486
617
|
catch (err) {
|
|
@@ -495,6 +626,7 @@ export class CloudflareScopeHost {
|
|
|
495
626
|
report.deadLettered += 1;
|
|
496
627
|
else
|
|
497
628
|
report.retrying += 1;
|
|
629
|
+
outcomeOf(event, exhausted ? 'dead-lettered' : 'retrying', err);
|
|
498
630
|
lines.write({
|
|
499
631
|
...unitOf(event.id, attempts),
|
|
500
632
|
startedAt,
|
|
@@ -518,13 +650,14 @@ export class CloudflareScopeHost {
|
|
|
518
650
|
// emitting operation's own tail made.
|
|
519
651
|
return this.drainExecutors(tenantId, scopeId, null);
|
|
520
652
|
}
|
|
521
|
-
registerJob(moduleId, name, handler, retry) {
|
|
653
|
+
registerJob(moduleId, name, handler, retry, options) {
|
|
522
654
|
// #1575: the kernel's own jobs are dispatched before this registry is read.
|
|
523
655
|
assertJobRegistrable(moduleId, name);
|
|
656
|
+
assertLeaseMs(options?.leaseMs);
|
|
524
657
|
const key = `${moduleId}/${name}`;
|
|
525
658
|
if (this.jobs.has(key))
|
|
526
659
|
throw new Error(`job '${key}' is already registered`);
|
|
527
|
-
this.jobs.set(key, { handler, retry });
|
|
660
|
+
this.jobs.set(key, { handler, retry, leaseMs: options?.leaseMs });
|
|
528
661
|
}
|
|
529
662
|
/**
|
|
530
663
|
* D-14's DURABLE driver: the run store is RPC to the scope DO, and nothing else.
|
|
@@ -542,13 +675,15 @@ export class CloudflareScopeHost {
|
|
|
542
675
|
const stub = this.scopeStub(scopeId);
|
|
543
676
|
return {
|
|
544
677
|
startOrJoin: (key, row) => stub.jobRunStartOrJoin(key.moduleId, key.job, key.instance, row),
|
|
545
|
-
|
|
546
|
-
|
|
678
|
+
dueKeys: (now, max) => stub.jobRunsDueKeys(now, max),
|
|
679
|
+
claim: (id, owner, leaseMs) => stub.jobRunClaim(id, owner, leaseMs),
|
|
680
|
+
begin: (id, owner, marginMs) => stub.jobRunBegin(id, owner, marginMs),
|
|
681
|
+
miss: (id, owner, note) => stub.jobRunMiss(id, owner, note),
|
|
547
682
|
list: (filter) => stub.jobRunList(filter),
|
|
548
|
-
patch: (id, patch) => stub.jobRunPatch(id, patch),
|
|
549
|
-
commitPass: (id, patch) => stub.jobCommitPass(id, patch),
|
|
550
|
-
|
|
551
|
-
recordStep: (runId, name, result, attempts, lastError, at) => stub.jobStepRecord(runId, name, result, attempts, lastError, at),
|
|
683
|
+
patch: (id, patch, owner) => stub.jobRunPatch(id, patch, owner),
|
|
684
|
+
commitPass: (id, patch, owner) => stub.jobCommitPass(id, patch, owner),
|
|
685
|
+
beginStep: (runId, name, owner, leaseMs) => stub.jobStepBegin(runId, name, owner, leaseMs),
|
|
686
|
+
recordStep: (runId, name, result, attempts, lastError, at, owner, leaseMs) => stub.jobStepRecord(runId, name, result, attempts, lastError, at, owner, leaseMs),
|
|
552
687
|
};
|
|
553
688
|
}
|
|
554
689
|
/**
|
|
@@ -585,7 +720,10 @@ export class CloudflareScopeHost {
|
|
|
585
720
|
store: this.jobStore(scopeId),
|
|
586
721
|
handlerFor: (run) => isAttachmentTextRun(run) ? attachmentText : this.jobs.get(`${run.module_id}/${run.job}`),
|
|
587
722
|
now: () => new Date().toISOString(),
|
|
588
|
-
|
|
723
|
+
// #1834: the door is opened FOR this pass, so its "not now" is tied to this pass alone.
|
|
724
|
+
openScope: async (run, pass) => this.buildStub(tenantId, scopeId, undefined, undefined, await this.openSystemDoor(run.module_id, tenantId, scopeId, undefined, pass)),
|
|
725
|
+
// #1834: only this host's own door refusals defer a pass, recognised by identity.
|
|
726
|
+
deferral: (err, pass) => this.takeDoorWait(err, pass),
|
|
589
727
|
maxPasses: options?.maxPasses,
|
|
590
728
|
limit: options?.limit,
|
|
591
729
|
});
|
|
@@ -665,9 +803,9 @@ export class CloudflareScopeHost {
|
|
|
665
803
|
const rec = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
666
804
|
// K-3: a scope under another tenant is indistinguishable from one that does not exist.
|
|
667
805
|
if (!rec)
|
|
668
|
-
throw
|
|
806
|
+
throw unknownScopeForTenant(tenantId, scopeId);
|
|
669
807
|
if (rec.status !== 'active' && rec.status !== 'provisioning') {
|
|
670
|
-
throw
|
|
808
|
+
throw substratError('conflict', `scope not migratable (status: ${rec.status}): ${scopeId}`);
|
|
671
809
|
}
|
|
672
810
|
const stub = this.scopeStub(scopeId);
|
|
673
811
|
try {
|
|
@@ -709,11 +847,11 @@ export class CloudflareScopeHost {
|
|
|
709
847
|
return this.scopeStub(scopeId).introspectTables();
|
|
710
848
|
}
|
|
711
849
|
async introspectScopeTable(scopeId, input) {
|
|
712
|
-
return this.scopeStub(scopeId).
|
|
850
|
+
return unwrapReply(await this.scopeStub(scopeId).introspectTableReply(input.table, input.limit, input.offset));
|
|
713
851
|
}
|
|
714
852
|
/** The SQL console's CP-less path (#219) — same trust line as the pair above. */
|
|
715
853
|
async introspectScopeQuery(scopeId, input) {
|
|
716
|
-
return this.scopeStub(scopeId).
|
|
854
|
+
return unwrapReply(await this.scopeStub(scopeId).introspectQueryReply(input.sql));
|
|
717
855
|
}
|
|
718
856
|
/**
|
|
719
857
|
* The K-35 denial log's CP-less path (#867) — same trust line as the reads above.
|
|
@@ -737,6 +875,31 @@ export class CloudflareScopeHost {
|
|
|
737
875
|
async listCapabilitiesLocal(scopeId, filter) {
|
|
738
876
|
return this.scopeStub(scopeId).listCapabilities(filter);
|
|
739
877
|
}
|
|
878
|
+
/**
|
|
879
|
+
* The platform's `become` mint on the CP-less path (#1686) — `HostAdmin.mintCapability`'s
|
|
880
|
+
* ScopeDO half, for the deployment that serves the scope. Its one caller is an owner claim
|
|
881
|
+
* link, minted on the platform's instruction behind the vertical's platform-gated
|
|
882
|
+
* `/internal/owner-claim`; the control plane made the K-3 check and keeps the admin log before
|
|
883
|
+
* it called. Not a module verb: a module mints through `ctx.capabilities`, which mints `act`
|
|
884
|
+
* capabilities only. Same lifecycle gate as the exchange, so a held scope mints nothing.
|
|
885
|
+
*/
|
|
886
|
+
async mintCapabilityLocal(tenantId, scopeId, input, actor) {
|
|
887
|
+
// Checked on this side too, as `HostAdmin.mintCapability` does: a typed refusal thrown
|
|
888
|
+
// across the RPC arrives as a bare message.
|
|
889
|
+
checkBecomeInput(input, new Date().toISOString());
|
|
890
|
+
await this.assertLive(tenantId, scopeId);
|
|
891
|
+
await this.migrateAndRecord(scopeId);
|
|
892
|
+
return mintedCapability.parse(await this.scopeStub(scopeId).mintBecomeCapability(input, actor));
|
|
893
|
+
}
|
|
894
|
+
/**
|
|
895
|
+
* The platform's revoke on the CP-less path (#1686) — how a re-minted claim link retires the
|
|
896
|
+
* previous one. True when the scope held the capability (revoked now, or already); false when
|
|
897
|
+
* it holds no such capability. No lifecycle gate: a revoke only narrows, so a held scope still
|
|
898
|
+
* takes one. Idempotent.
|
|
899
|
+
*/
|
|
900
|
+
async revokeCapabilityLocal(scopeId, capabilityId, actor) {
|
|
901
|
+
return (await this.scopeStub(scopeId).revokeCapabilityAsPlatform(capabilityId, actor)) !== null;
|
|
902
|
+
}
|
|
740
903
|
/**
|
|
741
904
|
* Copy one scope's data into a fresh scope DO, entirely within THIS deployment —
|
|
742
905
|
* the data half of an orchestrated snapshot (preview-and-snapshots.md §9). Like the
|
|
@@ -745,12 +908,24 @@ export class CloudflareScopeHost {
|
|
|
745
908
|
* activation, version bind) stays on the control plane's side. Because source and
|
|
746
909
|
* destination sit in the same SCOPE namespace, no scope bytes ever leave the
|
|
747
910
|
* deployment — the §9 property the trust line rests on.
|
|
911
|
+
*
|
|
912
|
+
* #2016: `tenantId`, the tenant the platform snapshots for (absent from an older platform). The
|
|
913
|
+
* source must not be foreign to it — a copy never reads another tenant's scope — and the fork
|
|
914
|
+
* records it as its own receipt, not the source's.
|
|
748
915
|
*/
|
|
749
|
-
async snapshotScopeLocal(sourceScopeId, destScopeId) {
|
|
916
|
+
async snapshotScopeLocal(sourceScopeId, destScopeId, tenantId) {
|
|
917
|
+
if (tenantId !== undefined && (await this.scopeStub(sourceScopeId).admission(tenantId)).verdict === 'foreign') {
|
|
918
|
+
throw unknownScopeForTenant(tenantId, sourceScopeId);
|
|
919
|
+
}
|
|
750
920
|
const tables = await this.scopeStub(sourceScopeId).exportDump();
|
|
751
921
|
// #2009: a snapshot is a fork by construction (the control plane's row names `forkedFrom`, so
|
|
752
922
|
// the directory never calls it primary), and a load classifies nothing by itself: marked here.
|
|
753
|
-
await this.scopeStub(destScopeId).importDump(tables, destScopeId, {
|
|
923
|
+
await this.scopeStub(destScopeId).importDump(tables, destScopeId, {
|
|
924
|
+
sourceScopeId,
|
|
925
|
+
exact: true,
|
|
926
|
+
markCopy: true,
|
|
927
|
+
provisionedFor: tenantId,
|
|
928
|
+
});
|
|
754
929
|
return { tables: tables.length };
|
|
755
930
|
}
|
|
756
931
|
/**
|
|
@@ -767,28 +942,34 @@ export class CloudflareScopeHost {
|
|
|
767
942
|
* #1722: `loadStamp`, the stamp a carry leaves on the copy it lands, for a fenced wipe later,
|
|
768
943
|
* and `expect`, the marker the carry read here: the load is refused if the store moved since.
|
|
769
944
|
* #2005: `markCopy` — the directory's classification of the scope, sent when it is not
|
|
770
|
-
* primary; the restore then marks it a copy. Refused if it classifies a primary.
|
|
945
|
+
* primary; the restore then marks it a copy. Refused if it classifies a primary.
|
|
946
|
+
* #2016: `tenantId`, the tenant the platform restores for: recorded as the scope's receipt in
|
|
947
|
+
* the load's own transaction, and a store provisioned for another tenant refuses the load. */
|
|
771
948
|
opts) {
|
|
772
949
|
if (opts?.markCopy)
|
|
773
950
|
assertCopyLineage(opts.markCopy);
|
|
951
|
+
const carry = opts ? recordedOffFromWire(opts) : undefined;
|
|
774
952
|
const load = {
|
|
775
|
-
switchOff:
|
|
953
|
+
switchOff: carry ? { at: new Date().toISOString(), ...carry } : undefined,
|
|
776
954
|
sourceScopeId: opts?.sourceScopeId,
|
|
777
955
|
exact: opts?.exact,
|
|
778
956
|
loadStamp: opts?.loadStamp,
|
|
779
957
|
markCopy: opts?.markCopy !== undefined,
|
|
958
|
+
provisionedFor: opts?.tenantId,
|
|
780
959
|
};
|
|
781
|
-
//
|
|
960
|
+
// The refusals come back as values (a throw over the RPC carries only its message) and are
|
|
782
961
|
// thrown here, in the vertical's own isolate, so its route answers 409 or 412, not a fault.
|
|
783
962
|
const out = await this.scopeStub(scopeId).importDumpChecked(tables, scopeId, {
|
|
784
963
|
...load,
|
|
785
964
|
...(opts?.expect ? { expect: opts.expect } : {}),
|
|
786
965
|
});
|
|
787
966
|
if (out.refused === false) {
|
|
788
|
-
return { tables: tables.length, ...(
|
|
967
|
+
return { tables: tables.length, ...(carry ? { switchedOff: out.switchedOff } : {}) };
|
|
789
968
|
}
|
|
790
969
|
if (out.refused === 'kept')
|
|
791
970
|
throw substratError('conflict', KEPT_COPY_REFUSAL);
|
|
971
|
+
if (out.refused === 'tenant')
|
|
972
|
+
throw substratError('conflict', out.message);
|
|
792
973
|
// A retry of a load that already committed (its answer was lost on the way back) is
|
|
793
974
|
// refused by the marker that load itself moved. The store holding THIS request's stamp
|
|
794
975
|
// says so: no other load writes it. Answered as applied, without `switchedOff`, so the
|
|
@@ -855,9 +1036,16 @@ export class CloudflareScopeHost {
|
|
|
855
1036
|
* the vertical's `/internal/lifecycle`. Kept unless the scope already holds a newer one, so a
|
|
856
1037
|
* push that arrives late cannot undo a later transition. `applied` is what the platform's heal
|
|
857
1038
|
* sweep records as delivered; `changed` says the gate's answer moved.
|
|
1039
|
+
*
|
|
1040
|
+
* #2016: `tenantId`, the tenant the directory delivered it for (absent from an older platform).
|
|
1041
|
+
* A scope provisioned for another tenant refuses the delivery (`conflict`) and stores nothing;
|
|
1042
|
+
* a scope provisioned before the tenant receipt existed records it here, its role rows agreeing.
|
|
858
1043
|
*/
|
|
859
|
-
async setLifecycleLocal(scopeId, next) {
|
|
860
|
-
|
|
1044
|
+
async setLifecycleLocal(scopeId, next, tenantId) {
|
|
1045
|
+
const out = await this.scopeStub(scopeId).setLifecycle(scopeLifecycle.parse(next), tenantId);
|
|
1046
|
+
if ('refused' in out)
|
|
1047
|
+
throw substratError('conflict', out.message);
|
|
1048
|
+
return lifecycleDelivery.parse(out);
|
|
861
1049
|
}
|
|
862
1050
|
/**
|
|
863
1051
|
* Mark one scope in THIS deployment a copy (#2005), behind the vertical's `/internal/mark-copy`:
|
|
@@ -899,7 +1087,7 @@ export class CloudflareScopeHost {
|
|
|
899
1087
|
* safe; scope-level tuples (the restored grants) are never touched.
|
|
900
1088
|
*/
|
|
901
1089
|
async projectRolesLocal(tenantId, scopeId, roles) {
|
|
902
|
-
await this.scopeStub(scopeId).
|
|
1090
|
+
unwrapReply(await this.scopeStub(scopeId).applyProjectionReply(tenantId, roles.map((r) => ({ role_key: r.key, permissions: JSON.stringify(r.permissions), source: r.source })), []));
|
|
903
1091
|
}
|
|
904
1092
|
/**
|
|
905
1093
|
* Wipe one scope DO's storage in THIS deployment — the reap half of an orchestrated
|
|
@@ -951,6 +1139,10 @@ export class CloudflareScopeHost {
|
|
|
951
1139
|
async lifecycleFlowLocal(scopeId, input) {
|
|
952
1140
|
return this.scopeStub(scopeId).lifecycleFlow(input);
|
|
953
1141
|
}
|
|
1142
|
+
/** Business volumes per bucket (#1750) on this host's own scope — the vertical-host read. */
|
|
1143
|
+
async operationSeriesLocal(scopeId, input) {
|
|
1144
|
+
return this.scopeStub(scopeId).operationSeries(input);
|
|
1145
|
+
}
|
|
954
1146
|
/** One event's causal chain (#1237) on this host's own scope — the vertical-host read. */
|
|
955
1147
|
async eventCauseLocal(scopeId, input) {
|
|
956
1148
|
return this.scopeStub(scopeId).eventCause(input);
|
|
@@ -1103,7 +1295,7 @@ export class CloudflareScopeHost {
|
|
|
1103
1295
|
const record = resolveScopeRecord(input);
|
|
1104
1296
|
// Fail-closed tenant gate throws out of the awaited cp call BEFORE migrate
|
|
1105
1297
|
// or audit, so a rejected provision creates nothing and writes no audit row.
|
|
1106
|
-
const created = await this.
|
|
1298
|
+
const created = await this.directory('provisionScope', input.tenantId, input.scopeId, record, new Date().toISOString());
|
|
1107
1299
|
// Instantiate the scope DO and trigger its lazy migration.
|
|
1108
1300
|
await this.migrateAndRecord(input.scopeId);
|
|
1109
1301
|
// Scope-local permissions: a freshly-provisioned scope evaluates from its own
|
|
@@ -1144,10 +1336,9 @@ export class CloudflareScopeHost {
|
|
|
1144
1336
|
// #1742: in the SAME unit as the seat, so no sweep can land between them; the re-assert
|
|
1145
1337
|
// after it then finds them off, and audits what the unit moved.
|
|
1146
1338
|
const ownStore = this.cpLess || record.vertical === null;
|
|
1147
|
-
const
|
|
1148
|
-
const
|
|
1149
|
-
|
|
1150
|
-
: undefined);
|
|
1339
|
+
const seatAt = new Date().toISOString();
|
|
1340
|
+
const carry = ownStore ? await this.recordedOffCarry(input.tenantId, input.scopeId, seatAt) : undefined;
|
|
1341
|
+
const switchedOff = await this.scopeStub(input.scopeId).seatTuples(seats, carry ? { scopeId: input.scopeId, at: seatAt, ...carry } : undefined);
|
|
1151
1342
|
if (ownStore) {
|
|
1152
1343
|
await this.admin.reassertSystemSwitches(actor, { tenantId: input.tenantId, scopeId: input.scopeId }, { appliedInUnit: switchedOff });
|
|
1153
1344
|
}
|
|
@@ -1169,10 +1360,10 @@ export class CloudflareScopeHost {
|
|
|
1169
1360
|
// inside the DO's ledger write, which is the serialization point that actually holds.
|
|
1170
1361
|
const tenant = await this.cp.getTenant(input.tenantId);
|
|
1171
1362
|
if (!tenant) {
|
|
1172
|
-
throw
|
|
1363
|
+
throw substratError('not_found', `cannot provision tenant store under unknown tenant: ${input.tenantId}`);
|
|
1173
1364
|
}
|
|
1174
1365
|
if (tenant.status !== 'active') {
|
|
1175
|
-
throw
|
|
1366
|
+
throw substratError('conflict', `cannot provision tenant store under non-active tenant (status: ${tenant.status}): ${input.tenantId}`);
|
|
1176
1367
|
}
|
|
1177
1368
|
// Idempotent on (tenant, vertical, binding): a retried provision re-resolves the SAME
|
|
1178
1369
|
// store rather than minting a second database (the K-31 ready-gate retries the whole
|
|
@@ -1187,7 +1378,7 @@ export class CloudflareScopeHost {
|
|
|
1187
1378
|
// the source of truth, carrying the D1 database_id Cloudflare assigned as the ref.
|
|
1188
1379
|
const name = await tenantStoreDatabaseName(input.tenantId, input.vertical, input.binding);
|
|
1189
1380
|
const ref = await d1.create(name);
|
|
1190
|
-
const stored = await this.
|
|
1381
|
+
const stored = await this.directory('putTenantStore', {
|
|
1191
1382
|
tenantId: input.tenantId,
|
|
1192
1383
|
vertical: input.vertical,
|
|
1193
1384
|
binding: input.binding,
|
|
@@ -1238,17 +1429,17 @@ export class CloudflareScopeHost {
|
|
|
1238
1429
|
const r2 = this.requireBlobStores(`provisionBlobStore(tenant=${input.tenantId} vertical=${input.vertical} binding=${input.binding})`);
|
|
1239
1430
|
const tenant = await this.cp.getTenant(input.tenantId);
|
|
1240
1431
|
if (!tenant) {
|
|
1241
|
-
throw
|
|
1432
|
+
throw substratError('not_found', `cannot provision blob store under unknown tenant: ${input.tenantId}`);
|
|
1242
1433
|
}
|
|
1243
1434
|
if (tenant.status !== 'active') {
|
|
1244
|
-
throw
|
|
1435
|
+
throw substratError('conflict', `cannot provision blob store under non-active tenant (status: ${tenant.status}): ${input.tenantId}`);
|
|
1245
1436
|
}
|
|
1246
1437
|
const existing = await this.cp.getBlobStore(input.tenantId, input.vertical, input.binding);
|
|
1247
1438
|
if (existing)
|
|
1248
1439
|
return { binding: input.binding, kind: 'blob', ref: existing.ref };
|
|
1249
1440
|
const name = await blobStoreBucketName(input.tenantId, input.vertical, input.binding);
|
|
1250
1441
|
const ref = await r2.create(name);
|
|
1251
|
-
const stored = await this.
|
|
1442
|
+
const stored = await this.directory('putBlobStore', {
|
|
1252
1443
|
tenantId: input.tenantId,
|
|
1253
1444
|
vertical: input.vertical,
|
|
1254
1445
|
binding: input.binding,
|
|
@@ -1292,15 +1483,15 @@ export class CloudflareScopeHost {
|
|
|
1292
1483
|
return this.buildAttachmentSurface({ principal }, tenantId, scopeId, store);
|
|
1293
1484
|
}
|
|
1294
1485
|
async getSystemAttachments(moduleId, tenantId, scopeId) {
|
|
1295
|
-
// Reuse the system invoke door's module, tenant and lifecycle gate
|
|
1296
|
-
// authorizes each open against the attachment target's read permission as
|
|
1486
|
+
// Reuse the system invoke door's module, tenant and lifecycle gate, and its rewind gate
|
|
1487
|
+
// (#1834). The DO authorizes each open against the attachment target's read permission as
|
|
1297
1488
|
// system:<moduleId>; the worker reads bytes only after that gate succeeds.
|
|
1298
|
-
await this.
|
|
1489
|
+
const door = await this.openSystemDoor(moduleId, tenantId, scopeId);
|
|
1299
1490
|
const store = await this.resolveAttachmentStore(tenantId);
|
|
1300
1491
|
const stub = this.scopeStub(scopeId);
|
|
1301
1492
|
return {
|
|
1302
1493
|
open: async (attachmentId) => {
|
|
1303
|
-
const record = await stub.systemAttachmentAuthorize(attachmentId, moduleId, tenantId, scopeId);
|
|
1494
|
+
const record = await door.through((instance) => stub.systemAttachmentAuthorize(attachmentId, moduleId, tenantId, scopeId, instance));
|
|
1304
1495
|
return record ? this.openAttachmentBytes(store, scopeId, record) : null;
|
|
1305
1496
|
},
|
|
1306
1497
|
};
|
|
@@ -1387,22 +1578,22 @@ export class CloudflareScopeHost {
|
|
|
1387
1578
|
// the DO only to be refused and recorded — never to the blob store.
|
|
1388
1579
|
const hash = subject.capabilitySession;
|
|
1389
1580
|
return {
|
|
1390
|
-
upload: async (input) =>
|
|
1581
|
+
upload: async (input) => unwrapReply(await stub.capabilityAttachmentRefuseWrite(hash, tenantId, scopeId, 'attachments.upload', {
|
|
1391
1582
|
entityType: input.entity.entityType,
|
|
1392
1583
|
})),
|
|
1393
|
-
list: async (entity) =>
|
|
1584
|
+
list: async (entity) => unwrapReply(await stub.capabilityAttachmentList(entity, hash, tenantId, scopeId)),
|
|
1394
1585
|
open: async (attachmentId) => {
|
|
1395
|
-
const record =
|
|
1586
|
+
const record = unwrapReply(await stub.capabilityAttachmentOpen(attachmentId, hash, tenantId, scopeId));
|
|
1396
1587
|
return record ? this.openAttachmentBytes(store, scopeId, record) : null;
|
|
1397
1588
|
},
|
|
1398
|
-
remove: async (attachmentId) =>
|
|
1589
|
+
remove: async (attachmentId) => unwrapReply(await stub.capabilityAttachmentRefuseWrite(hash, tenantId, scopeId, 'attachments.remove', {
|
|
1399
1590
|
attachmentId,
|
|
1400
1591
|
})),
|
|
1401
1592
|
search: async (term, options) => {
|
|
1402
1593
|
// Judged here too, so a short term reaches the caller as `SearchTermTooShort`
|
|
1403
1594
|
// rather than as a message that crossed the RPC boundary. The DO judges again.
|
|
1404
1595
|
searchMatchExpression(term, 'prefix');
|
|
1405
|
-
return
|
|
1596
|
+
return unwrapReply(await stub.capabilityAttachmentSearch(term, searchLimit(options?.limit), hash, tenantId, scopeId));
|
|
1406
1597
|
},
|
|
1407
1598
|
};
|
|
1408
1599
|
}
|
|
@@ -1457,7 +1648,7 @@ export class CloudflareScopeHost {
|
|
|
1457
1648
|
throw new Error('attachments.search is not available on a delivery-scoped surface');
|
|
1458
1649
|
}
|
|
1459
1650
|
searchMatchExpression(term, 'prefix');
|
|
1460
|
-
return
|
|
1651
|
+
return unwrapReply(await stub.attachmentSearch(term, searchLimit(options?.limit), asPrincipalId, tenantId, scopeId, connectionId));
|
|
1461
1652
|
},
|
|
1462
1653
|
};
|
|
1463
1654
|
}
|
|
@@ -1505,16 +1696,51 @@ export class CloudflareScopeHost {
|
|
|
1505
1696
|
// this is the scope's real store, as for the provision's seat above; a hosted scope is
|
|
1506
1697
|
// restored in its deployment, which the platform carries the same list to.
|
|
1507
1698
|
const ownStore = this.cpLess || existing.vertical === null;
|
|
1508
|
-
const
|
|
1699
|
+
const replayAt = new Date().toISOString();
|
|
1700
|
+
const carry = ownStore ? await this.recordedOffCarry(tenantId, scopeId, replayAt) : undefined;
|
|
1509
1701
|
const switchedOff = await this.scopeStub(scopeId).importDump(dump.tables, scopeId, {
|
|
1510
|
-
switchOff:
|
|
1702
|
+
switchOff: carry ? { at: replayAt, ...carry } : undefined,
|
|
1511
1703
|
sourceScopeId: opts?.sourceScopeId ?? dump.scopeId,
|
|
1512
1704
|
});
|
|
1513
1705
|
await this.recordAdmin(actor, 'restoreScope', { tenantId, scopeId }, null, { sourceScopeId: dump.scopeId, tables: dump.tables.length, capturedAt: dump.capturedAt });
|
|
1514
|
-
if (
|
|
1706
|
+
if (carry) {
|
|
1515
1707
|
await this.admin.reassertSystemSwitches(actor, { tenantId, scopeId }, { appliedInUnit: switchedOff });
|
|
1516
1708
|
}
|
|
1517
1709
|
}
|
|
1710
|
+
/**
|
|
1711
|
+
* #1742, #2029: what this host's own unit carries of the directory's record — the modules and
|
|
1712
|
+
* the peers recorded OFF on the scope, each with the ones held there only by a tenant-level
|
|
1713
|
+
* grant (#1823, #2030). Undefined when nothing is recorded off, so such a unit is unchanged.
|
|
1714
|
+
*/
|
|
1715
|
+
async recordedOffCarry(tenantId, scopeId, at) {
|
|
1716
|
+
const [modules, peers] = await Promise.all([
|
|
1717
|
+
this.recordedOff('system', tenantId, scopeId, at),
|
|
1718
|
+
this.recordedOff('peer', tenantId, scopeId, at),
|
|
1719
|
+
]);
|
|
1720
|
+
if (!modules.keys.length && !peers.keys.length)
|
|
1721
|
+
return undefined;
|
|
1722
|
+
return {
|
|
1723
|
+
moduleIds: modules.keys,
|
|
1724
|
+
tenantHeld: modules.tenantHeld,
|
|
1725
|
+
verticals: peers.keys,
|
|
1726
|
+
tenantHeldVerticals: peers.tenantHeld,
|
|
1727
|
+
fences: switchFencesCarried([
|
|
1728
|
+
['system', modules],
|
|
1729
|
+
['peer', peers],
|
|
1730
|
+
]),
|
|
1731
|
+
};
|
|
1732
|
+
}
|
|
1733
|
+
/** The subjects of one kind the directory records OFF on the scope, and of those the tenant-held ones. */
|
|
1734
|
+
async recordedOff(kind, tenantId, scopeId, at) {
|
|
1735
|
+
// #2045: the OFF rows and their fences in one read, so every key carried has its own row's fence.
|
|
1736
|
+
const fences = new Map((await this.cp.switchRecordStatesOf(kind, tenantId, scopeId))
|
|
1737
|
+
.filter(([, row]) => row.position === 'off')
|
|
1738
|
+
.map(([key, row]) => [key, row.fence]));
|
|
1739
|
+
const keys = [...fences.keys()];
|
|
1740
|
+
if (!keys.length)
|
|
1741
|
+
return { keys, tenantHeld: [], fences };
|
|
1742
|
+
return { keys, tenantHeld: await this.cp.tenantHeldOf(kind, tenantId, keys, at), fences };
|
|
1743
|
+
}
|
|
1518
1744
|
async snapshotScope(actor, tenantId, scopeId, opts) {
|
|
1519
1745
|
const source = await this.admin.getScopeRecord(actor, tenantId, scopeId);
|
|
1520
1746
|
if (!source)
|
|
@@ -1545,7 +1771,7 @@ export class CloudflareScopeHost {
|
|
|
1545
1771
|
if (!rec)
|
|
1546
1772
|
throw substratError('not_found', `unknown scope ${scopeId} in tenant ${tenantId}`);
|
|
1547
1773
|
if (!rec.forkedFrom && rec.kind !== 'preview') {
|
|
1548
|
-
throw
|
|
1774
|
+
throw substratError('forbidden', `scope ${scopeId} is not a fork or preview — only previews may be deleted; ` +
|
|
1549
1775
|
`archive a primary scope instead`);
|
|
1550
1776
|
}
|
|
1551
1777
|
// Storage BEFORE the directory row: a crash between the two leaves a visible row
|
|
@@ -1685,12 +1911,134 @@ export class CloudflareScopeHost {
|
|
|
1685
1911
|
* against `system:<moduleId>` grants inside the stub.
|
|
1686
1912
|
*/
|
|
1687
1913
|
async getSystemScope(moduleId, tenantId, scopeId) {
|
|
1914
|
+
return this.buildStub(tenantId, scopeId, undefined, undefined, await this.openSystemDoor(moduleId, tenantId, scopeId));
|
|
1915
|
+
}
|
|
1916
|
+
/**
|
|
1917
|
+
* #1834: the system door — the ONE place every entry acting as `system:<moduleId>` passes
|
|
1918
|
+
* through: a schedule's fire, a job run's `pass.scope()`, a module's attachment open. The
|
|
1919
|
+
* module must be registered here and the scope live, as before. Then the door GATES: the
|
|
1920
|
+
* scope's own state read for the module, and after it the rewind hold (#1819), in the order
|
|
1921
|
+
* `SWITCH_HOLD_SETTLE_MS` relies on. A module the hold keeps off is refused here, as a
|
|
1922
|
+
* schedule pass skips it.
|
|
1923
|
+
*
|
|
1924
|
+
* A gate read is about the instance that answered it, and a door can be held for as long as
|
|
1925
|
+
* a job pass runs. So every call through the door is PINNED to that instance: the DO refuses
|
|
1926
|
+
* one that lands on another (`SYSTEM_DOOR_MOVED`), since a PITR restore always restarts the
|
|
1927
|
+
* object, and the door gates again and retries. A restart within one call's round trip is
|
|
1928
|
+
* rare, so past `SYSTEM_DOOR_REGATES` it fails closed (`unavailable`). No clock is involved:
|
|
1929
|
+
* the gate and the call meet the same storage, or the call does not run.
|
|
1930
|
+
*/
|
|
1931
|
+
async openSystemDoor(moduleId, tenantId, scopeId, gated,
|
|
1932
|
+
/** #1834: the job pass this door is opened for; its "not now" refusals are tied to it. */
|
|
1933
|
+
pass) {
|
|
1688
1934
|
if (!this.moduleIds.has(moduleId)) {
|
|
1689
|
-
throw
|
|
1935
|
+
throw substratError('not_found', `module not registered on this host: ${moduleId}`);
|
|
1690
1936
|
}
|
|
1691
1937
|
await this.assertLive(tenantId, scopeId);
|
|
1692
1938
|
await this.migrateAndRecord(scopeId);
|
|
1693
|
-
return this.
|
|
1939
|
+
return this.openDoor('system', moduleId, scopeId, gated, pass);
|
|
1940
|
+
}
|
|
1941
|
+
/**
|
|
1942
|
+
* #1834: one door, for either kind of switched subject (#2029). The gate is read here unless the
|
|
1943
|
+
* caller read it (`runDueSchedules` reports a hold it could not read on its pass report); a held
|
|
1944
|
+
* subject is refused, and a call landing on another instance than the gate's re-gates, at most
|
|
1945
|
+
* `SYSTEM_DOOR_REGATES` times, then fails closed.
|
|
1946
|
+
*
|
|
1947
|
+
* The peer door (`kind: 'peer'`) is the system door's for a peer vertical. A PITR rewind to before
|
|
1948
|
+
* a peer was switched off brings its grants back live and its OFF marker gone, so the rewind holds
|
|
1949
|
+
* it outside the scope (`rewindHolding`, as for a module), and every peer entry — an invoke, a
|
|
1950
|
+
* delivery, a coverage read, an export read for it as consumer — opens this door after
|
|
1951
|
+
* `peerScopeGate`.
|
|
1952
|
+
*/
|
|
1953
|
+
async openDoor(kind, key, scopeId, gated, pass) {
|
|
1954
|
+
const words = DOOR_WORDS[kind];
|
|
1955
|
+
const what = words.what(key);
|
|
1956
|
+
// A gate this door reads itself reports a hold it could not read; one handed in was the
|
|
1957
|
+
// caller's to report.
|
|
1958
|
+
const regate = async () => {
|
|
1959
|
+
const fresh = await this.doorGate(kind, scopeId, key);
|
|
1960
|
+
if (fresh.error)
|
|
1961
|
+
console.error(`substrat: door of ${what} on ${scopeId}: ${fresh.error}`);
|
|
1962
|
+
return fresh;
|
|
1963
|
+
};
|
|
1964
|
+
let gate = gated ?? (await regate());
|
|
1965
|
+
return {
|
|
1966
|
+
kind,
|
|
1967
|
+
key,
|
|
1968
|
+
through: async (call) => {
|
|
1969
|
+
for (let regates = 0;; regates += 1) {
|
|
1970
|
+
if (gate.held) {
|
|
1971
|
+
// The hold ends when the switch is applied again: a job run waits it out without
|
|
1972
|
+
// spending its retries, because the driver defers on THIS refusal (`doorWait`).
|
|
1973
|
+
const held = substratError('forbidden', `${what} is held off on this scope: it was rewound to before its ` +
|
|
1974
|
+
`${words.switchName} switch was turned off, and it stays off until ` +
|
|
1975
|
+
'the switch is applied again', { reason: SYSTEM_DOOR_WAIT });
|
|
1976
|
+
this.heldRefusals.add(held);
|
|
1977
|
+
throw this.doorWait(pass, held);
|
|
1978
|
+
}
|
|
1979
|
+
const answer = await call(gate.instance);
|
|
1980
|
+
if (!isSystemDoorMoved(answer))
|
|
1981
|
+
return answer;
|
|
1982
|
+
if (regates >= SYSTEM_DOOR_REGATES) {
|
|
1983
|
+
throw this.doorWait(pass, substratError('unavailable', `the scope kept restarting under the ${words.door} of ${what} ` +
|
|
1984
|
+
`(${regates + 1} attempts); nothing ran. Retry later`, { reason: SYSTEM_DOOR_WAIT }));
|
|
1985
|
+
}
|
|
1986
|
+
gate = await regate();
|
|
1987
|
+
}
|
|
1988
|
+
},
|
|
1989
|
+
};
|
|
1990
|
+
}
|
|
1991
|
+
/**
|
|
1992
|
+
* #1834: the refusals this host's system door threw that mean "not now" (a hold, a scope that
|
|
1993
|
+
* kept restarting), each tied by identity to the job pass whose door threw it. The job driver
|
|
1994
|
+
* defers a pass only on one of these, never on an error's shape, which any step or operation
|
|
1995
|
+
* could copy. A mark is ORIGIN and FRESHNESS together: it is consumed when the driver takes it,
|
|
1996
|
+
* and only the pass it names can take it, so a handler that keeps a refusal and throws it again
|
|
1997
|
+
* on a later pass is an ordinary failure there. Held weakly: an error nobody holds goes.
|
|
1998
|
+
*/
|
|
1999
|
+
doorWaits = new WeakMap();
|
|
2000
|
+
/**
|
|
2001
|
+
* #2029: the hold refusals this host's doors threw, by identity — so a peer delivery pauses its
|
|
2002
|
+
* edge, and a coverage read answers "holds nothing", on exactly those and on no error that only
|
|
2003
|
+
* looks like one. Held weakly, as `doorWaits` is.
|
|
2004
|
+
*/
|
|
2005
|
+
heldRefusals = new WeakSet();
|
|
2006
|
+
/** #2029: is `err` a hold refusal this host's door threw? */
|
|
2007
|
+
isHeldRefusal(err) {
|
|
2008
|
+
return typeof err === 'object' && err !== null && this.heldRefusals.has(err);
|
|
2009
|
+
}
|
|
2010
|
+
/** #1834: mark this door's own refusal as "not now" for the pass it was opened for, and hand it back. */
|
|
2011
|
+
doorWait(pass, err) {
|
|
2012
|
+
if (pass)
|
|
2013
|
+
this.doorWaits.set(err, pass);
|
|
2014
|
+
return err;
|
|
2015
|
+
}
|
|
2016
|
+
/** #1834: is `err` a "not now" this host's door threw on `pass`? Consumes the mark when it is. */
|
|
2017
|
+
takeDoorWait(err, pass) {
|
|
2018
|
+
if (typeof err !== 'object' || err === null || this.doorWaits.get(err) !== pass)
|
|
2019
|
+
return false;
|
|
2020
|
+
this.doorWaits.delete(err);
|
|
2021
|
+
return true;
|
|
2022
|
+
}
|
|
2023
|
+
/**
|
|
2024
|
+
* #1834: a door's gate — the scope's state read for the subject, the instance that answered it,
|
|
2025
|
+
* and then the rewind hold (#1819), for any subject the scope does not already have off. The
|
|
2026
|
+
* hold is read AFTER the state, which is the order `switchHeld` needs.
|
|
2027
|
+
*
|
|
2028
|
+
* `ungranted` reads the hold too (#2029): a subject held on the scope only by a TENANT-level
|
|
2029
|
+
* grant (#1823, #2030) comes back from a rewind with no scope grant and no marker, and the
|
|
2030
|
+
* tenant grant still authorizes it there. A rewind captured it off, so the hold must keep it so.
|
|
2031
|
+
*/
|
|
2032
|
+
async doorGate(kind, scopeId, key) {
|
|
2033
|
+
const stub = this.scopeStub(scopeId);
|
|
2034
|
+
const { state, instance } = await (kind === 'system' ? stub.systemDoorState(key) : stub.peerDoorState(key));
|
|
2035
|
+
const cleared = `${scopeId} ${instance}`;
|
|
2036
|
+
if (state === 'off' || CLEAR_SCOPE_INSTANCES.has(cleared))
|
|
2037
|
+
return { state, instance, held: false };
|
|
2038
|
+
const hold = await this.switchHeld(scopeId, holdKeyOf(kind, key));
|
|
2039
|
+
if (hold.scopeClear)
|
|
2040
|
+
rememberClearInstance(cleared);
|
|
2041
|
+
return { state, instance, held: hold.held, ...(hold.error ? { error: hold.error } : {}) };
|
|
1694
2042
|
}
|
|
1695
2043
|
/**
|
|
1696
2044
|
* #1705: the consumer half. The same lifecycle gate as every door (K-3), then the ScopeDO
|
|
@@ -1703,8 +2051,25 @@ export class CloudflareScopeHost {
|
|
|
1703
2051
|
// The peer door's own gate (#1706): K-3's pair check, the refusal for a scope served
|
|
1704
2052
|
// elsewhere, the lifecycle check and the migration. A delivery is a write into the
|
|
1705
2053
|
// consumer's scope, so it takes the gate an invoke through that door takes.
|
|
1706
|
-
await this.peerScopeGate(tenantId, scopeId, 'deliverToPeer');
|
|
1707
|
-
|
|
2054
|
+
const record = await this.peerScopeGate(tenantId, scopeId, 'deliverToPeer');
|
|
2055
|
+
// #2004: a copy consumes no other vertical's events, at the door as in the sweep
|
|
2056
|
+
// (`scope-copy.ts`). A pause: nothing runs and the watermark stays.
|
|
2057
|
+
if (await this.isInertScope(tenantId, scopeId, record)) {
|
|
2058
|
+
return { ...emptyImportResult(batch), paused: { reason: INERT_SCOPE_REASON } };
|
|
2059
|
+
}
|
|
2060
|
+
// #2029: the producer is a peer, so the delivery goes through its door: a producer the rewind
|
|
2061
|
+
// hold keeps off pauses the edge, exactly as a switched-off one does inside the DO.
|
|
2062
|
+
const door = await this.openDoor('peer', batch.source.vertical, scopeId);
|
|
2063
|
+
let applied;
|
|
2064
|
+
try {
|
|
2065
|
+
applied = await door.through((instance) => this.scopeStub(scopeId).importApply(batch, tenantId, scopeId, instance));
|
|
2066
|
+
}
|
|
2067
|
+
catch (err) {
|
|
2068
|
+
if (!this.isHeldRefusal(err))
|
|
2069
|
+
throw err;
|
|
2070
|
+
return { ...emptyImportResult(batch), paused: { reason: err.message } };
|
|
2071
|
+
}
|
|
2072
|
+
const result = importResult.parse(applied);
|
|
1708
2073
|
if (result.delivered > 0) {
|
|
1709
2074
|
try {
|
|
1710
2075
|
await this.drainExecutors(tenantId, scopeId, null);
|
|
@@ -1748,7 +2113,9 @@ export class CloudflareScopeHost {
|
|
|
1748
2113
|
async getVerticalScope(caller, tenantId, scopeId, options) {
|
|
1749
2114
|
const parsed = verticalCaller.parse(caller);
|
|
1750
2115
|
await this.peerScopeGate(tenantId, scopeId, 'getVerticalScope');
|
|
1751
|
-
|
|
2116
|
+
// #2029: gated against the rewind hold and pinned, as the system door is.
|
|
2117
|
+
const door = await this.openDoor('peer', parsed.vertical, scopeId);
|
|
2118
|
+
return this.buildStub(tenantId, scopeId, undefined, undefined, door, options, undefined, undefined, parsed);
|
|
1752
2119
|
}
|
|
1753
2120
|
/**
|
|
1754
2121
|
* Does peer `vertical` hold each key at this scope's node now (#1706) — the ScopeDO's own
|
|
@@ -1756,9 +2123,19 @@ export class CloudflareScopeHost {
|
|
|
1756
2123
|
*/
|
|
1757
2124
|
async peerCovers(tenantId, scopeId, vertical, permissions) {
|
|
1758
2125
|
await this.peerScopeGate(tenantId, scopeId, 'peerCovers');
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
2126
|
+
const slug = verticalSlug.parse(vertical);
|
|
2127
|
+
// #2029: a peer the rewind hold keeps off holds nothing here, as a switched-off one doesn't.
|
|
2128
|
+
const door = await this.openDoor('peer', slug, scopeId);
|
|
2129
|
+
try {
|
|
2130
|
+
return peerCoverage
|
|
2131
|
+
.array()
|
|
2132
|
+
.parse(await door.through((instance) => this.scopeStub(scopeId).peerCovers(tenantId, scopeId, slug, [...permissions], instance)));
|
|
2133
|
+
}
|
|
2134
|
+
catch (err) {
|
|
2135
|
+
if (this.isHeldRefusal(err))
|
|
2136
|
+
return permissions.map((permission) => ({ permission, held: false }));
|
|
2137
|
+
throw err;
|
|
2138
|
+
}
|
|
1762
2139
|
}
|
|
1763
2140
|
/**
|
|
1764
2141
|
* `ctx.canAssign`'s bound for a principal the host names (#1931) — the ScopeDO's own role read
|
|
@@ -1772,7 +2149,7 @@ export class CloudflareScopeHost {
|
|
|
1772
2149
|
// provisioning receipt is K-3 here (#1738), read before anything migrates the DO, and the
|
|
1773
2150
|
// refusal reads as the pure host's does: an unknown scope, not an unknown role.
|
|
1774
2151
|
if (this.cpLess && !(await this.scopeStub(scopeId).servesTenant(tenantId))) {
|
|
1775
|
-
throw
|
|
2152
|
+
throw unknownScopeForTenant(tenantId, scopeId);
|
|
1776
2153
|
}
|
|
1777
2154
|
await this.peerScopeGate(tenantId, scopeId, 'canAssign');
|
|
1778
2155
|
const bound = await this.scopeStub(scopeId).canAssignFor(tenantId, scopeId, principalId.parse(principal), roleKey);
|
|
@@ -1782,7 +2159,7 @@ export class CloudflareScopeHost {
|
|
|
1782
2159
|
}
|
|
1783
2160
|
async assignScopeRoleBounded(tenantId, scopeId, caller, assignee, roleKey) {
|
|
1784
2161
|
if (this.cpLess && !(await this.scopeStub(scopeId).servesTenant(tenantId))) {
|
|
1785
|
-
throw
|
|
2162
|
+
throw unknownScopeForTenant(tenantId, scopeId);
|
|
1786
2163
|
}
|
|
1787
2164
|
await this.peerScopeGate(tenantId, scopeId, 'assignScopeRoleBounded');
|
|
1788
2165
|
const bound = await this.scopeStub(scopeId).assignScopeRoleBoundedFor(tenantId, scopeId, principalId.parse(caller), principalId.parse(assignee), roleKey);
|
|
@@ -1804,15 +2181,33 @@ export class CloudflareScopeHost {
|
|
|
1804
2181
|
// door's own tenant-confinement test proved. A RECORD crosses intact, so the coordinator
|
|
1805
2182
|
// reads the record and types the refusal itself. `validateScopeAccess` still runs below
|
|
1806
2183
|
// and still owns the lifecycle half (a suspended tenant or scope).
|
|
2184
|
+
let record;
|
|
1807
2185
|
if (!this.cpLess) {
|
|
1808
|
-
|
|
2186
|
+
record = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
1809
2187
|
if (!record) {
|
|
1810
|
-
throw
|
|
2188
|
+
throw unknownScopeForTenant(tenantId, scopeId);
|
|
1811
2189
|
}
|
|
1812
2190
|
this.assertServedHere(record, scopeId, verb);
|
|
1813
2191
|
}
|
|
1814
2192
|
await this.assertLive(tenantId, scopeId);
|
|
1815
2193
|
await this.migrateAndRecord(scopeId);
|
|
2194
|
+
return record;
|
|
2195
|
+
}
|
|
2196
|
+
/**
|
|
2197
|
+
* Whether this scope is held inert (#2005): a fork, a snapshot or a preview, which causes no
|
|
2198
|
+
* outbound effects and consumes no other vertical's events (#2004). The one primacy question
|
|
2199
|
+
* every such door asks. Where there is a directory its record answers (`record`, the one
|
|
2200
|
+
* `peerScopeGate` returns, when the caller has already read it). A CP-less host has none, so it asks the scope's own storage
|
|
2201
|
+
* whether it is classified a copy (`_substrat_copy_origin`'s `is_copy`, #2009, set on the
|
|
2202
|
+
* directory's word): a preview and a snapshot reach a hosted vertical as restores the
|
|
2203
|
+
* platform marks.
|
|
2204
|
+
*/
|
|
2205
|
+
isInertScope(tenantId, scopeId, record) {
|
|
2206
|
+
if (this.cpLess)
|
|
2207
|
+
return this.scopeStub(scopeId).isCopy();
|
|
2208
|
+
if (record)
|
|
2209
|
+
return Promise.resolve(!isPrimaryScopeRow(record));
|
|
2210
|
+
return this.cp.getScopeRecord(tenantId, scopeId).then((row) => !isPrimaryScopeRow(row));
|
|
1816
2211
|
}
|
|
1817
2212
|
/**
|
|
1818
2213
|
* The capability's attachment door (#1686) — `getCapabilityScope`'s gate, then the
|
|
@@ -1875,7 +2270,7 @@ export class CloudflareScopeHost {
|
|
|
1875
2270
|
async capabilityScopeStub(tenantId, scopeId, verb) {
|
|
1876
2271
|
const rec = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
1877
2272
|
if (!rec)
|
|
1878
|
-
throw
|
|
2273
|
+
throw unknownScopeForTenant(tenantId, scopeId);
|
|
1879
2274
|
if (this.servesScopesElsewhereNow && rec.vertical !== null) {
|
|
1880
2275
|
throw substratError('unavailable', `${verb} cannot reach scope ${scopeId}: it is served by the '${rec.vertical}' ` +
|
|
1881
2276
|
'deployment, and platform capability verbs are not delegated there yet');
|
|
@@ -1921,7 +2316,9 @@ export class CloudflareScopeHost {
|
|
|
1921
2316
|
if (!rec || rec.status !== 'active')
|
|
1922
2317
|
return report;
|
|
1923
2318
|
}
|
|
1924
|
-
else if (await this.
|
|
2319
|
+
else if (lifecycleRefusal(await this.validateScopeAccess(tenantId, scopeId)) !== null) {
|
|
2320
|
+
// #2016: the pair is held to the scope's own record first, so a sweep roster entry whose
|
|
2321
|
+
// tenant disagrees with the scope is refused before the probe reads or writes anything.
|
|
1925
2322
|
// #1713: a held scope emits nothing, so judging it stale would only raise a false alarm.
|
|
1926
2323
|
return report;
|
|
1927
2324
|
}
|
|
@@ -1963,13 +2360,15 @@ export class CloudflareScopeHost {
|
|
|
1963
2360
|
return report;
|
|
1964
2361
|
// Only run on a live scope of this tenant; a scope archived between the sweep's
|
|
1965
2362
|
// enumeration and here simply has nothing due. A CP-less host has no directory
|
|
1966
|
-
// to ask (#461), so it reads the lifecycle the platform delivered to the scope (#1713)
|
|
2363
|
+
// to ask (#461), so it reads the lifecycle the platform delivered to the scope (#1713), in
|
|
2364
|
+
// the same call that holds the pair to the scope's own record (#2016): a roster entry whose
|
|
2365
|
+
// tenant disagrees is refused before a grant, a cadence row or a run is read or written.
|
|
1967
2366
|
if (!this.cpLess) {
|
|
1968
2367
|
const rec = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
1969
2368
|
if (!rec || rec.status !== 'active')
|
|
1970
2369
|
return report;
|
|
1971
2370
|
}
|
|
1972
|
-
else if (await this.
|
|
2371
|
+
else if (lifecycleRefusal(await this.validateScopeAccess(tenantId, scopeId)) !== null) {
|
|
1973
2372
|
// #1713: the lifecycle the platform delivered holds the scope. Every schedule is
|
|
1974
2373
|
// `skipped` and no cadence row moves, as under the kill switch, so a schedule that
|
|
1975
2374
|
// came due meanwhile fires once on the first pass after the scope is live again.
|
|
@@ -1986,15 +2385,15 @@ export class CloudflareScopeHost {
|
|
|
1986
2385
|
// never held the module's grant (a foreign vertical's) is a quiet no-op, exactly as
|
|
1987
2386
|
// before. A scope switched OFF reports every schedule `skipped`, never `failed`, and
|
|
1988
2387
|
// does not touch its cadence rows — so a restore fires a due schedule on the next pass.
|
|
1989
|
-
|
|
1990
|
-
|
|
2388
|
+
// #1819: a scope rewound to before its switch was pulled answers `on`, and the rewind held
|
|
2389
|
+
// the module outside the scope. Both are read by the system door's own gate (#1834), which
|
|
2390
|
+
// every fire below then goes through, pinned to the instance this read came from.
|
|
2391
|
+
const gate = await this.doorGate('system', scopeId, moduleId);
|
|
2392
|
+
if (gate.state === 'ungranted')
|
|
1991
2393
|
return report;
|
|
1992
|
-
|
|
1993
|
-
|
|
1994
|
-
|
|
1995
|
-
if (hold.error)
|
|
1996
|
-
report.errors.push({ operation: 'switch-hold', error: hold.error });
|
|
1997
|
-
if (state === 'off' || hold.held) {
|
|
2394
|
+
if (gate.error)
|
|
2395
|
+
report.errors.push({ operation: 'switch-hold', error: gate.error });
|
|
2396
|
+
if (gate.state === 'off' || gate.held) {
|
|
1998
2397
|
for (const schedule of schedules) {
|
|
1999
2398
|
report.skipped += 1;
|
|
2000
2399
|
report.runs.push({ operation: schedule.operation, outcome: 'skipped' });
|
|
@@ -2004,6 +2403,7 @@ export class CloudflareScopeHost {
|
|
|
2004
2403
|
}
|
|
2005
2404
|
const now = Date.now();
|
|
2006
2405
|
const lines = asyncLinePass();
|
|
2406
|
+
let door;
|
|
2007
2407
|
for (const schedule of schedules) {
|
|
2008
2408
|
const last = await stub.scheduleLastRun(schedule.operation);
|
|
2009
2409
|
const lastRun = last ? Date.parse(last) : null;
|
|
@@ -2023,7 +2423,10 @@ export class CloudflareScopeHost {
|
|
|
2023
2423
|
let emitted;
|
|
2024
2424
|
let failure;
|
|
2025
2425
|
try {
|
|
2026
|
-
|
|
2426
|
+
// The gate above already answered for this pass; a fire that meets a restarted scope
|
|
2427
|
+
// is gated again by the door (#1834).
|
|
2428
|
+
door ??= await this.openSystemDoor(moduleId, tenantId, scopeId, gate);
|
|
2429
|
+
const scope = this.buildStub(tenantId, scopeId, undefined, undefined, door);
|
|
2027
2430
|
await scope.invoke(schedule.operation, schedule.input, { invocationId, onEmitted: (r) => (emitted = r) });
|
|
2028
2431
|
report.fired += 1;
|
|
2029
2432
|
}
|
|
@@ -2058,7 +2461,13 @@ export class CloudflareScopeHost {
|
|
|
2058
2461
|
return report;
|
|
2059
2462
|
}
|
|
2060
2463
|
/** The stub body, shared by the principal, connection, system and impersonation doors. */
|
|
2061
|
-
buildStub(tenantId, scopeId, principal, connectionId,
|
|
2464
|
+
buildStub(tenantId, scopeId, principal, connectionId,
|
|
2465
|
+
/**
|
|
2466
|
+
* #1834: the door this stub acts through. The ONLY way to make a stub acting as
|
|
2467
|
+
* `system:<moduleId>`: the module is the door's, and every invoke is gated and pinned by it.
|
|
2468
|
+
* #2029: a peer stub's door too (`openDoor('peer', …)`), gating and pinning each invoke the same way.
|
|
2469
|
+
*/
|
|
2470
|
+
door, options,
|
|
2062
2471
|
/**
|
|
2063
2472
|
* K-42: the session this stub acts under. Re-read from the directory on
|
|
2064
2473
|
* EVERY invoke rather than captured here — a stub is a capability and
|
|
@@ -2079,6 +2488,7 @@ export class CloudflareScopeHost {
|
|
|
2079
2488
|
*/
|
|
2080
2489
|
verticalCaller) {
|
|
2081
2490
|
const stub = this.scopeStub(scopeId);
|
|
2491
|
+
const systemModuleId = door?.kind === 'system' ? door.key : undefined;
|
|
2082
2492
|
const cp = this.cp;
|
|
2083
2493
|
const operationEntitlement = this.operationEntitlement;
|
|
2084
2494
|
// The DO needs SOME principal-shaped value for `ctx.principal`; for a
|
|
@@ -2133,7 +2543,15 @@ export class CloudflareScopeHost {
|
|
|
2133
2543
|
}
|
|
2134
2544
|
// K-42: resolved per invoke, so expiry and `endImpersonation` both bite.
|
|
2135
2545
|
const session = sessionId === undefined ? undefined : await this.resolveImpersonation(sessionId, tenantId, scopeId);
|
|
2136
|
-
const
|
|
2546
|
+
const send = (systemDoorInstance) => stub.invoke(operation, input, asPrincipalId, tenantId, scopeId, connectionId, requiredKey, systemModuleId, true, invokeOptions, session, capabilitySession, verticalCaller, systemDoorInstance);
|
|
2547
|
+
// #1834: through the door (a module's, or #2029 a peer's), pinned to the instance its gate
|
|
2548
|
+
// read. A missed pin comes back as an answer, which `through` re-gates on.
|
|
2549
|
+
const envelope = door
|
|
2550
|
+
? await door.through(async (instance) => {
|
|
2551
|
+
const sent = await send(instance);
|
|
2552
|
+
return sent.systemDoorMoved === true ? SYSTEM_DOOR_MOVED : sent;
|
|
2553
|
+
})
|
|
2554
|
+
: await send();
|
|
2137
2555
|
// The operation failed and the DO handed the error back as DATA — so it still
|
|
2138
2556
|
// has its code and extensions, which a throw across this boundary would have
|
|
2139
2557
|
// stripped down to a message (#113 §3). Rethrown here, where the caller expects
|
|
@@ -2182,6 +2600,12 @@ export class CloudflareScopeHost {
|
|
|
2182
2600
|
throw substratError('unavailable', `${operation} ran on a scope host that did not understand peer callers. ` +
|
|
2183
2601
|
'Retry once the scope has been migrated');
|
|
2184
2602
|
}
|
|
2603
|
+
// #1834, the same shape: a DO too old to know the door's pin ran the call without
|
|
2604
|
+
// checking it met the storage the door's gate read.
|
|
2605
|
+
if (door !== undefined && envelope.systemDoor?.honoured !== true) {
|
|
2606
|
+
throw substratError('unavailable', `${operation} ran on a scope host that did not understand the system door's pin. ` +
|
|
2607
|
+
'Retry once the scope has been migrated');
|
|
2608
|
+
}
|
|
2185
2609
|
if (session !== undefined && envelope.impersonation?.honoured !== true) {
|
|
2186
2610
|
throw substratError('unavailable', `${operation} ran on a scope host that did not understand impersonation — the ` +
|
|
2187
2611
|
'operation may have run unrecorded and unbounded. Retry once the scope has ' +
|
|
@@ -2192,11 +2616,13 @@ export class CloudflareScopeHost {
|
|
|
2192
2616
|
// this scope's outbox — and draining anyway would run executors and
|
|
2193
2617
|
// connectors as a side effect of a session that may not have side effects.
|
|
2194
2618
|
// Whatever the outbox already held is the drain sweep's own backstop.
|
|
2619
|
+
// #1184: what this call's executors did inline, for `onExecutorOutcomes`.
|
|
2620
|
+
const executorOutcomes = invokeOptions?.onExecutorOutcomes ? [] : undefined;
|
|
2195
2621
|
const drained = session?.mode === 'read-only'
|
|
2196
2622
|
? { attempted: 0, delivered: 0, retrying: 0, deadLettered: 0, routedToPlatform: 0 }
|
|
2197
2623
|
// #1525: this drain is part of the call that emitted the events — the
|
|
2198
2624
|
// coordinator's half of the post-commit tail the DO ran the consumers in.
|
|
2199
|
-
: await this.drainExecutors(tenantId, scopeId, invokeOptions?.invocationId ?? null);
|
|
2625
|
+
: await this.drainExecutors(tenantId, scopeId, invokeOptions?.invocationId ?? null, executorOutcomes);
|
|
2200
2626
|
// #458: the operation committed having enqueued platform intents — tell the
|
|
2201
2627
|
// caller's harness so it can flag the response for the router kick (#381).
|
|
2202
2628
|
// Routed connector deliveries (#574 phase 3) count too: the inline drain just
|
|
@@ -2213,6 +2639,9 @@ export class CloudflareScopeHost {
|
|
|
2213
2639
|
invokeOptions?.onIdempotentReplay?.();
|
|
2214
2640
|
if (envelope.emitted)
|
|
2215
2641
|
invokeOptions?.onEmitted?.(envelope.emitted);
|
|
2642
|
+
if (executorOutcomes && session?.mode !== 'read-only' && !envelope.idempotency?.replayed) {
|
|
2643
|
+
invokeOptions?.onExecutorOutcomes?.(executorOutcomes);
|
|
2644
|
+
}
|
|
2216
2645
|
return envelope.result;
|
|
2217
2646
|
},
|
|
2218
2647
|
};
|
|
@@ -2305,7 +2734,7 @@ export class CloudflareScopeHost {
|
|
|
2305
2734
|
*/
|
|
2306
2735
|
const requireOrg = async (tenant, id) => {
|
|
2307
2736
|
if (!(await this.cp.readOrg(tenant, id))) {
|
|
2308
|
-
throw
|
|
2737
|
+
throw substratError('not_found', `unknown org ${id} in tenant ${tenant}`);
|
|
2309
2738
|
}
|
|
2310
2739
|
};
|
|
2311
2740
|
const mapScope = (r) => scopeSchema.parse({
|
|
@@ -2392,202 +2821,224 @@ export class CloudflareScopeHost {
|
|
|
2392
2821
|
}
|
|
2393
2822
|
};
|
|
2394
2823
|
/**
|
|
2395
|
-
*
|
|
2824
|
+
* Where one scope's kill switch of one kind lands — the ONE rule the operator's switch and
|
|
2825
|
+
* the #1674 re-assert share, so a re-assert moves exactly the switch an operator's OFF would
|
|
2826
|
+
* have. `kind` is a module's schedule switch (#1666) or a peer's (#1706, #2029); `key` is the
|
|
2827
|
+
* module id or the peer's slug.
|
|
2396
2828
|
*
|
|
2397
2829
|
* Where the write lands is the whole point of the delegation branch. The shared control
|
|
2398
|
-
* plane's own `SCOPE` namespace is the module-less placeholder, so a hosted scope's
|
|
2399
|
-
*
|
|
2400
|
-
* switch pulled while every schedule kept firing
|
|
2401
|
-
*
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
* #1706: move one PEER's kill switch on one scope — see `HostAdmin.revokeFromPeer`. The
|
|
2405
|
-
* schedule switch's audit discipline (intent first, outcome after, every attempt) and,
|
|
2406
|
-
* since part 3, its delegation: a scope served by a vertical's own deployment is switched
|
|
2407
|
-
* THERE, over `PeerSwitchDelegation`. Without one configured the scope is still refused
|
|
2408
|
-
* `unavailable` (`peerScopeGate`) rather than switched in the placeholder namespace.
|
|
2830
|
+
* plane's own `SCOPE` namespace is the module-less placeholder, so a hosted scope's switch
|
|
2831
|
+
* has to be moved in the deployment serving it, over that kind's delegation; writing it here
|
|
2832
|
+
* would report a switch pulled while every schedule kept firing, or every peer call kept
|
|
2833
|
+
* being admitted. A scope bound to no vertical (#1666 review) has no deployment to delegate
|
|
2834
|
+
* to: its store is the DO here, so the switch moves (or answers `held: false`) here too.
|
|
2835
|
+
* Delegating it would throw "no deployment serving scope" instead.
|
|
2409
2836
|
*/
|
|
2410
|
-
const
|
|
2411
|
-
const input = peerSwitch.parse(raw);
|
|
2412
|
-
const { tenantId, scopeId } = input.node;
|
|
2837
|
+
const switchTarget = async (kind, tenantId, scopeId) => {
|
|
2413
2838
|
let vertical = null;
|
|
2414
2839
|
if (!this.cpLess) {
|
|
2415
2840
|
const rec = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
2416
2841
|
if (!rec)
|
|
2417
|
-
throw
|
|
2842
|
+
throw unknownScopeForTenant(tenantId, scopeId);
|
|
2418
2843
|
vertical = rec.vertical;
|
|
2419
2844
|
}
|
|
2420
|
-
const
|
|
2421
|
-
|
|
2422
|
-
|
|
2423
|
-
|
|
2424
|
-
//
|
|
2425
|
-
//
|
|
2426
|
-
//
|
|
2427
|
-
//
|
|
2428
|
-
//
|
|
2429
|
-
|
|
2430
|
-
|
|
2431
|
-
|
|
2432
|
-
|
|
2433
|
-
|
|
2434
|
-
|
|
2435
|
-
|
|
2436
|
-
|
|
2437
|
-
|
|
2438
|
-
|
|
2439
|
-
|
|
2440
|
-
|
|
2441
|
-
|
|
2442
|
-
|
|
2443
|
-
|
|
2444
|
-
|
|
2445
|
-
|
|
2446
|
-
|
|
2447
|
-
|
|
2448
|
-
|
|
2449
|
-
|
|
2450
|
-
|
|
2451
|
-
|
|
2452
|
-
|
|
2453
|
-
|
|
2454
|
-
|
|
2455
|
-
|
|
2456
|
-
|
|
2457
|
-
|
|
2458
|
-
|
|
2459
|
-
|
|
2460
|
-
|
|
2461
|
-
|
|
2462
|
-
|
|
2463
|
-
|
|
2464
|
-
|
|
2465
|
-
|
|
2466
|
-
|
|
2467
|
-
|
|
2468
|
-
vertical: input.vertical,
|
|
2469
|
-
calls: to,
|
|
2470
|
-
changed: outcome.changed,
|
|
2471
|
-
permissions: outcome.permissions,
|
|
2845
|
+
const hosted = this.cpLess || vertical !== null;
|
|
2846
|
+
const systemDelegation = kind === 'system' && hosted ? this.systemSwitchDelegation : undefined;
|
|
2847
|
+
const peerDelegation = kind === 'peer' && hosted ? this.peerSwitchDelegation : undefined;
|
|
2848
|
+
const delegated = systemDelegation !== undefined || peerDelegation !== undefined;
|
|
2849
|
+
// #1819: a scope whose store is here was rewound here, so `switchInScope` releases its
|
|
2850
|
+
// hold here too. A delegated move releases in the deployment serving the scope.
|
|
2851
|
+
// #1823 (#2030 for a peer): a subject whose only authority on the scope is a TENANT-level
|
|
2852
|
+
// grant has nothing in the scope's storage for the switch to find, so the directory says
|
|
2853
|
+
// whether it is held. A caller moving several reads them in one call and passes each in.
|
|
2854
|
+
// An OFF of a tenant-held MODULE is refused, after the far end answers, unless that answer
|
|
2855
|
+
// attests the evaluator that denies a tenant-level grant (`deniesTenantGrants`). A
|
|
2856
|
+
// deployment built before #1823 drops `tenantHeld`, still answers `held: true` (the module's
|
|
2857
|
+
// scope-level grants were there to switch), and keeps authorizing the tenant-level grant —
|
|
2858
|
+
// so its `held` alone would have the platform record a scope off that is not. A peer needs
|
|
2859
|
+
// no attestation: its door refuses on the marker in every build that has the peer switch at
|
|
2860
|
+
// all (`admitPeer` and `switchPeer` arrived together, #1714). Every OFF the platform drives
|
|
2861
|
+
// moves through here: an operator's (`switchSubjectAt`) and a re-assert's, which every carry
|
|
2862
|
+
// ends with.
|
|
2863
|
+
//
|
|
2864
|
+
// #2045 (Codex r3): whether a delegated deployment honours the switch fence is settled
|
|
2865
|
+
// BEFORE anything moves (`attestFence`, once per target), never after the fact by putting a
|
|
2866
|
+
// move back. Without the fence an ON is refused and an OFF goes through unfenced (see
|
|
2867
|
+
// `UnattestedSwitch`). A move a fenced deployment answers without the attestation still
|
|
2868
|
+
// throws, as defence in depth against a rollback between the preflight and the move.
|
|
2869
|
+
const delegation = systemDelegation ?? peerDelegation;
|
|
2870
|
+
let attested;
|
|
2871
|
+
const attestFence = () => (attested ??= delegation ? delegation.fenceSupported({ tenantId, scopeId }) : Promise.resolve(true));
|
|
2872
|
+
const move = async (key, to, at, tenantHeld,
|
|
2873
|
+
/** #2045: the switch call's fence — its operation id, or for a re-assert the record's. */
|
|
2874
|
+
fence) => {
|
|
2875
|
+
const fenced = await attestFence();
|
|
2876
|
+
if (!fenced && to === 'on')
|
|
2877
|
+
throw new UnattestedSwitch(unfencedMessage(scopeId));
|
|
2878
|
+
const held = tenantHeld ?? (await this.cp.tenantHeldOf(kind, tenantId, [key], at)).length > 0;
|
|
2879
|
+
const outcome = await (systemDelegation
|
|
2880
|
+
? systemDelegation.switch({ tenantId, scopeId, moduleId: key, to, tenantHeld: held, fence })
|
|
2881
|
+
: peerDelegation
|
|
2882
|
+
? peerDelegation.switch({ tenantId, scopeId, vertical: key, to, tenantHeld: held, fence })
|
|
2883
|
+
: this.switchInScope(kind, scopeId, key, to, at, held, fence));
|
|
2884
|
+
// An unfenced OFF answers without `fenced`, and that is what the callers read it by.
|
|
2885
|
+
if (fenced && outcome.fenced !== true)
|
|
2886
|
+
throw new UnattestedSwitch(unfencedMessage(scopeId));
|
|
2887
|
+
if (outcome.superseded)
|
|
2888
|
+
return outcome;
|
|
2889
|
+
if (kind === 'system' && to === 'off' && held && outcome.held && outcome.deniesTenantGrants !== true) {
|
|
2890
|
+
throw new UnattestedSwitch(undeniedMessage(scopeId, key));
|
|
2891
|
+
}
|
|
2892
|
+
return outcome;
|
|
2472
2893
|
};
|
|
2894
|
+
return { vertical, delegated, move, attestFence };
|
|
2473
2895
|
};
|
|
2474
2896
|
/**
|
|
2475
|
-
*
|
|
2476
|
-
*
|
|
2477
|
-
*
|
|
2478
|
-
*
|
|
2479
|
-
* would throw "no deployment serving scope" instead. A scope WITH a vertical still
|
|
2480
|
-
* delegates, and still fails loudly when none serves it.
|
|
2897
|
+
* One kill switch moved on one scope — the ONE body `revokeFromSystem` / `restoreToSystem`
|
|
2898
|
+
* (#1666) and `revokeFromPeer` / `restoreToPeer` (#1706) share, so the two levers cannot
|
|
2899
|
+
* disagree about the record, the audit, the readback or the undo (#2029). The audit row is
|
|
2900
|
+
* written HERE either way — a deployment's host is CP-less, and its `recordAdmin` is a no-op.
|
|
2481
2901
|
*/
|
|
2482
|
-
const
|
|
2483
|
-
|
|
2484
|
-
|
|
2485
|
-
|
|
2486
|
-
|
|
2487
|
-
|
|
2488
|
-
|
|
2902
|
+
const switchSubjectAt = async (actor, kind, node, key, reason, to) => {
|
|
2903
|
+
const { tenantId, scopeId } = node;
|
|
2904
|
+
const { vertical, delegated, move, attestFence } = await switchTarget(kind, tenantId, scopeId);
|
|
2905
|
+
const action = switchActionOf(kind, to);
|
|
2906
|
+
if (kind === 'peer') {
|
|
2907
|
+
// The peer door's gate (#1706): with a delegation, its lifecycle half (a suspended tenant
|
|
2908
|
+
// or scope cannot be switched), the `scopeStub` half being what the delegation replaces.
|
|
2909
|
+
// Without one, a hosted scope is refused `unavailable` rather than switched in the
|
|
2910
|
+
// placeholder namespace, where it would tombstone nothing a peer call reads.
|
|
2911
|
+
if (delegated)
|
|
2912
|
+
await this.validateScopeAccess(tenantId, scopeId);
|
|
2913
|
+
else
|
|
2914
|
+
await this.peerScopeGate(tenantId, scopeId, action);
|
|
2489
2915
|
}
|
|
2490
|
-
const delegation = this.cpLess || vertical !== null ? this.systemSwitchDelegation : undefined;
|
|
2491
|
-
// #1819: a scope whose store is here was rewound here, so `switchInScope` releases its
|
|
2492
|
-
// hold here too. A delegated move releases in the deployment serving the scope.
|
|
2493
|
-
const move = (moduleId, to, at) => delegation
|
|
2494
|
-
? delegation.switch({ tenantId, scopeId, moduleId, to })
|
|
2495
|
-
: this.switchInScope(scopeId, moduleId, to, at);
|
|
2496
|
-
return { vertical, move };
|
|
2497
|
-
};
|
|
2498
|
-
const switchSystem = async (actor, raw, to) => {
|
|
2499
|
-
const input = systemSwitch.parse(raw);
|
|
2500
|
-
const { tenantId, scopeId } = input.node;
|
|
2501
|
-
const { vertical, move } = await systemSwitchTarget(tenantId, scopeId);
|
|
2502
2916
|
// AUDIT FIRST (#1666 review): the intent row lands before anything moves, and the
|
|
2503
2917
|
// outcome row after — every attempt, a repeat included. The scope's store and the
|
|
2504
2918
|
// admin log are separate, so no order makes the pair atomic; this one fails toward
|
|
2505
2919
|
// "an intent with no recorded outcome" and never toward "a switch that moved with no
|
|
2506
2920
|
// audit row". A retry after a crash re-audits even though it answers `changed: false`.
|
|
2507
2921
|
const operationId = ulid();
|
|
2508
|
-
const action = to === 'off' ? 'revokeFromSystem' : 'restoreToSystem';
|
|
2509
2922
|
const target = { tenantId, scopeId, vertical };
|
|
2510
|
-
const base = { operationId,
|
|
2511
|
-
await this.recordAdmin(actor, action, target, null, { ...base, phase: 'intent', reason
|
|
2512
|
-
// The directory's record (#1674): ON before the scope moves, OFF after it held — see
|
|
2513
|
-
// `recordSystemSwitchedOn` for why that order is the safe one.
|
|
2514
|
-
const at = new Date().toISOString();
|
|
2515
|
-
const record = { tenantId, scopeId, moduleId: input.moduleId, actor, reason: input.reason, operationId, at };
|
|
2923
|
+
const base = { operationId, ...switchAuditSubject(kind, key, to) };
|
|
2924
|
+
await this.recordAdmin(actor, action, target, null, { ...base, phase: 'intent', reason });
|
|
2516
2925
|
const errorOf = (err) => (err instanceof Error ? err.message : String(err));
|
|
2517
|
-
|
|
2518
|
-
|
|
2519
|
-
|
|
2520
|
-
|
|
2521
|
-
|
|
2522
|
-
|
|
2523
|
-
// Nothing has moved: fail the call here, audited, rather than switch a scope on
|
|
2524
|
-
// whose record still says off (the next reconcile would switch it back off).
|
|
2525
|
-
await this.recordAdmin(actor, action, target, null, { ...base, phase: 'failed', error: errorOf(err) }).catch(() => undefined);
|
|
2526
|
-
throw err;
|
|
2926
|
+
// #2045 (Codex r3): the deployment's fence is attested BEFORE anything is recorded or moved. An
|
|
2927
|
+
// ON on a deployment built before it is refused here, with nothing written anywhere but the
|
|
2928
|
+
// audit. An OFF goes on: the kill switch must work on every deployment (see `UnattestedSwitch`).
|
|
2929
|
+
try {
|
|
2930
|
+
if (!(await attestFence()) && to === 'on') {
|
|
2931
|
+
throw substratError('precondition_failed', `${unfencedMessage(scopeId)} Nothing was switched.`);
|
|
2527
2932
|
}
|
|
2528
2933
|
}
|
|
2529
|
-
|
|
2530
|
-
|
|
2531
|
-
|
|
2532
|
-
|
|
2533
|
-
|
|
2534
|
-
|
|
2535
|
-
|
|
2934
|
+
catch (err) {
|
|
2935
|
+
await this.recordAdmin(actor, action, target, null, { ...base, phase: 'refused', error: errorOf(err) }).catch(() => undefined);
|
|
2936
|
+
throw err;
|
|
2937
|
+
}
|
|
2938
|
+
// The directory's record (#1674; #2029 for a peer), written BEFORE the scope moves, both
|
|
2939
|
+
// ways — see `recordSwitchedOn` and (#1823) `recordSwitchedOff` for why that order is the
|
|
2940
|
+
// safe one. #2045 (Codex r3): the same write marks the subject owed and clears the scope's
|
|
2941
|
+
// reconcile receipt, so from here on the scope converges to this record whatever happens to
|
|
2942
|
+
// the move. The mark goes only when the scope confirms the move under this call's fence.
|
|
2943
|
+
const at = new Date().toISOString();
|
|
2944
|
+
const record = { kind, tenantId, scopeId, key, actor, reason, operationId, at };
|
|
2945
|
+
let prior;
|
|
2946
|
+
try {
|
|
2947
|
+
prior = to === 'on' ? await this.cp.recordSwitchedOn(record) : await this.cp.recordSwitchedOff(record);
|
|
2948
|
+
}
|
|
2949
|
+
catch (err) {
|
|
2950
|
+
// Nothing has moved: fail the call here, audited. An ON must not switch a scope on
|
|
2951
|
+
// whose record still says off (the next reconcile would switch it back off), and an
|
|
2952
|
+
// OFF must not leave a scope off that the record does not know of (#1823).
|
|
2953
|
+
await this.recordAdmin(actor, action, target, null, { ...base, phase: 'failed', error: errorOf(err) }).catch(() => undefined);
|
|
2954
|
+
throw err;
|
|
2955
|
+
}
|
|
2956
|
+
// #2045: a newer call on this subject has recorded its position already: this one writes
|
|
2957
|
+
// nothing, here or in the scope, and says so.
|
|
2958
|
+
if (recordWriteSuperseded(prior, record)) {
|
|
2959
|
+
await this.recordAdmin(actor, action, target, null, { ...base, phase: 'refused', superseded: true });
|
|
2960
|
+
throw substratError('conflict', switchSupersededMessage(kind, scopeId, key, to));
|
|
2961
|
+
}
|
|
2962
|
+
let outcome;
|
|
2963
|
+
try {
|
|
2536
2964
|
try {
|
|
2537
|
-
await
|
|
2538
|
-
return null;
|
|
2965
|
+
outcome = await move(key, to, at, undefined, operationId);
|
|
2539
2966
|
}
|
|
2540
|
-
catch {
|
|
2541
|
-
|
|
2542
|
-
|
|
2543
|
-
|
|
2544
|
-
|
|
2545
|
-
|
|
2546
|
-
return errorOf(err);
|
|
2547
|
-
}
|
|
2967
|
+
catch (first) {
|
|
2968
|
+
if (first instanceof UnattestedSwitch)
|
|
2969
|
+
throw first;
|
|
2970
|
+
// #2045 (Codex r2): one retry under this call's own fence — idempotent, and refused by
|
|
2971
|
+
// the scope if a newer call has moved it meanwhile.
|
|
2972
|
+
outcome = await move(key, to, new Date().toISOString(), undefined, operationId);
|
|
2548
2973
|
}
|
|
2549
|
-
};
|
|
2550
|
-
let outcome;
|
|
2551
|
-
try {
|
|
2552
|
-
outcome = await move(input.moduleId, to, at);
|
|
2553
2974
|
}
|
|
2554
2975
|
catch (err) {
|
|
2555
|
-
|
|
2556
|
-
//
|
|
2557
|
-
//
|
|
2976
|
+
// The move threw twice, or was answered by a deployment that could not attest it. Either
|
|
2977
|
+
// way it may have landed, and an older call's move may still be in flight behind it, so no
|
|
2978
|
+
// readback can settle the scope and nothing is put back: the record is this call's position
|
|
2979
|
+
// under the newest fence, and its write-ahead mark makes the next re-assert move the scope
|
|
2980
|
+
// to it, under that fence, whichever way it lies now.
|
|
2558
2981
|
await this.recordAdmin(actor, action, target, null, {
|
|
2559
2982
|
...base,
|
|
2560
|
-
phase: 'failed',
|
|
2983
|
+
phase: err instanceof UnattestedSwitch ? 'refused' : 'failed',
|
|
2561
2984
|
error: errorOf(err),
|
|
2562
|
-
|
|
2985
|
+
recordKept: true,
|
|
2986
|
+
reassertOwed: true,
|
|
2563
2987
|
}).catch(() => undefined);
|
|
2988
|
+
if (err instanceof UnattestedSwitch)
|
|
2989
|
+
throw substratError('precondition_failed', err.message);
|
|
2564
2990
|
throw err;
|
|
2565
2991
|
}
|
|
2566
|
-
//
|
|
2567
|
-
//
|
|
2568
|
-
//
|
|
2569
|
-
|
|
2570
|
-
|
|
2571
|
-
|
|
2572
|
-
|
|
2573
|
-
|
|
2992
|
+
// #2045: the scope has applied a newer call on this subject, so this move wrote nothing. The
|
|
2993
|
+
// record is that newer call's too (its write overwrote this one's, mark included), so nothing
|
|
2994
|
+
// is undone and no mark is this call's to clear.
|
|
2995
|
+
if (outcome.superseded) {
|
|
2996
|
+
await this.recordAdmin(actor, action, target, null, { ...base, phase: 'refused', superseded: true });
|
|
2997
|
+
throw substratError('conflict', switchSupersededMessage(kind, scopeId, key, to));
|
|
2998
|
+
}
|
|
2999
|
+
// A call that held nothing moved nothing, so its record write is undone: left `off`, it
|
|
3000
|
+
// would switch the subject off the day it is installed; left `on`, the next reconcile of
|
|
3001
|
+
// a wiped scope would leave it running. Retried once; a failure of both lands on the
|
|
3002
|
+
// outcome row as `recordError`, and the call says so.
|
|
3003
|
+
//
|
|
3004
|
+
// The scope confirmed this call's move (or that there was nothing to move), so its mark goes.
|
|
3005
|
+
// Held nothing, it goes WITH the undo, in one directory transaction: the row put back carries
|
|
3006
|
+
// the prior call's older id, and no later re-assert confirms under a fence older than this
|
|
3007
|
+
// call's mark. An undo that fails twice leaves the record AND the mark this call's, and the
|
|
3008
|
+
// next re-assert confirms both under it. Held, a failed clear is not the call's: the mark only
|
|
3009
|
+
// costs the next re-assert one idempotent move under this same record.
|
|
3010
|
+
//
|
|
3011
|
+
// An unfenced OFF that held something (a deployment built before the fence) confirmed nothing
|
|
3012
|
+
// under its fence, so its mark stays for the first re-assert after the vertical is redeployed.
|
|
3013
|
+
const unfenced = outcome.held && outcome.fenced !== true;
|
|
3014
|
+
let recordError = null;
|
|
3015
|
+
let owedError = null;
|
|
3016
|
+
if (!outcome.held) {
|
|
3017
|
+
const undo = () => this.cp.restoreSwitchRecord(record, prior, true);
|
|
3018
|
+
recordError = await undo().then(() => null, () => undo().then(() => null, errorOf));
|
|
3019
|
+
}
|
|
3020
|
+
else if (!unfenced) {
|
|
3021
|
+
owedError = await this.cp.clearSwitchOwed(kind, tenantId, scopeId, key, operationId).then(() => null, errorOf);
|
|
3022
|
+
}
|
|
2574
3023
|
await this.recordAdmin(actor, action, target, null, {
|
|
2575
3024
|
...base,
|
|
2576
3025
|
phase: outcome.held ? 'applied' : 'refused',
|
|
2577
3026
|
changed: outcome.changed,
|
|
2578
3027
|
permissions: outcome.permissions,
|
|
3028
|
+
...(unfenced ? { unfenced: true, reassertOwed: true } : {}),
|
|
2579
3029
|
...(recordError ? { recordError } : {}),
|
|
3030
|
+
...(owedError ? { owedError } : {}),
|
|
2580
3031
|
});
|
|
2581
3032
|
if (!outcome.held) {
|
|
2582
|
-
throw substratError('not_found',
|
|
2583
|
-
`(check the module id: it is the module's manifest id, e.g. '@substrat-run/engine-absence')` +
|
|
3033
|
+
throw substratError('not_found', switchNotFoundMessage(kind, scopeId, key, to) +
|
|
2584
3034
|
(recordError ? `; and its directory record could not be put back (${recordError})` : ''));
|
|
2585
3035
|
}
|
|
2586
|
-
|
|
2587
|
-
|
|
2588
|
-
|
|
2589
|
-
|
|
2590
|
-
|
|
3036
|
+
return { operationId, outcome };
|
|
3037
|
+
};
|
|
3038
|
+
/** #1666: move one module's schedule switch on one scope — see `HostAdmin.revokeFromSystem`. */
|
|
3039
|
+
const switchSystem = async (actor, raw, to) => {
|
|
3040
|
+
const input = systemSwitch.parse(raw);
|
|
3041
|
+
const { operationId, outcome } = await switchSubjectAt(actor, 'system', input.node, input.moduleId, input.reason, to);
|
|
2591
3042
|
return {
|
|
2592
3043
|
operationId,
|
|
2593
3044
|
moduleId: input.moduleId,
|
|
@@ -2596,6 +3047,18 @@ export class CloudflareScopeHost {
|
|
|
2596
3047
|
permissions: outcome.permissions,
|
|
2597
3048
|
};
|
|
2598
3049
|
};
|
|
3050
|
+
/** #1706: move one PEER's kill switch on one scope — see `HostAdmin.revokeFromPeer`. */
|
|
3051
|
+
const switchPeerAt = async (actor, raw, to) => {
|
|
3052
|
+
const input = peerSwitch.parse(raw);
|
|
3053
|
+
const { operationId, outcome } = await switchSubjectAt(actor, 'peer', input.node, input.vertical, input.reason, to);
|
|
3054
|
+
return {
|
|
3055
|
+
operationId,
|
|
3056
|
+
vertical: input.vertical,
|
|
3057
|
+
calls: to,
|
|
3058
|
+
changed: outcome.changed,
|
|
3059
|
+
permissions: outcome.permissions,
|
|
3060
|
+
};
|
|
3061
|
+
};
|
|
2599
3062
|
/**
|
|
2600
3063
|
* The status read's admin-log join (#1674): the `intent` row of the `revokeFromSystem`
|
|
2601
3064
|
* still in force for each OFF module — actor, reason, when. Paired to its `applied` row
|
|
@@ -2651,7 +3114,7 @@ export class CloudflareScopeHost {
|
|
|
2651
3114
|
if (!this.cpLess) {
|
|
2652
3115
|
const rec = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
2653
3116
|
if (!rec)
|
|
2654
|
-
throw
|
|
3117
|
+
throw unknownScopeForTenant(tenantId, scopeId);
|
|
2655
3118
|
vertical = rec.vertical;
|
|
2656
3119
|
}
|
|
2657
3120
|
// A hosted scope (a vertical bound, on a non-cpLess host) has no local storage worth
|
|
@@ -2668,7 +3131,7 @@ export class CloudflareScopeHost {
|
|
|
2668
3131
|
delegation
|
|
2669
3132
|
? delegation.status({ tenantId, scopeId })
|
|
2670
3133
|
: this.scopeStub(scopeId).systemGrantsStatus().then((rows) => systemScheduleEntry.array().parse(rows)),
|
|
2671
|
-
this.cp.
|
|
3134
|
+
this.cp.switchRecordsOf('system', tenantId, scopeId),
|
|
2672
3135
|
]);
|
|
2673
3136
|
const offModules = new Set(states.filter((s) => s.schedules === 'off').map((s) => s.moduleId));
|
|
2674
3137
|
const explanations = offModules.size > 0
|
|
@@ -2687,70 +3150,137 @@ export class CloudflareScopeHost {
|
|
|
2687
3150
|
};
|
|
2688
3151
|
/**
|
|
2689
3152
|
* #1674: put the directory's OFF positions back into one scope — see
|
|
2690
|
-
* `HostAdmin.reassertSystemSwitches`.
|
|
2691
|
-
* so it lands wherever an operator's
|
|
2692
|
-
*
|
|
3153
|
+
* `HostAdmin.reassertSystemSwitches`. Every kind (#2029): the recorded-off modules, then the
|
|
3154
|
+
* recorded-off peers. Each moves through `switchTarget`, so it lands wherever an operator's
|
|
3155
|
+
* OFF would have — for a hosted scope, over that kind's `/internal/*` switch route.
|
|
2693
3156
|
*/
|
|
2694
3157
|
const reassertSystemSwitchesOf = async (actor, node, opts) => {
|
|
3158
|
+
const results = [];
|
|
3159
|
+
for (const kind of SWITCH_KINDS)
|
|
3160
|
+
results.push(...(await reassertKind(actor, kind, node, opts)));
|
|
3161
|
+
return results;
|
|
3162
|
+
};
|
|
3163
|
+
/** One kind's half of `reassertSystemSwitchesOf`. */
|
|
3164
|
+
const reassertKind = async (actor, kind, node, opts) => {
|
|
2695
3165
|
const { tenantId, scopeId } = node;
|
|
2696
|
-
const { vertical, move } = await
|
|
2697
|
-
const
|
|
3166
|
+
const { vertical, delegated, move, attestFence } = await switchTarget(kind, tenantId, scopeId);
|
|
3167
|
+
const action = reassertActionOf(kind);
|
|
3168
|
+
const target = { tenantId, scopeId, vertical };
|
|
3169
|
+
const recorded = new Map(await this.cp.switchRecordsOf(kind, tenantId, scopeId));
|
|
3170
|
+
// #2045: every switch call's write-ahead mark, with the newest call that wrote it.
|
|
3171
|
+
const owed = new Map(await this.cp.switchesOwedOf(kind, tenantId, scopeId));
|
|
2698
3172
|
// #1742 review: moves the deployment made from a stale list (restored ON after the list
|
|
2699
3173
|
// was read), to undo before the OFF pass below.
|
|
2700
|
-
const reverts = staleCarryReverts(recorded, opts?.appliedInUnit);
|
|
3174
|
+
const reverts = staleCarryReverts(kind, recorded, opts?.appliedInUnit);
|
|
2701
3175
|
// A hosted scope with no delegation configured (Copilot review): this host's own
|
|
2702
3176
|
// namespace is the module-less placeholder, where the switch would answer `held: false`
|
|
2703
3177
|
// quietly — a re-assert reported done, and a receipt written, while the deployment
|
|
2704
|
-
// serving the scope keeps its schedules running
|
|
2705
|
-
// read is, and only when a re-assert is owed: nothing recorded,
|
|
2706
|
-
|
|
2707
|
-
|
|
3178
|
+
// serving the scope keeps its schedules running or its peer admitted. Refused loudly
|
|
3179
|
+
// instead, as the status read is, and only when a re-assert is owed: nothing recorded,
|
|
3180
|
+
// nothing to refuse.
|
|
3181
|
+
const reassertOwed = reverts.length > 0 || owed.size > 0 || [...recorded.values()].includes('off');
|
|
3182
|
+
if (reassertOwed && !this.cpLess && vertical !== null && !delegated) {
|
|
2708
3183
|
throw substratError('unavailable', `no delegation configured for hosted scope ${scopeId} (vertical '${vertical}') — cannot re-assert ` +
|
|
2709
|
-
`its switched-off
|
|
3184
|
+
`its switched-off ${DOOR_WORDS[kind].switched} in the deployment serving it`);
|
|
2710
3185
|
}
|
|
3186
|
+
// #2045 (Codex r3): a subject's mark goes only once the scope has confirmed a move under the
|
|
3187
|
+
// record's fence — a fenced answer that was not superseded. A throw anywhere below leaves
|
|
3188
|
+
// every mark not yet confirmed for the next pass.
|
|
3189
|
+
// An unfenced OFF (a deployment built before the fence) confirms nothing, so its mark stays.
|
|
3190
|
+
const confirm = async (key, fence, outcome) => {
|
|
3191
|
+
if (owed.has(key) && !outcome.superseded && outcome.fenced === true) {
|
|
3192
|
+
await this.cp.clearSwitchOwed(kind, tenantId, scopeId, key, fence);
|
|
3193
|
+
}
|
|
3194
|
+
};
|
|
3195
|
+
// #2045 follow-up: on a deployment built before the fence no ON is sent (see
|
|
3196
|
+
// `UnattestedSwitch`). Every OFF below still moves, and the pass refuses at its end, so a
|
|
3197
|
+
// sweep records no receipt for a scope whose owed ON was not sent. Asked only when an ON is due.
|
|
3198
|
+
const onWithheld = [];
|
|
3199
|
+
const withholdOn = async (key) => {
|
|
3200
|
+
if (await attestFence())
|
|
3201
|
+
return false;
|
|
3202
|
+
onWithheld.push(key);
|
|
3203
|
+
return true;
|
|
3204
|
+
};
|
|
2711
3205
|
const at = new Date().toISOString();
|
|
2712
3206
|
const reverted = new Set();
|
|
2713
|
-
for (const
|
|
3207
|
+
for (const key of reverts) {
|
|
2714
3208
|
// Re-read immediately before the move: a staff OFF that completed since the read above
|
|
2715
|
-
// (record `off`) must not get a transient ON a due schedule could
|
|
2716
|
-
|
|
2717
|
-
|
|
3209
|
+
// (record `off`) must not get a transient ON a due schedule, or a peer call, could use.
|
|
3210
|
+
// #2045: under the fence of the ON the record holds, which is the operator's own call — read
|
|
3211
|
+
// with the position, so it is that ON's fence and no other call's.
|
|
3212
|
+
const current = new Map(await this.cp.switchRecordStatesOf(kind, tenantId, scopeId)).get(key);
|
|
3213
|
+
if (current?.position !== 'on')
|
|
2718
3214
|
continue;
|
|
2719
|
-
const
|
|
2720
|
-
|
|
3215
|
+
const fence = current.fence;
|
|
3216
|
+
if (await withholdOn(key))
|
|
3217
|
+
continue;
|
|
3218
|
+
const outcome = await move(key, 'on', at, undefined, fence);
|
|
3219
|
+
reverted.add(key);
|
|
2721
3220
|
if (outcome.changed) {
|
|
2722
|
-
await this.recordAdmin(actor,
|
|
2723
|
-
operationId: ulid(),
|
|
2724
|
-
...staleCarryRevertRow(moduleId, outcome),
|
|
2725
|
-
});
|
|
3221
|
+
await this.recordAdmin(actor, action, target, null, { operationId: ulid(), ...staleCarryRevertRow(kind, key, outcome) });
|
|
2726
3222
|
}
|
|
3223
|
+
await confirm(key, fence, outcome);
|
|
2727
3224
|
}
|
|
2728
3225
|
// Read AFTER the reverts: an OFF that landed meanwhile is switched off below.
|
|
2729
|
-
const
|
|
3226
|
+
const { keys, tenantHeld, fences } = await this.recordedOff(kind, tenantId, scopeId, at);
|
|
2730
3227
|
// #1742: what the deployment already switched off inside its own unit, audited here —
|
|
2731
|
-
// the move below answers `changed: false` for it and would write no row.
|
|
2732
|
-
//
|
|
2733
|
-
const
|
|
2734
|
-
|
|
2735
|
-
await this.recordAdmin(actor, 'reassertSystemSwitch', { tenantId, scopeId, vertical }, null, {
|
|
2736
|
-
operationId: ulid(),
|
|
2737
|
-
...row,
|
|
2738
|
-
});
|
|
3228
|
+
// the move below answers `changed: false` for it and would write no row. A move the
|
|
3229
|
+
// revert above undid is not credited as an in-unit OFF.
|
|
3230
|
+
for (const row of inUnitMovesToAudit(kind, keys, opts?.appliedInUnit, reverted)) {
|
|
3231
|
+
await this.recordAdmin(actor, action, target, null, { operationId: ulid(), ...row });
|
|
2739
3232
|
}
|
|
3233
|
+
// #2045: every move here carries the fence of the call the record holds, so a scope a newer call
|
|
3234
|
+
// has moved since refuses it. A deployment from before the fence takes the OFF unfenced.
|
|
2740
3235
|
const results = [];
|
|
2741
|
-
|
|
2742
|
-
|
|
3236
|
+
const held = new Set(tenantHeld);
|
|
3237
|
+
for (const key of keys) {
|
|
3238
|
+
const fence = fences.get(key); // `recordedOff` read each key with its own row's fence
|
|
3239
|
+
const outcome = await move(key, 'off', at, held.has(key), fence);
|
|
2743
3240
|
if (outcome.changed) {
|
|
2744
|
-
await this.recordAdmin(actor,
|
|
3241
|
+
await this.recordAdmin(actor, action, target, null, {
|
|
2745
3242
|
operationId: ulid(),
|
|
2746
|
-
|
|
2747
|
-
|
|
2748
|
-
phase: 'applied',
|
|
2749
|
-
changed: true,
|
|
2750
|
-
permissions: outcome.permissions,
|
|
3243
|
+
...reassertOffRow(kind, key, outcome.permissions),
|
|
3244
|
+
...(outcome.fenced === true ? {} : { unfenced: true }),
|
|
2751
3245
|
});
|
|
2752
3246
|
}
|
|
2753
|
-
results.push(
|
|
3247
|
+
results.push(reassertEntry(kind, key, outcome));
|
|
3248
|
+
await confirm(key, fence, outcome);
|
|
3249
|
+
}
|
|
3250
|
+
// A subject marked owed converges to its record in either direction: OFF above, and here a
|
|
3251
|
+
// record of ON — sent whatever the scope reads now, because a scope that reads ON may still
|
|
3252
|
+
// be behind the record's fence, with an older call's OFF in flight that only the fence can
|
|
3253
|
+
// refuse (Codex r3, finding 1). Idempotent: an ON already in place changes nothing and moves
|
|
3254
|
+
// the fence. Every other record of ON keeps #1674's rule and turns nothing on.
|
|
3255
|
+
const settled = new Set([...keys, ...reverted]);
|
|
3256
|
+
const pending = [...owed.keys()].filter((key) => !settled.has(key));
|
|
3257
|
+
if (pending.length) {
|
|
3258
|
+
// Position and fence in one read: an ON is sent only under the fence of the ON it read.
|
|
3259
|
+
const now = new Map(await this.cp.switchRecordStatesOf(kind, tenantId, scopeId));
|
|
3260
|
+
for (const key of pending) {
|
|
3261
|
+
const row = now.get(key);
|
|
3262
|
+
if (row?.position !== 'on') {
|
|
3263
|
+
// No record (a call that held nothing, whose undo removed its row), or one that turned
|
|
3264
|
+
// OFF since the read above, which the next pass moves: nothing to send on this one. A
|
|
3265
|
+
// missing row's mark goes under its own id, so a newer call's mark stays.
|
|
3266
|
+
if (row === undefined)
|
|
3267
|
+
await this.cp.clearSwitchOwed(kind, tenantId, scopeId, key, owed.get(key));
|
|
3268
|
+
continue;
|
|
3269
|
+
}
|
|
3270
|
+
const fence = row.fence;
|
|
3271
|
+
if (await withholdOn(key))
|
|
3272
|
+
continue;
|
|
3273
|
+
const outcome = await move(key, 'on', at, undefined, fence);
|
|
3274
|
+
if (outcome.changed) {
|
|
3275
|
+
await this.recordAdmin(actor, action, target, null, { operationId: ulid(), ...reassertOnRow(kind, key, outcome.permissions) });
|
|
3276
|
+
results.push(reassertEntry(kind, key, outcome));
|
|
3277
|
+
}
|
|
3278
|
+
await confirm(key, fence, outcome);
|
|
3279
|
+
}
|
|
3280
|
+
}
|
|
3281
|
+
if (onWithheld.length) {
|
|
3282
|
+
throw substratError('precondition_failed', `${unfencedMessage(scopeId)} Its recorded OFF positions were applied; the owed ON of ` +
|
|
3283
|
+
`${onWithheld.map((k) => `'${k}'`).join(', ')} was not sent.`);
|
|
2754
3284
|
}
|
|
2755
3285
|
return results;
|
|
2756
3286
|
};
|
|
@@ -2769,7 +3299,7 @@ export class CloudflareScopeHost {
|
|
|
2769
3299
|
if (!this.cpLess) {
|
|
2770
3300
|
const rec = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
2771
3301
|
if (!rec)
|
|
2772
|
-
throw
|
|
3302
|
+
throw unknownScopeForTenant(tenantId, scopeId);
|
|
2773
3303
|
vertical = rec.vertical;
|
|
2774
3304
|
}
|
|
2775
3305
|
if (!this.cpLess && vertical !== null && !this.peerSwitchDelegation) {
|
|
@@ -2837,18 +3367,21 @@ export class CloudflareScopeHost {
|
|
|
2837
3367
|
unassignRole: async (actor, assignment) => {
|
|
2838
3368
|
// Tombstone (K-21) — the checker skips revoked rows. A no-op returns false so
|
|
2839
3369
|
// a repeat unassign stays silent (no second audit row, no needless fan-out).
|
|
2840
|
-
const subject = `principal:${assignment.principalId}`;
|
|
2841
3370
|
const relation = `role:${assignment.roleKey}`;
|
|
2842
|
-
const
|
|
2843
|
-
|
|
2844
|
-
|
|
2845
|
-
|
|
3371
|
+
const { tenantId, scopeId } = assignment.node;
|
|
3372
|
+
if (!scopeId) {
|
|
3373
|
+
// One ControlPlaneDO method (#1184): the revoke, the removal fence (raised even when
|
|
3374
|
+
// nothing was held) and the audit row, so no add can land between any two of them.
|
|
3375
|
+
const changed = await this.cp.revokeAndFence(tenantId, assignment.principalId, relation, `tenant:${tenantId}`, this.adminEntry(actor, 'unassignRole', { tenantId, scopeId: null }, assignment, null));
|
|
3376
|
+
// A tenant-level revoke changes the projected set — the tombstone must reach scopes.
|
|
3377
|
+
if (changed)
|
|
3378
|
+
await this.fanOut(tenantId);
|
|
3379
|
+
return;
|
|
3380
|
+
}
|
|
3381
|
+
const changed = await this.scopeStub(scopeId).revokeTuple(`principal:${assignment.principalId}`, relation, `scope:${scopeId}`, new Date().toISOString());
|
|
2846
3382
|
if (!changed)
|
|
2847
3383
|
return;
|
|
2848
|
-
await this.recordAdmin(actor, 'unassignRole', { tenantId
|
|
2849
|
-
// A tenant-level revoke changes the projected set — the tombstone must reach scopes.
|
|
2850
|
-
if (!assignment.node.scopeId)
|
|
2851
|
-
await this.fanOut(assignment.node.tenantId);
|
|
3384
|
+
await this.recordAdmin(actor, 'unassignRole', { tenantId, scopeId }, assignment, null);
|
|
2852
3385
|
},
|
|
2853
3386
|
grant: async (actor, raw) => {
|
|
2854
3387
|
// Parsed like its `grantToConnection`/`grantToSystem` siblings, not taken on
|
|
@@ -2968,6 +3501,19 @@ export class CloudflareScopeHost {
|
|
|
2968
3501
|
await this.recordAccess(actor, 'listSystemSwitches', { tenantId: filter?.tenantId ?? null }, filter ?? null, rows.length);
|
|
2969
3502
|
return rows.map((r) => systemSwitchRecord.parse(r));
|
|
2970
3503
|
},
|
|
3504
|
+
tenantHeldSystemModules: async (actor, tenantId, moduleIds) => {
|
|
3505
|
+
const held = moduleIds.length
|
|
3506
|
+
? await this.cp.tenantHeldOf('system', tenantId, moduleIds, new Date().toISOString())
|
|
3507
|
+
: [];
|
|
3508
|
+
await this.recordAccess(actor, 'tenantHeldSystemModules', { tenantId }, { moduleIds: [...moduleIds] }, held.length);
|
|
3509
|
+
return held;
|
|
3510
|
+
},
|
|
3511
|
+
peerSwitchCarry: async (actor, node) => {
|
|
3512
|
+
const { keys: switchedOffPeers, tenantHeld: tenantHeldPeers, fences: all, } = await this.recordedOff('peer', node.tenantId, node.scopeId, new Date().toISOString());
|
|
3513
|
+
const fences = Object.fromEntries(switchedOffPeers.map((v) => [v, all.get(v)]));
|
|
3514
|
+
await this.recordAccess(actor, 'peerSwitchCarry', node, null, switchedOffPeers.length);
|
|
3515
|
+
return { switchedOffPeers, tenantHeldPeers, fences };
|
|
3516
|
+
},
|
|
2971
3517
|
reassertSystemSwitches: reassertSystemSwitchesOf,
|
|
2972
3518
|
// #1672 — the platform's two capability verbs. Audited AFTER the write, on both, and
|
|
2973
3519
|
// the failure each leaves is the safe one: a mint whose audit row did not land never
|
|
@@ -3033,7 +3579,7 @@ export class CloudflareScopeHost {
|
|
|
3033
3579
|
// another tenant's traffic. Exception: the holder is ARCHIVED or REAPED (a
|
|
3034
3580
|
// deleted app, storage since wiped) — it has released the name, so the rebind
|
|
3035
3581
|
// reclaims it.
|
|
3036
|
-
throw
|
|
3582
|
+
throw substratError('conflict', `hostname '${parsed.hostname}' is already bound to another scope`);
|
|
3037
3583
|
}
|
|
3038
3584
|
// Exactly one canonical per (scope, surface).
|
|
3039
3585
|
if (parsed.canonical)
|
|
@@ -3054,7 +3600,7 @@ export class CloudflareScopeHost {
|
|
|
3054
3600
|
const hostname = raw.toLowerCase(); // DNS is case-insensitive; the map is normalized
|
|
3055
3601
|
const row = await this.cp.readHostname(hostname);
|
|
3056
3602
|
if (!row)
|
|
3057
|
-
throw
|
|
3603
|
+
throw substratError('not_found', `unknown hostname '${hostname}'`);
|
|
3058
3604
|
if (row.status === status)
|
|
3059
3605
|
return; // idempotent, unaudited
|
|
3060
3606
|
await this.cp.setHostnameStatus(hostname, status, note ?? null);
|
|
@@ -3064,7 +3610,7 @@ export class CloudflareScopeHost {
|
|
|
3064
3610
|
const hostname = raw.toLowerCase(); // DNS is case-insensitive; the map is normalized
|
|
3065
3611
|
const row = await this.cp.readHostname(hostname);
|
|
3066
3612
|
if (!row)
|
|
3067
|
-
throw
|
|
3613
|
+
throw substratError('not_found', `unknown hostname '${hostname}'`);
|
|
3068
3614
|
// A poll that finds nothing changed (same status, same records, id already set)
|
|
3069
3615
|
// is not an event — skip the write and the audit entry, so the reconcile sweep
|
|
3070
3616
|
// does not flood the admin log with no-op rows every interval.
|
|
@@ -3150,7 +3696,7 @@ export class CloudflareScopeHost {
|
|
|
3150
3696
|
if (existing.owner_tenant !== parsed.ownerTenant) {
|
|
3151
3697
|
throw substratError('conflict', `vertical '${parsed.slug}' is owned by ${existing.owner_tenant ?? 'the platform'}, not ${parsed.ownerTenant ?? 'the platform'}`);
|
|
3152
3698
|
}
|
|
3153
|
-
throw
|
|
3699
|
+
throw substratError('conflict', `vertical '${parsed.slug}' is already registered as ${existing.source}`);
|
|
3154
3700
|
}
|
|
3155
3701
|
await this.cp.insertVertical(parsed.slug, parsed.name, parsed.source, parsed.ownerTenant, envSpecJson, installSpecJson, parsed.listed ? 1 : 0, new Date().toISOString());
|
|
3156
3702
|
await this.recordAdmin(actor, 'registerVertical', { tenantId: null }, null, parsed);
|
|
@@ -3331,11 +3877,11 @@ export class CloudflareScopeHost {
|
|
|
3331
3877
|
// nothing to diff against — the gate is about change, not existence.
|
|
3332
3878
|
if (outgoing) {
|
|
3333
3879
|
if (outgoing.permission_digest !== incoming.permission_digest && !ack.permissionChange) {
|
|
3334
|
-
throw
|
|
3880
|
+
throw substratError('conflict', `promotion changes the permission surface (${outgoing.permission_digest} → ` +
|
|
3335
3881
|
`${incoming.permission_digest}) — acknowledge it explicitly to promote`);
|
|
3336
3882
|
}
|
|
3337
3883
|
if (outgoing.migration_digest !== incoming.migration_digest && !ack.migrationChange) {
|
|
3338
|
-
throw
|
|
3884
|
+
throw substratError('conflict', `promotion changes migrations (${outgoing.migration_digest} → ` +
|
|
3339
3885
|
`${incoming.migration_digest}) — acknowledge it explicitly to promote`);
|
|
3340
3886
|
}
|
|
3341
3887
|
// #1705 PR 3: an export an installed consumer imports, dropped or re-versioned.
|
|
@@ -3595,7 +4141,7 @@ export class CloudflareScopeHost {
|
|
|
3595
4141
|
},
|
|
3596
4142
|
createOrg: async (actor, input) => {
|
|
3597
4143
|
const parsed = createOrgInput.parse(input);
|
|
3598
|
-
const created = await this.
|
|
4144
|
+
const created = await this.directory('createOrg', parsed.id, parsed.tenantId, parsed.slug, parsed.name, new Date().toISOString());
|
|
3599
4145
|
if (!created)
|
|
3600
4146
|
return; // idempotent, and a no-op is not audited
|
|
3601
4147
|
await this.recordAdmin(actor, 'createOrg', { tenantId: parsed.tenantId }, null, parsed);
|
|
@@ -3616,15 +4162,27 @@ export class CloudflareScopeHost {
|
|
|
3616
4162
|
await this.recordAdmin(actor, 'addMember', { tenantId }, null, { principal, orgId });
|
|
3617
4163
|
await this.fanOut(tenantId); // membership is a tenant-level tuple
|
|
3618
4164
|
},
|
|
4165
|
+
applyMembership: async (actor, change) => {
|
|
4166
|
+
// The row is minted here, where attribution and `causedBy` live, and written by the
|
|
4167
|
+
// ControlPlaneDO in the same synchronous method as the fence, the bound and the tuple
|
|
4168
|
+
// (#1184): one DO unit, so nothing lands between the check and the write.
|
|
4169
|
+
const { tenantId, principal, roleKey, op } = change;
|
|
4170
|
+
const assignment = { principalId: principal, roleKey, node: { tenantId, scopeId: null } };
|
|
4171
|
+
const result = await this.cp.applyMembership(change, op === 'add'
|
|
4172
|
+
? this.adminEntry(actor, 'assignRole', { tenantId, scopeId: null }, null, assignment)
|
|
4173
|
+
: this.adminEntry(actor, 'unassignRole', { tenantId, scopeId: null }, assignment, null));
|
|
4174
|
+
// The tenant-level tuple, or its tombstone, reaches the projections.
|
|
4175
|
+
if (result.applied && (op === 'add' || result.changed))
|
|
4176
|
+
await this.fanOut(tenantId);
|
|
4177
|
+
return result;
|
|
4178
|
+
},
|
|
3619
4179
|
removeMember: async (actor, tenantId, principal, orgId) => {
|
|
3620
4180
|
await requireOrg(tenantId, orgId);
|
|
3621
|
-
// Tombstone (K-21), never DELETE
|
|
3622
|
-
//
|
|
3623
|
-
const changed = await this.cp.
|
|
3624
|
-
if (
|
|
3625
|
-
|
|
3626
|
-
await this.recordAdmin(actor, 'removeMember', { tenantId }, { principal, orgId }, null);
|
|
3627
|
-
await this.fanOut(tenantId); // the tombstone must reach the projections
|
|
4181
|
+
// Tombstone (K-21), never DELETE, with the removal fence in the same DO method
|
|
4182
|
+
// (#1184). A repeat revoke stays a silent no-op rather than a second audit row.
|
|
4183
|
+
const changed = await this.cp.revokeAndFence(tenantId, principal, 'member', `org:${orgId}`, this.adminEntry(actor, 'removeMember', { tenantId }, { principal, orgId }, null));
|
|
4184
|
+
if (changed)
|
|
4185
|
+
await this.fanOut(tenantId); // the tombstone must reach the projections
|
|
3628
4186
|
},
|
|
3629
4187
|
listMembers: async (actor, tenantId, orgId, options) => {
|
|
3630
4188
|
await requireOrg(tenantId, orgId);
|
|
@@ -3638,20 +4196,20 @@ export class CloudflareScopeHost {
|
|
|
3638
4196
|
},
|
|
3639
4197
|
createTenant: async (actor, input) => {
|
|
3640
4198
|
const parsed = createTenantInput.parse(input);
|
|
3641
|
-
const created = await this.
|
|
4199
|
+
const created = await this.directory('createTenant', parsed.id, parsed.slug, parsed.name, new Date().toISOString(), parsed.provisionedByTenant ?? null);
|
|
3642
4200
|
// Idempotent: re-creating an existing tenant is a no-op, not audited.
|
|
3643
4201
|
if (!created)
|
|
3644
4202
|
return;
|
|
3645
4203
|
await this.recordAdmin(actor, 'createTenant', { tenantId: parsed.id }, null, created);
|
|
3646
4204
|
},
|
|
3647
4205
|
setTenantStatus: async (actor, tenantId, status) => {
|
|
3648
|
-
const before = await this.
|
|
4206
|
+
const before = await this.directory('setTenantStatus', tenantId, status);
|
|
3649
4207
|
await this.recordAdmin(actor, 'setTenantStatus', { tenantId }, { status: before }, { status });
|
|
3650
4208
|
// #1713: every hosted scope under the tenant holds or resumes its work by this.
|
|
3651
4209
|
await this.deliverLifecycles(actor, { tenantId });
|
|
3652
4210
|
},
|
|
3653
4211
|
setTenantName: async (actor, tenantId, name) => {
|
|
3654
|
-
const before = await this.
|
|
4212
|
+
const before = await this.directory('setTenantName', tenantId, name);
|
|
3655
4213
|
if (before === name)
|
|
3656
4214
|
return; // no-op is not audited — nothing changed
|
|
3657
4215
|
await this.recordAdmin(actor, 'setTenantName', { tenantId }, { name: before }, { name });
|
|
@@ -3661,7 +4219,7 @@ export class CloudflareScopeHost {
|
|
|
3661
4219
|
// first (archive-if-needed → reapScope in the vertical deployment); the DO clears
|
|
3662
4220
|
// the tenant's PII/config rows and flips the row to a `reaped` tombstone, keeping
|
|
3663
4221
|
// the row + admin log. Only a `deleting` tenant may be reaped (checked in the DO).
|
|
3664
|
-
const before = await this.
|
|
4222
|
+
const before = await this.directory('reapTenant', tenantId);
|
|
3665
4223
|
await this.recordAdmin(actor, 'reapTenant', { tenantId }, { status: before }, { status: 'reaped' });
|
|
3666
4224
|
},
|
|
3667
4225
|
listTenants: async (actor, page) => {
|
|
@@ -3745,7 +4303,7 @@ export class CloudflareScopeHost {
|
|
|
3745
4303
|
// Not `peerScopeGate`: a read must not migrate, and the phase makes this call once
|
|
3746
4304
|
// per edge per pass, where an extra round trip is a fleet-wide cost.
|
|
3747
4305
|
this.assertServedHere(await this.scopeRecordForRead(tenantId, scopeId), scopeId, 'readExportedEvents');
|
|
3748
|
-
const batch =
|
|
4306
|
+
const batch = await this.readExportsThroughDoor(input, tenantId, scopeId);
|
|
3749
4307
|
await this.recordAccess(actor, 'readExportedEvents', { tenantId, scopeId }, { consumer: input.consumer, after: input.after, limit: input.limit }, batch.events.length);
|
|
3750
4308
|
return batch;
|
|
3751
4309
|
},
|
|
@@ -3925,12 +4483,18 @@ export class CloudflareScopeHost {
|
|
|
3925
4483
|
await this.recordAccess(actor, 'lifecycleFlow', { tenantId, scopeId }, delegatedReadParams.lifecycleFlow(input), flow.observation.events);
|
|
3926
4484
|
return flow;
|
|
3927
4485
|
},
|
|
4486
|
+
operationSeries: async (actor, tenantId, scopeId, input) => {
|
|
4487
|
+
await this.scopeRecordForRead(tenantId, scopeId);
|
|
4488
|
+
const series = await this.scopeStub(scopeId).operationSeries(input);
|
|
4489
|
+
await this.recordAccess(actor, 'operationSeries', { tenantId, scopeId }, delegatedReadParams.operationSeries(input), operationSeriesCount(series));
|
|
4490
|
+
return series;
|
|
4491
|
+
},
|
|
3928
4492
|
readScopeTable: async (actor, tenantId, scopeId, input) => {
|
|
3929
4493
|
await this.scopeRecordForRead(tenantId, scopeId);
|
|
3930
4494
|
// Checked here as well as in the ScopeDO, so the refusal keeps its code across the hop.
|
|
3931
4495
|
assertRowLimit('limit', input.limit);
|
|
3932
4496
|
assertRowOffset('offset', input.offset);
|
|
3933
|
-
const page = await this.scopeStub(scopeId).
|
|
4497
|
+
const page = unwrapReply(await this.scopeStub(scopeId).introspectTableReply(input.table, input.limit, input.offset));
|
|
3934
4498
|
await this.recordAccess(actor, 'readScopeTable', { tenantId, scopeId }, { table: input.table, limit: page.limit, offset: page.offset }, page.rows.length);
|
|
3935
4499
|
return page;
|
|
3936
4500
|
},
|
|
@@ -3971,7 +4535,7 @@ export class CloudflareScopeHost {
|
|
|
3971
4535
|
},
|
|
3972
4536
|
queryScope: async (actor, tenantId, scopeId, input) => {
|
|
3973
4537
|
await this.scopeRecordForRead(tenantId, scopeId);
|
|
3974
|
-
const result = await this.scopeStub(scopeId).
|
|
4538
|
+
const result = unwrapReply(await this.scopeStub(scopeId).introspectQueryReply(input.sql));
|
|
3975
4539
|
// The statement is the logged argument: the access log is the evidence trail,
|
|
3976
4540
|
// and for a console read the SQL is the whole story.
|
|
3977
4541
|
await this.recordAccess(actor, 'queryScope', { tenantId, scopeId }, { sql: input.sql }, result.rows.length);
|
|
@@ -4064,7 +4628,7 @@ export class CloudflareScopeHost {
|
|
|
4064
4628
|
if (!rec)
|
|
4065
4629
|
throw substratError('not_found', `unknown scope ${scopeId} in tenant ${tenantId}`);
|
|
4066
4630
|
if (rec.status !== 'archived') {
|
|
4067
|
-
throw
|
|
4631
|
+
throw substratError('conflict', `scope ${scopeId} is ${rec.status}, not archived — only an archived scope may be reaped`);
|
|
4068
4632
|
}
|
|
4069
4633
|
// A serving scope always holds ≥1 bound hostname; a truly-dead one has been
|
|
4070
4634
|
// unbound. The dashboard delete path unbinds AT archive, but a bare console
|
|
@@ -4076,7 +4640,7 @@ export class CloudflareScopeHost {
|
|
|
4076
4640
|
// being released anyway; the interactive per-scope reap never sets it.
|
|
4077
4641
|
const bound = opts?.force ? [] : await this.cp.listHostnames({ scopeId, limit: 1 });
|
|
4078
4642
|
if (bound.length > 0) {
|
|
4079
|
-
throw
|
|
4643
|
+
throw substratError('conflict', `scope ${scopeId} still resolves hostname '${bound[0].hostname}' — ` +
|
|
4080
4644
|
`unbind it before reaping (reap wipes storage and cannot be undone)`);
|
|
4081
4645
|
}
|
|
4082
4646
|
await this.scopeStub(scopeId).destroyStorage();
|
|
@@ -4274,7 +4838,7 @@ export class CloudflareScopeHost {
|
|
|
4274
4838
|
},
|
|
4275
4839
|
registerIdentityPool: async (actor, input) => {
|
|
4276
4840
|
const parsed = identityPool.parse(input);
|
|
4277
|
-
const created = await this.
|
|
4841
|
+
const created = await this.directory('registerIdentityPool', parsed.provider, parsed.topology, parsed.tenantId, new Date().toISOString());
|
|
4278
4842
|
if (!created)
|
|
4279
4843
|
return; // identical registration is idempotent, unaudited
|
|
4280
4844
|
// Null tenant for a central pool: it belongs to no single tenant, which is
|
|
@@ -4291,9 +4855,9 @@ export class CloudflareScopeHost {
|
|
|
4291
4855
|
listIdentityTenants: async (actor, provider, externalId) => {
|
|
4292
4856
|
const r = await this.cp.readPool(provider);
|
|
4293
4857
|
if (!r)
|
|
4294
|
-
throw
|
|
4858
|
+
throw substratError('not_found', `identity pool '${provider}' is not registered`);
|
|
4295
4859
|
if (r.topology !== 'central') {
|
|
4296
|
-
throw
|
|
4860
|
+
throw substratError('forbidden', `identity pool '${provider}' is tenant-bound — enumerating tenants is only ` +
|
|
4297
4861
|
`meaningful on a central pool, where the same externalId is the same person`);
|
|
4298
4862
|
}
|
|
4299
4863
|
const tenants = (await this.cp.identityTenants(provider, externalId));
|
|
@@ -4310,9 +4874,9 @@ export class CloudflareScopeHost {
|
|
|
4310
4874
|
at: new Date().toISOString(),
|
|
4311
4875
|
});
|
|
4312
4876
|
if (topology === null)
|
|
4313
|
-
throw
|
|
4877
|
+
throw substratError('not_found', `identity pool '${provider}' is not registered`);
|
|
4314
4878
|
if (topology !== 'central') {
|
|
4315
|
-
throw
|
|
4879
|
+
throw substratError('forbidden', `identity pool '${provider}' is tenant-bound — enumerating tenants is only ` +
|
|
4316
4880
|
`meaningful on a central pool, where the same externalId is the same person`);
|
|
4317
4881
|
}
|
|
4318
4882
|
return memberships.map((m) => identityMembership.parse(m));
|
|
@@ -4465,14 +5029,14 @@ export class CloudflareScopeHost {
|
|
|
4465
5029
|
const parsed = identityLink.parse(input);
|
|
4466
5030
|
const pool = await this.cp.readPool(parsed.provider);
|
|
4467
5031
|
if (!pool) {
|
|
4468
|
-
throw
|
|
5032
|
+
throw substratError('conflict', `identity pool '${parsed.provider}' is not registered — a pool must declare ` +
|
|
4469
5033
|
`its topology before it may link (central vs tenant-bound decides whether ` +
|
|
4470
5034
|
`the same externalId in two tenants is one person or two)`);
|
|
4471
5035
|
}
|
|
4472
5036
|
if (pool.topology === 'tenant-bound' && pool.tenant_id !== parsed.tenantId) {
|
|
4473
|
-
throw
|
|
5037
|
+
throw substratError('conflict', `identity pool '${parsed.provider}' is bound to tenant ${pool.tenant_id} and cannot link into ${parsed.tenantId}`);
|
|
4474
5038
|
}
|
|
4475
|
-
const changed = await this.
|
|
5039
|
+
const changed = await this.directory('linkIdentity', parsed.provider, parsed.externalId, parsed.principal, parsed.tenantId, parsed.scopeId ?? null, new Date().toISOString());
|
|
4476
5040
|
// Idempotent: an identity already bound is a no-op, not audited.
|
|
4477
5041
|
if (!changed)
|
|
4478
5042
|
return;
|
|
@@ -4734,18 +5298,46 @@ export class CloudflareScopeHost {
|
|
|
4734
5298
|
};
|
|
4735
5299
|
}
|
|
4736
5300
|
// -- helpers --------------------------------------------------------------
|
|
5301
|
+
/** A directory write whose refusal must keep its code across the DO hop (#113). */
|
|
5302
|
+
async directory(method, ...args) {
|
|
5303
|
+
return unwrapReply(await this.cp.reply(method, args));
|
|
5304
|
+
}
|
|
4737
5305
|
/**
|
|
4738
5306
|
* The getScope gate (control-plane.md §4.1/§4.2) — the pair check, then the tenant's and
|
|
4739
5307
|
* the scope's lifecycle — thrown on THIS side of the RPC (#1718). The ControlPlaneDO
|
|
4740
5308
|
* answers its refusal as data, because an error thrown there arrives here flattened, with
|
|
4741
|
-
* no code; a record crosses intact.
|
|
4742
|
-
*
|
|
5309
|
+
* no code; a record crosses intact.
|
|
5310
|
+
*
|
|
5311
|
+
* A CP-less host's null control plane answers nothing: the router made the pair check from
|
|
5312
|
+
* the shared directory. The scope's own storage then makes it again (#2016), against the
|
|
5313
|
+
* tenant it was provisioned for (`ScopeDO.admission`), so a vertical that passes a scope id
|
|
5314
|
+
* under the wrong tenant meets K-3's refusal at the door instead of relying on the permission
|
|
5315
|
+
* gate's grant data to deny it.
|
|
5316
|
+
*
|
|
5317
|
+
* Answers with the lifecycle the CP-less scope holds (#1713), read in the same DO call as the
|
|
5318
|
+
* pair check, so an `assertLive` door still costs one round-trip; a door that reads the scope
|
|
5319
|
+
* without the lifecycle half pays the one DO call the pair check needs. Null with a directory,
|
|
5320
|
+
* whose refusal already covered the lifecycle.
|
|
4743
5321
|
*/
|
|
4744
5322
|
async validateScopeAccess(tenantId, scopeId) {
|
|
4745
5323
|
const refusal = await this.cp.scopeAccessRefusal(tenantId, scopeId);
|
|
4746
|
-
if (
|
|
4747
|
-
|
|
4748
|
-
|
|
5324
|
+
if (refusal) {
|
|
5325
|
+
throw refusal.code
|
|
5326
|
+
? substratError(refusal.code, refusal.message, refusal.reason ? { reason: refusal.reason } : undefined)
|
|
5327
|
+
: new Error(refusal.message);
|
|
5328
|
+
}
|
|
5329
|
+
if (!this.cpLess)
|
|
5330
|
+
return null;
|
|
5331
|
+
const { verdict, lifecycle } = await this.scopeStub(scopeId).admission(tenantId);
|
|
5332
|
+
if (verdict === 'foreign')
|
|
5333
|
+
throw unknownScopeForTenant(tenantId, scopeId);
|
|
5334
|
+
// A scope with no tenant record and no role rows has nothing to hold the pair against, so it is
|
|
5335
|
+
// let through — a compatibility state, not a steady one: the next reconcile (every push runs
|
|
5336
|
+
// one against the scopes behind it, #1172) or lifecycle delivery records the directory's tenant.
|
|
5337
|
+
// Logged on every admission, so an operator sees a scope that has not converged.
|
|
5338
|
+
if (verdict === 'unknown')
|
|
5339
|
+
console.warn(JSON.stringify(tenantUnrecordedLine(tenantId, scopeId)));
|
|
5340
|
+
return lifecycle;
|
|
4749
5341
|
}
|
|
4750
5342
|
/**
|
|
4751
5343
|
* THE lifecycle gate (#1713) at every door that runs a scope's work: a request's stubs and
|
|
@@ -4763,16 +5355,9 @@ export class CloudflareScopeHost {
|
|
|
4763
5355
|
* grants) keep `validateScopeAccess` alone: an operator still reads a suspended scope.
|
|
4764
5356
|
*/
|
|
4765
5357
|
async assertLive(tenantId, scopeId) {
|
|
4766
|
-
await this.validateScopeAccess(tenantId, scopeId);
|
|
4767
|
-
const refusal = await this.cpLessRefusal(scopeId, { tenantId, scopeId });
|
|
5358
|
+
const refusal = lifecycleRefusal(await this.validateScopeAccess(tenantId, scopeId), { tenantId, scopeId });
|
|
4768
5359
|
if (refusal)
|
|
4769
|
-
throw
|
|
4770
|
-
}
|
|
4771
|
-
/** The delivered lifecycle's refusal on a CP-less host (#1713); always null with a directory. */
|
|
4772
|
-
async cpLessRefusal(scopeId, ids) {
|
|
4773
|
-
if (!this.cpLess)
|
|
4774
|
-
return null;
|
|
4775
|
-
return lifecycleRefusal(await this.scopeStub(scopeId).lifecycle(), ids);
|
|
5360
|
+
throw substratError('conflict', refusal, { reason: SCOPE_GATE_REASONS.notActive });
|
|
4776
5361
|
}
|
|
4777
5362
|
/**
|
|
4778
5363
|
* Deliver the directory's lifecycle to the deployments serving the matching scopes (#1713):
|
|
@@ -4792,8 +5377,25 @@ export class CloudflareScopeHost {
|
|
|
4792
5377
|
if (!delegation || this.cpLess)
|
|
4793
5378
|
return report;
|
|
4794
5379
|
let targets;
|
|
5380
|
+
// #2016: the heal also asks the served scopes whose deployment has not answered that it holds a
|
|
5381
|
+
// record of the scope's tenant, a copy as much as a primary (no push reconciles a copy), until
|
|
5382
|
+
// each answers that it does. A bounded slice per pass, after the drift and the holds.
|
|
5383
|
+
const askTenant = new Set();
|
|
4795
5384
|
try {
|
|
4796
5385
|
targets = await this.cp.lifecycleTargets(filter);
|
|
5386
|
+
if (filter.drift) {
|
|
5387
|
+
const unrecorded = await this.cp.lifecycleTargets({ unrecorded: true, limit: TENANT_UNRECORDED_PER_PASS });
|
|
5388
|
+
// Noted before asking, so an ask that fails (or a deployment that never answers) rotates
|
|
5389
|
+
// behind the scopes not asked yet, and the next pass reaches them.
|
|
5390
|
+
if (unrecorded.length > 0)
|
|
5391
|
+
await this.cp.recordTenantAsks(unrecorded.map((u) => u.scope_id), new Date().toISOString());
|
|
5392
|
+
const seen = new Set(targets.map((t) => t.scope_id));
|
|
5393
|
+
for (const u of unrecorded) {
|
|
5394
|
+
askTenant.add(u.scope_id);
|
|
5395
|
+
if (!seen.has(u.scope_id))
|
|
5396
|
+
targets.push(u);
|
|
5397
|
+
}
|
|
5398
|
+
}
|
|
4797
5399
|
}
|
|
4798
5400
|
catch (err) {
|
|
4799
5401
|
console.error('substrat: could not read the scopes to deliver a lifecycle to (#1713)', err);
|
|
@@ -4815,8 +5417,10 @@ export class CloudflareScopeHost {
|
|
|
4815
5417
|
// no lifecycle runs live. That keeps an activation, and every transition before a
|
|
4816
5418
|
// deployment carries the route, from posting a delivery that changes nothing. After a
|
|
4817
5419
|
// restore (epoch > 0) no receipt says nothing about what the deployment holds, so it is
|
|
4818
|
-
// delivered. A HOLD is always delivered.
|
|
4819
|
-
if (
|
|
5420
|
+
// delivered. A HOLD is always delivered, and so is a scope the heal asks for its tenant record.
|
|
5421
|
+
if (!askTenant.has(scopeId) &&
|
|
5422
|
+
lifecycleRefusal(lifecycle) === null &&
|
|
5423
|
+
(t.delivered === null ? t.epoch === 0 : t.delivered === lifecycleReceipt(lifecycle))) {
|
|
4820
5424
|
continue;
|
|
4821
5425
|
}
|
|
4822
5426
|
report.attempted += 1;
|
|
@@ -4874,7 +5478,7 @@ export class CloudflareScopeHost {
|
|
|
4874
5478
|
}
|
|
4875
5479
|
}
|
|
4876
5480
|
// The receipt is what the scope HOLDS: a newer delivery it kept instead is the truth.
|
|
4877
|
-
await this.cp.recordLifecycleReceipt(scopeId, lifecycleReceipt(answer.lifecycle), new Date().toISOString());
|
|
5481
|
+
await this.cp.recordLifecycleReceipt(scopeId, lifecycleReceipt(answer.lifecycle), new Date().toISOString(), answer.tenantRecorded === true);
|
|
4878
5482
|
report.delivered += 1;
|
|
4879
5483
|
}
|
|
4880
5484
|
catch (err) {
|
|
@@ -4923,7 +5527,7 @@ export class CloudflareScopeHost {
|
|
|
4923
5527
|
* points ask the directory themselves.
|
|
4924
5528
|
*/
|
|
4925
5529
|
async lifecycleHeld(scopeId) {
|
|
4926
|
-
return (await this.
|
|
5530
|
+
return this.cpLess && lifecycleRefusal(await this.scopeStub(scopeId).lifecycle()) !== null;
|
|
4927
5531
|
}
|
|
4928
5532
|
/**
|
|
4929
5533
|
* K-3's cross-check on its own: the (tenant, scope) pair must exist and agree before a
|
|
@@ -4933,7 +5537,7 @@ export class CloudflareScopeHost {
|
|
|
4933
5537
|
async assertScope(tenantId, scopeId) {
|
|
4934
5538
|
const rec = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
4935
5539
|
if (!rec)
|
|
4936
|
-
throw
|
|
5540
|
+
throw unknownScopeForTenant(tenantId, scopeId);
|
|
4937
5541
|
}
|
|
4938
5542
|
/**
|
|
4939
5543
|
* The same K-3 cross-check, for a read that then opens the scope's STORAGE — every
|
|
@@ -4948,9 +5552,9 @@ export class CloudflareScopeHost {
|
|
|
4948
5552
|
async scopeRecordForRead(tenantId, scopeId) {
|
|
4949
5553
|
const row = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
4950
5554
|
if (!row)
|
|
4951
|
-
throw
|
|
5555
|
+
throw unknownScopeForTenant(tenantId, scopeId);
|
|
4952
5556
|
if (row.status === 'reaped') {
|
|
4953
|
-
throw
|
|
5557
|
+
throw substratError('conflict', `scope ${scopeId} is reaped — its storage is gone and cannot be read`);
|
|
4954
5558
|
}
|
|
4955
5559
|
return row;
|
|
4956
5560
|
}
|
|
@@ -4983,7 +5587,14 @@ export class CloudflareScopeHost {
|
|
|
4983
5587
|
});
|
|
4984
5588
|
}
|
|
4985
5589
|
async recordAdmin(actor, action, target, before, after) {
|
|
4986
|
-
await this.cp.recordAdmin(
|
|
5590
|
+
await this.cp.recordAdmin(this.adminEntry(actor, action, target, before, after));
|
|
5591
|
+
}
|
|
5592
|
+
/**
|
|
5593
|
+
* One admin-log row, minted here where attribution and `causedBy` live — for `recordAdmin`,
|
|
5594
|
+
* and for a ControlPlaneDO method that writes its row in the same unit as its effect (#1184).
|
|
5595
|
+
*/
|
|
5596
|
+
adminEntry(actor, action, target, before, after) {
|
|
5597
|
+
return {
|
|
4987
5598
|
id: ulid(),
|
|
4988
5599
|
actor,
|
|
4989
5600
|
action,
|
|
@@ -4995,7 +5606,7 @@ export class CloudflareScopeHost {
|
|
|
4995
5606
|
before: before ?? null,
|
|
4996
5607
|
after: after ?? null,
|
|
4997
5608
|
at: new Date().toISOString(),
|
|
4998
|
-
}
|
|
5609
|
+
};
|
|
4999
5610
|
}
|
|
5000
5611
|
/**
|
|
5001
5612
|
* The scope's own answer to "what may this connection do here" (#726 gap 1).
|
|
@@ -5120,6 +5731,11 @@ export class CloudflareScopeHost {
|
|
|
5120
5731
|
// rewind CLAIMS them on `SWITCH_HOLDS_NAME`, which the rewind cannot reach. A module is held
|
|
5121
5732
|
// while any claim on it exists, and `runDueSchedules` skips a held module.
|
|
5122
5733
|
//
|
|
5734
|
+
// #2029: the same for a PEER switched off (`vertical:<slug>`): the rewind claims the scope's
|
|
5735
|
+
// off peers beside its off modules, each under its hold key (`holdKeyOf`), and the peer door
|
|
5736
|
+
// refuses a held peer as the system door refuses a held module. Everything below holds for
|
|
5737
|
+
// both kinds; "module" reads as "subject".
|
|
5738
|
+
//
|
|
5123
5739
|
// A claim is released only by a switch move KNOWN to survive its rewind (`switchInScope`): a
|
|
5124
5740
|
// move on the restored storage, after the rewind armed. A move during the settle, or one the
|
|
5125
5741
|
// doomed instance applied, lands in storage the restart discards, so it releases nothing. A
|
|
@@ -5138,7 +5754,7 @@ export class CloudflareScopeHost {
|
|
|
5138
5754
|
* reopens only the gap this closes, for a rewind that lands during the outage, and the next
|
|
5139
5755
|
* reconcile still switches that module off. The error is answered, for the pass to report.
|
|
5140
5756
|
*/
|
|
5141
|
-
async switchHeld(scopeId,
|
|
5757
|
+
async switchHeld(scopeId, holdKey) {
|
|
5142
5758
|
const at = Date.now();
|
|
5143
5759
|
let error;
|
|
5144
5760
|
if (!this.holdSnapshot || at - this.holdSnapshot.at > SWITCH_HOLD_SNAPSHOT_MS) {
|
|
@@ -5161,8 +5777,11 @@ export class CloudflareScopeHost {
|
|
|
5161
5777
|
}
|
|
5162
5778
|
}
|
|
5163
5779
|
}
|
|
5164
|
-
const
|
|
5165
|
-
|
|
5780
|
+
const snapshot = this.holdSnapshot;
|
|
5781
|
+
const held = snapshot?.held.has(`${scopeId} ${holdKey}`) ?? false;
|
|
5782
|
+
// #2029: a good read naming no row for this scope at all, for `doorGate`'s instance cache.
|
|
5783
|
+
const scopeClear = !error && snapshot?.good === true && !snapshot.scopes.has(scopeId);
|
|
5784
|
+
return { held, ...(error ? { error } : {}), ...(scopeClear ? { scopeClear: true } : {}) };
|
|
5166
5785
|
}
|
|
5167
5786
|
/** One read of the whole hold into the snapshot, stamped with when it went out. Never throws. */
|
|
5168
5787
|
async refreshHolds(at) {
|
|
@@ -5170,7 +5789,12 @@ export class CloudflareScopeHost {
|
|
|
5170
5789
|
const rows = await this.switchHoldsStub().switchHoldsAll();
|
|
5171
5790
|
// A slow read that lands after a newer one never replaces it.
|
|
5172
5791
|
if (!this.holdSnapshot || at >= this.holdSnapshot.at) {
|
|
5173
|
-
this.holdSnapshot = {
|
|
5792
|
+
this.holdSnapshot = {
|
|
5793
|
+
at,
|
|
5794
|
+
held: new Set(rows.map((r) => `${r.scopeId} ${r.moduleId}`)),
|
|
5795
|
+
scopes: new Set(rows.map((r) => r.scopeId)),
|
|
5796
|
+
good: true,
|
|
5797
|
+
};
|
|
5174
5798
|
}
|
|
5175
5799
|
return {};
|
|
5176
5800
|
}
|
|
@@ -5181,8 +5805,10 @@ export class CloudflareScopeHost {
|
|
|
5181
5805
|
// than hammered per scope. For that long, other scopes reuse this possibly pre-hold set
|
|
5182
5806
|
// silently, and only the consult that sent this read reports the error: the declared
|
|
5183
5807
|
// fail-open, attributed once.
|
|
5184
|
-
if (!this.holdSnapshot || at >= this.holdSnapshot.at)
|
|
5185
|
-
|
|
5808
|
+
if (!this.holdSnapshot || at >= this.holdSnapshot.at) {
|
|
5809
|
+
const scopes = new Set([...(known ?? [])].map((k) => k.slice(0, k.indexOf(' '))));
|
|
5810
|
+
this.holdSnapshot = { at, held: known ?? new Set(), scopes, good: false };
|
|
5811
|
+
}
|
|
5186
5812
|
return {
|
|
5187
5813
|
error: known
|
|
5188
5814
|
? `switch hold unreadable (${why}); the ${known.size} hold(s) from this pass's last good read ` +
|
|
@@ -5229,7 +5855,11 @@ export class CloudflareScopeHost {
|
|
|
5229
5855
|
* its claims before the OFF's join lands. The joined row then holds the module the operator just
|
|
5230
5856
|
* turned ON, until the next surviving move of that switch releases it (another ON does).
|
|
5231
5857
|
*/
|
|
5232
|
-
async switchInScope(scopeId,
|
|
5858
|
+
async switchInScope(kind, scopeId, key, to, at, tenantHeld = false,
|
|
5859
|
+
/** #2045: the switch call's fence; see `SWITCH_FENCES_DDL`. */
|
|
5860
|
+
fence) {
|
|
5861
|
+
// #2029: a peer's claims live beside the modules', under its own hold key.
|
|
5862
|
+
const holdKey = holdKeyOf(kind, key);
|
|
5233
5863
|
let scopeClaims;
|
|
5234
5864
|
let readError;
|
|
5235
5865
|
try {
|
|
@@ -5238,23 +5868,30 @@ export class CloudflareScopeHost {
|
|
|
5238
5868
|
catch (err) {
|
|
5239
5869
|
readError = err;
|
|
5240
5870
|
}
|
|
5241
|
-
const
|
|
5871
|
+
const stub = this.scopeStub(scopeId);
|
|
5872
|
+
const { instance, ...outcome } = await (kind === 'system'
|
|
5873
|
+
? stub.switchSystemSchedules(key, scopeId, to, at, tenantHeld, fence)
|
|
5874
|
+
: stub.switchPeer(key, scopeId, to, at, tenantHeld, fence));
|
|
5242
5875
|
if (!scopeClaims)
|
|
5243
5876
|
throw readError;
|
|
5877
|
+
// #2045: a move the fence refused wrote nothing, so it is known to survive nothing and to
|
|
5878
|
+
// release nothing; the newer call that superseded it did whatever the hold needed.
|
|
5879
|
+
if (outcome.superseded)
|
|
5880
|
+
return outcome;
|
|
5244
5881
|
const now = Date.now();
|
|
5245
5882
|
const pendingLive = (c) => c.state === 'pending' && now - Date.parse(c.heldAt) <= SWITCH_HOLD_PENDING_MAX_MS;
|
|
5246
|
-
const before = scopeClaims.filter((c) => c.moduleId ===
|
|
5883
|
+
const before = scopeClaims.filter((c) => c.moduleId === holdKey);
|
|
5247
5884
|
if (to === 'on') {
|
|
5248
5885
|
// Every ON, claims or not: its tombstone is what keeps a rewind's stale read from claiming
|
|
5249
5886
|
// the module after this (#1839).
|
|
5250
5887
|
this.holdSnapshot = null;
|
|
5251
|
-
await this.switchHoldsStub().switchHoldOn(scopeId,
|
|
5888
|
+
await this.switchHoldsStub().switchHoldOn(scopeId, holdKey, before.map((c) => c.claimId));
|
|
5252
5889
|
}
|
|
5253
5890
|
else if (outcome.held && before.length > 0) {
|
|
5254
5891
|
const survived = before.filter((c) => !pendingLive(c) && c.doomed !== instance);
|
|
5255
5892
|
if (survived.length > 0) {
|
|
5256
5893
|
this.holdSnapshot = null;
|
|
5257
|
-
await this.switchHoldsStub().switchHoldRelease(scopeId,
|
|
5894
|
+
await this.switchHoldsStub().switchHoldRelease(scopeId, holdKey, survived.map((c) => c.claimId));
|
|
5258
5895
|
}
|
|
5259
5896
|
}
|
|
5260
5897
|
if (to === 'off' && outcome.changed) {
|
|
@@ -5265,7 +5902,7 @@ export class CloudflareScopeHost {
|
|
|
5265
5902
|
const joins = new Set([...scopeClaims, ...after].filter((c) => pendingLive(c) || c.doomed === instance).map((c) => c.claimId));
|
|
5266
5903
|
if (joins.size > 0) {
|
|
5267
5904
|
this.holdSnapshot = null;
|
|
5268
|
-
await this.switchHoldsStub().switchHoldJoin(scopeId,
|
|
5905
|
+
await this.switchHoldsStub().switchHoldJoin(scopeId, holdKey, [...joins]);
|
|
5269
5906
|
}
|
|
5270
5907
|
}
|
|
5271
5908
|
return outcome;
|
|
@@ -5329,11 +5966,16 @@ export class CloudflareScopeHost {
|
|
|
5329
5966
|
await holds.switchHoldArm(scopeId, claimId, result.instance ?? null).catch(() => undefined);
|
|
5330
5967
|
return { rewindingTo: result.rewindingTo };
|
|
5331
5968
|
}
|
|
5332
|
-
/**
|
|
5969
|
+
/**
|
|
5970
|
+
* The subjects this scope's own storage has switched off, as a rewind reads them before it arms:
|
|
5971
|
+
* its modules, and (#2029) its peers, each by its hold key (`holdKeyOf`).
|
|
5972
|
+
*/
|
|
5333
5973
|
async offInStorage(scopeId) {
|
|
5334
|
-
|
|
5335
|
-
|
|
5336
|
-
.map((e) => e.moduleId)
|
|
5974
|
+
const [modules, peers] = await Promise.all([this.systemGrantsStatusLocal(scopeId), this.peerGrantsStatusLocal(scopeId)]);
|
|
5975
|
+
return [
|
|
5976
|
+
...modules.filter((e) => e.schedules === 'off').map((e) => holdKeyOf('system', e.moduleId)),
|
|
5977
|
+
...peers.filter((e) => e.calls === 'off').map((e) => holdKeyOf('peer', e.vertical)),
|
|
5978
|
+
];
|
|
5337
5979
|
}
|
|
5338
5980
|
/**
|
|
5339
5981
|
* #1839: wait until every row of this rewind's claim is `SWITCH_HOLD_SETTLE_MS` old. The settle
|
|
@@ -5402,7 +6044,11 @@ export class CloudflareScopeHost {
|
|
|
5402
6044
|
* arrive as a different class of answer just because the caller asked for a socket.
|
|
5403
6045
|
*/
|
|
5404
6046
|
liveReads = {
|
|
5405
|
-
subscribe: async ({ tenantId, scopeId, principal, request }) => {
|
|
6047
|
+
subscribe: async ({ tenantId, scopeId, principal, request, within }) => {
|
|
6048
|
+
// Decided before anything else, and thrown: a `within` that is neither a plain
|
|
6049
|
+
// EntityRef nor a value `vouchedWithin` built is a caller bug, and the two ways of
|
|
6050
|
+
// guessing at it both open a feed wider than was asked for (#1853).
|
|
6051
|
+
const narrowed = liveWithinOf(within);
|
|
5406
6052
|
if (!isUpgradeRequest(request)) {
|
|
5407
6053
|
return new Response('live reads are a WebSocket surface', {
|
|
5408
6054
|
status: 426,
|
|
@@ -5460,6 +6106,10 @@ export class CloudflareScopeHost {
|
|
|
5460
6106
|
headers.set(LIVE_PRINCIPAL_HEADER, principal);
|
|
5461
6107
|
headers.set(LIVE_TENANT_HEADER, tenantId);
|
|
5462
6108
|
headers.set(LIVE_SCOPE_HEADER, scopeId);
|
|
6109
|
+
if (narrowed)
|
|
6110
|
+
headers.set(LIVE_WITHIN_HEADER, encodeLiveWithin(narrowed));
|
|
6111
|
+
else
|
|
6112
|
+
headers.delete(LIVE_WITHIN_HEADER);
|
|
5463
6113
|
const forwarded = new Request(new URL(LIVE_SUBSCRIBE_PATH, 'https://scope.substrat.internal'),
|
|
5464
6114
|
// `new Request(url, { …, headers })` rather than `new Request(request, …)`:
|
|
5465
6115
|
// the DO is addressed by its own path, not the client's, and the upgrade
|
|
@@ -5538,10 +6188,10 @@ export class CloudflareScopeHost {
|
|
|
5538
6188
|
// good. Read first, so a pair the directory does not hold writes nothing.
|
|
5539
6189
|
const scope = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
5540
6190
|
if (!scope)
|
|
5541
|
-
throw
|
|
6191
|
+
throw unknownScopeForTenant(tenantId, scopeId);
|
|
5542
6192
|
const { roles, tuples, entitlements, identities } = await this.tenantProjection(tenantId);
|
|
5543
6193
|
const connectionKeys = await this.connectionKeyRows(tenantId, scope.vertical);
|
|
5544
|
-
await this.scopeStub(scopeId).
|
|
6194
|
+
unwrapReply(await this.scopeStub(scopeId).applyProjectionReply(tenantId, roles, tuples, entitlements, undefined, identities, connectionKeys));
|
|
5545
6195
|
}
|
|
5546
6196
|
/**
|
|
5547
6197
|
* Fan the tenant's current state out into ALL its scopes — called after any
|
|
@@ -5590,7 +6240,7 @@ export class CloudflareScopeHost {
|
|
|
5590
6240
|
if (!keysByVertical.has(vertical)) {
|
|
5591
6241
|
keysByVertical.set(vertical, this.connectionKeyRows(tenantId, s.vertical));
|
|
5592
6242
|
}
|
|
5593
|
-
await this.scopeStub(s.scope_id).
|
|
6243
|
+
unwrapReply(await this.scopeStub(s.scope_id).applyProjectionReply(tenantId, roles, tuples, entitlements, undefined, identities, await keysByVertical.get(vertical)));
|
|
5594
6244
|
}));
|
|
5595
6245
|
const failed = settled.flatMap((r, i) => (r.status === 'rejected' ? [{ scope: scopes[i].scope_id, reason: r.reason }] : []));
|
|
5596
6246
|
if (failed.length === 0)
|
|
@@ -5620,10 +6270,11 @@ export class CloudflareScopeHost {
|
|
|
5620
6270
|
* from its own storage. No tenant-level tuples, no control plane.
|
|
5621
6271
|
*/
|
|
5622
6272
|
async provisionScopeLocal(input) {
|
|
6273
|
+
const carry = recordedOffFromWire(input);
|
|
5623
6274
|
const services = await this.servicePrincipals?.(input.tenantId, input.scopeId);
|
|
5624
6275
|
const stub = this.scopeStub(input.scopeId);
|
|
5625
6276
|
await this.migrateAndRecord(input.scopeId); // create the module tables (setMigrationState no-ops on a null CP)
|
|
5626
|
-
const switchedOff = await stub.
|
|
6277
|
+
const switchedOff = unwrapReply(await stub.applyProjectionReply(input.tenantId, input.roles.map((r) => ({ role_key: r.key, permissions: JSON.stringify(r.permissions), source: r.source })), [], // no tenant-level tuples — a CP-less vertical grants at scope level only
|
|
5627
6278
|
// Only PROJECT (and thereby switch on strict enforcement) when the platform actually
|
|
5628
6279
|
// supplies entitlements. Omitting them leaves the scope un-projected and trusting-
|
|
5629
6280
|
// upstream, so a vertical whose provision path predates #304 is never denied — it
|
|
@@ -5713,12 +6364,10 @@ export class CloudflareScopeHost {
|
|
|
5713
6364
|
public_key: k.publicKey,
|
|
5714
6365
|
}))
|
|
5715
6366
|
: undefined,
|
|
5716
|
-
// #1742: the recorded-off modules, switched off after the seat in the
|
|
5717
|
-
// scope this call provisions — the one the seat's tuples name.
|
|
5718
|
-
input.
|
|
5719
|
-
|
|
5720
|
-
: undefined, services?.map((id) => `principal:${id}`));
|
|
5721
|
-
return input.switchedOff ? { switchedOff } : {};
|
|
6367
|
+
// #1742: the recorded-off modules and (#2029) peers, switched off after the seat in the
|
|
6368
|
+
// same unit, on the scope this call provisions — the one the seat's tuples name.
|
|
6369
|
+
carry ? { scopeId: input.scopeId, at: new Date().toISOString(), ...carry } : undefined, services?.map((id) => `principal:${id}`)));
|
|
6370
|
+
return carry ? { switchedOff } : {};
|
|
5722
6371
|
}
|
|
5723
6372
|
/**
|
|
5724
6373
|
* Resolve an external identity from a scope's PROJECTED links (#406) — the CP-less
|
|
@@ -5857,6 +6506,8 @@ export class CloudflareScopeHost {
|
|
|
5857
6506
|
async connectorGrantLocal(connectionId, scopeId, permission, expiresAt) {
|
|
5858
6507
|
await this.writeScopeTuple(scopeId, subjectRef({ kind: 'connection', id: connectionId }), `granted:${permission}`, `scope:${scopeId}`, expiresAt ?? null);
|
|
5859
6508
|
}
|
|
6509
|
+
/** #2045 (Codex r3): every switch mover here honours the fence — what `/internal/switch-fence` reports. */
|
|
6510
|
+
switchFenced = true;
|
|
5860
6511
|
/**
|
|
5861
6512
|
* The far end of the schedule kill switch for a scope served HERE (#1666):
|
|
5862
6513
|
* `/internal/system-switch` lands on this, from the shared control plane's
|
|
@@ -5865,11 +6516,11 @@ export class CloudflareScopeHost {
|
|
|
5865
6516
|
* admin log and writes the row once this returns. `held: false` is an answer, not a
|
|
5866
6517
|
* throw — see `systemSwitchOutcome`.
|
|
5867
6518
|
*/
|
|
5868
|
-
async systemSwitchLocal(scopeId, moduleId, to) {
|
|
6519
|
+
async systemSwitchLocal(scopeId, moduleId, to, opts) {
|
|
5869
6520
|
// Parsed on the way out: this is the wire answer the platform reads, and the DO's
|
|
5870
6521
|
// plain strings become the published shape here rather than on trust.
|
|
5871
6522
|
// #1819: `switchInScope` releases a rewind's hold on the module once the switch lands.
|
|
5872
|
-
return systemSwitchOutcome.parse(await this.switchInScope(scopeId, moduleId, to, new Date().toISOString()));
|
|
6523
|
+
return systemSwitchOutcome.parse(await this.switchInScope('system', scopeId, moduleId, to, new Date().toISOString(), opts?.tenantHeld ?? false, opts?.fence));
|
|
5873
6524
|
}
|
|
5874
6525
|
/**
|
|
5875
6526
|
* The far end of the status read for a scope served HERE (#1674): `/internal/system-grants`
|
|
@@ -5904,9 +6555,12 @@ export class CloudflareScopeHost {
|
|
|
5904
6555
|
* Move one peer's kill switch on a scope served HERE (#1706) — the far end of
|
|
5905
6556
|
* `revokeFromPeer` / `restoreToPeer` for a hosted scope. Nothing is audited here; the
|
|
5906
6557
|
* platform writes the admin rows around this call, as for the schedule switch.
|
|
6558
|
+
* #2029: through `switchInScope`, so the move releases a rewind's hold on the peer once it
|
|
6559
|
+
* lands, as a module's does. #2030: `tenantHeld`, the platform's word that the peer holds a
|
|
6560
|
+
* live tenant-level grant, which this deployment has no directory to read.
|
|
5907
6561
|
*/
|
|
5908
|
-
async peerSwitchLocal(scopeId, vertical, to) {
|
|
5909
|
-
return peerSwitchOutcome.parse(await this.
|
|
6562
|
+
async peerSwitchLocal(scopeId, vertical, to, opts) {
|
|
6563
|
+
return peerSwitchOutcome.parse(await this.switchInScope('peer', scopeId, verticalSlug.parse(vertical), to, new Date().toISOString(), opts?.tenantHeld ?? false, opts?.fence));
|
|
5910
6564
|
}
|
|
5911
6565
|
// -- the cross-vertical far ends (#1705 PR 2) --------------------------------
|
|
5912
6566
|
// The shared control plane runs the cross-vertical phase for every hosted edge and reaches the
|
|
@@ -5920,7 +6574,25 @@ export class CloudflareScopeHost {
|
|
|
5920
6574
|
async exportedEventsLocal(tenantId, scopeId, raw) {
|
|
5921
6575
|
const input = exportReadInput.parse(raw);
|
|
5922
6576
|
await this.assertServesLocally(tenantId, scopeId, 'readExportedEvents');
|
|
5923
|
-
return
|
|
6577
|
+
return this.readExportsThroughDoor(input, tenantId, scopeId);
|
|
6578
|
+
}
|
|
6579
|
+
/**
|
|
6580
|
+
* #2029: a producer's export read, through the CONSUMER's peer door: the consumer's grants on
|
|
6581
|
+
* this scope decide what leaves it, so a consumer the rewind hold keeps off must hold nothing
|
|
6582
|
+
* here, exactly as a switched-off one does. Gated and pinned like every door call; held, the
|
|
6583
|
+
* producer answers every key missing, so the edge pauses rather than reading past the hold.
|
|
6584
|
+
*/
|
|
6585
|
+
async readExportsThroughDoor(input, tenantId, scopeId) {
|
|
6586
|
+
const stub = this.scopeStub(scopeId);
|
|
6587
|
+
const door = await this.openDoor('peer', input.consumer, scopeId);
|
|
6588
|
+
try {
|
|
6589
|
+
return exportedBatch.parse(await door.through((instance) => stub.exportedEventsRead(input, tenantId, scopeId, { instance })));
|
|
6590
|
+
}
|
|
6591
|
+
catch (err) {
|
|
6592
|
+
if (!this.isHeldRefusal(err))
|
|
6593
|
+
throw err;
|
|
6594
|
+
return exportedBatch.parse(await stub.exportedEventsRead(input, tenantId, scopeId, { held: true }));
|
|
6595
|
+
}
|
|
5924
6596
|
}
|
|
5925
6597
|
/**
|
|
5926
6598
|
* The consumer's imports and watermarks (#1705), for a scope served HERE. The served-here
|
|
@@ -5970,7 +6642,7 @@ export class CloudflareScopeHost {
|
|
|
5970
6642
|
}
|
|
5971
6643
|
const rec = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
5972
6644
|
if (!rec)
|
|
5973
|
-
throw
|
|
6645
|
+
throw unknownScopeForTenant(tenantId, scopeId);
|
|
5974
6646
|
const source = await importCursorSourceOf((t, v) => this.admin.resolveVerticalInstance(t, v), { tenantId, scopeId, vertical: rec.vertical }, move.from);
|
|
5975
6647
|
// Where the write lands is the peer switch's rule: a scope bound to a vertical is served by
|
|
5976
6648
|
// that vertical's deployment, and its watermark lives there. Without a delegation that
|