@substrat-run/adapter-cloudflare 0.137.0 → 0.138.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/checker.d.ts +7 -1
- package/dist/checker.d.ts.map +1 -1
- package/dist/checker.js +2 -2
- package/dist/checker.js.map +1 -1
- package/dist/control-plane-do.d.ts +104 -31
- package/dist/control-plane-do.d.ts.map +1 -1
- package/dist/control-plane-do.js +224 -68
- package/dist/control-plane-do.js.map +1 -1
- package/dist/do-reply.d.ts +25 -0
- package/dist/do-reply.d.ts.map +1 -0
- package/dist/do-reply.js +17 -0
- package/dist/do-reply.js.map +1 -0
- package/dist/host.d.ts +117 -12
- package/dist/host.d.ts.map +1 -1
- package/dist/host.js +795 -435
- package/dist/host.js.map +1 -1
- package/dist/live-reads.d.ts +39 -1
- package/dist/live-reads.d.ts.map +1 -1
- package/dist/live-reads.js +79 -0
- package/dist/live-reads.js.map +1 -1
- package/dist/scope-do.d.ts +2 -7
- package/dist/scope-do.d.ts.map +1 -1
- package/dist/scope-do.js +336 -99
- package/dist/scope-do.js.map +1 -1
- package/package.json +11 -11
package/dist/host.js
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
import { env as ambientEnv } from 'cloudflare:workers';
|
|
2
2
|
import { isRewindRefusal, REWIND_REFUSED } from './rewind-refusal.js';
|
|
3
3
|
import { isSystemDoorMoved, SYSTEM_DOOR_MOVED, SYSTEM_DOOR_REGATES } from './system-door.js';
|
|
4
|
-
import { delegatedReadParams, fromWireFailure, exportReadInput, exportedBatch, importBatch, importResult, importState, importsOfManifestJson, exportsOfManifestJson, importCursorMove, importCursorMoveAt, importCursorMoved, accessLogEntry, adminLogEntry, opsFailureEntry, opsFailureFingerprint, issueEntry, sweepRunEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunsPayload, modelUsageEntry, attachmentRecord, createTenantInput, identityLink, identityPool, createOrgInput, promotionAcknowledgement, bindAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, registerVerticalInput, vertical as verticalSchema, verticalServingState, verticalChannel, verticalVersion, connection, capabilityExchange, mintedCapability, capabilityGrant, principalId, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, systemSwitch, systemSwitchOutcome, systemScheduleEntry, systemSwitchRecord, entitlementGrant, entitlementGrantInput, capabilityFilter, instant, meterReading, subjectRef, createConnectionInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, org as orgSchema, orgMembership, resolvedIdentity, identityMembership, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, delegatedReadRecord, ownerTransferAudit, copyMarkAudit, subjectShredReceipt, connectorDispatchKind, callsOfManifestJson, outboundOfManifestJson, substratError, redrainEventsInput, coverage, peerCoverage, peerGrantsEntry, peerSwitch, peerSwitchOutcome, verticalCaller, verticalResolution, verticalSlug, entityObjectRef, ATTACHMENT_BLOB_BINDING, blobStoreBindingName, lifecycleDelivery, scopeLifecycle, } from '@substrat-run/contracts';
|
|
4
|
+
import { delegatedReadParams, operationSeriesCount, fromWireFailure, exportReadInput, exportedBatch, importBatch, importResult, importState, importsOfManifestJson, exportsOfManifestJson, importCursorMove, importCursorMoveAt, importCursorMoved, accessLogEntry, adminLogEntry, opsFailureEntry, opsFailureFingerprint, issueEntry, sweepRunEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunsPayload, modelUsageEntry, attachmentRecord, createTenantInput, identityLink, identityPool, createOrgInput, promotionAcknowledgement, bindAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, registerVerticalInput, vertical as verticalSchema, verticalServingState, verticalChannel, verticalVersion, connection, capabilityExchange, mintedCapability, capabilityGrant, principalId, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, systemSwitch, systemSwitchOutcome, systemScheduleEntry, systemSwitchRecord, entitlementGrant, entitlementGrantInput, capabilityFilter, instant, meterReading, subjectRef, createConnectionInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, org as orgSchema, orgMembership, resolvedIdentity, identityMembership, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, delegatedReadRecord, ownerTransferAudit, copyMarkAudit, subjectShredReceipt, connectorDispatchKind, callsOfManifestJson, outboundOfManifestJson, SCOPE_GATE_REASONS, substratError, redrainEventsInput, coverage, peerCoverage, peerGrantsEntry, peerSwitch, peerSwitchOutcome, verticalCaller, verticalResolution, verticalSlug, entityObjectRef, ATTACHMENT_BLOB_BINDING, blobStoreBindingName, lifecycleDelivery, scopeLifecycle, } from '@substrat-run/contracts';
|
|
5
5
|
import { normalizeHostname, toRouteTarget } from './route-resolver.js';
|
|
6
|
-
import { attachmentBlobKey, attachmentSha256, entitlementDenial, requiredEntitlementFor, foldMeterReading, parseValidationRecords, resolveScopeRecord, ulid, KEPT_COPY_REFUSAL, capabilityTokenHash, checkBecomeInput, plausibleSessionToken, foldModelUsage, backoffAt, resolveRetryPolicy, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, openSealed, ImpersonationRefused, assertSessionUsable, impersonationRowValues, mapImpersonationRow, newImpersonationSession, jobRunOf, runDueJobRuns, SYSTEM_DOOR_WAIT, startJobRun, attachmentTextJob, assertAttachmentExtractors, assertJobRegistrable, isAttachmentTextRun, searchLimit, searchMatchExpression, globalFetch, assertRedrainWindow, platformRequestOf, undrainedEventsOf, inUnitMovesToAudit, staleCarryRevertRow, staleCarryReverts, withRecorded, systemSwitchedOffMessage, tenantSystemSwitchedOffMessage, CrossVerticalRegistry, importCursorSourceOf, exportBreaksOf, exportBreakRefusal, bindExportBreaksOf, bindExportBreakRefusal, collectPeers, peerSeats, connectorCallRecord, noopConnectorCallRecorder, recordConnectorCall, settleConnectionUse, splitManifestMigrations, unknownRoleError, assertRowLimit, assertRowOffset, INERT_SCOPE_REASON, lifecycleReceipt, lifecycleRefusal, isPrimaryScope, isPrimaryScopeRow, asyncInvocationId, asyncLinePass, } from '@substrat-run/kernel';
|
|
6
|
+
import { attachmentBlobKey, attachmentSha256, entitlementDenial, requiredEntitlementFor, foldMeterReading, parseValidationRecords, resolveScopeRecord, ulid, KEPT_COPY_REFUSAL, capabilityTokenHash, checkBecomeInput, plausibleSessionToken, foldModelUsage, backoffAt, resolveRetryPolicy, executorOutcomeOf, isDeliveryRefusal, refusalJournalText, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, openSealed, ImpersonationRefused, assertSessionUsable, impersonationRowValues, mapImpersonationRow, newImpersonationSession, jobRunOf, assertLeaseMs, runDueJobRuns, SYSTEM_DOOR_WAIT, startJobRun, attachmentTextJob, assertAttachmentExtractors, assertJobRegistrable, isAttachmentTextRun, searchLimit, searchMatchExpression, globalFetch, assertRedrainWindow, platformRequestOf, undrainedEventsOf, inUnitMovesToAudit, staleCarryRevertRow, staleCarryReverts, recordedOffFromWire, peerSubjectRef, SWITCH_KINDS, reassertActionOf, reassertEntry, reassertOffRow, reassertOnRow, switchActionOf, switchAuditSubject, switchNotFoundMessage, recordWriteSuperseded, switchSubjectOf, switchSupersededMessage, withRecorded, systemSwitchedOffMessage, tenantSystemSwitchedOffMessage, CrossVerticalRegistry, importCursorSourceOf, exportBreaksOf, exportBreakRefusal, bindExportBreaksOf, bindExportBreakRefusal, collectPeers, peerSeats, connectorCallRecord, noopConnectorCallRecorder, recordConnectorCall, settleConnectionUse, splitManifestMigrations, unknownRoleError, assertRowLimit, assertRowOffset, emptyImportResult, INERT_SCOPE_REASON, lifecycleReceipt, lifecycleRefusal, isPrimaryScope, isPrimaryScopeRow, asyncInvocationId, asyncLinePass, } from '@substrat-run/kernel';
|
|
7
7
|
import { attributedHost } from '@substrat-run/kernel';
|
|
8
|
-
import { isOrangeToOrange, isUpgradeRequest, LIVE_MODE_HEADER, LIVE_PRINCIPAL_HEADER, LIVE_SCOPE_HEADER, LIVE_SUBSCRIBE_PATH, LIVE_TENANT_HEADER, } from './live-reads.js';
|
|
8
|
+
import { isOrangeToOrange, isUpgradeRequest, LIVE_MODE_HEADER, LIVE_PRINCIPAL_HEADER, LIVE_SCOPE_HEADER, LIVE_SUBSCRIBE_PATH, LIVE_TENANT_HEADER, LIVE_WITHIN_HEADER, encodeLiveWithin, liveWithinOf, } from './live-reads.js';
|
|
9
9
|
import { tenantStoreDatabaseName } from './d1.js';
|
|
10
10
|
import { blobStoreBucketName, r2TenantBlobStore } from './r2.js';
|
|
11
|
+
import { unwrapReply } from './do-reply.js';
|
|
11
12
|
/** DO row → contract shape. Never reads the secrets table — that is the split. */
|
|
12
13
|
const toConnection = (r) => connection.parse({
|
|
13
14
|
id: r.id,
|
|
@@ -26,12 +27,6 @@ const toConnection = (r) => connection.parse({
|
|
|
26
27
|
createdAt: r.created_at,
|
|
27
28
|
revokedAt: r.revoked_at,
|
|
28
29
|
});
|
|
29
|
-
/** Rethrow a capability attachment verb's failure, rebuilt with its code; else its value. */
|
|
30
|
-
function unwrapCapabilityReply(reply) {
|
|
31
|
-
if (reply.failure)
|
|
32
|
-
throw fromWireFailure(reply.failure);
|
|
33
|
-
return reply.value;
|
|
34
|
-
}
|
|
35
30
|
/**
|
|
36
31
|
* How far ahead of the clock a held lifecycle epoch may be and still be learned past (#1713). An
|
|
37
32
|
* epoch is the time a directory restore minted it, so a legitimate one is behind its minter's
|
|
@@ -92,24 +87,31 @@ function nullControlPlane() {
|
|
|
92
87
|
recordAdmin: noop,
|
|
93
88
|
recordAccess: noop,
|
|
94
89
|
tenantHoldsEntitlement: async () => true,
|
|
95
|
-
// #1674: the
|
|
90
|
+
// #1674, #2029: the switches' record is a DIRECTORY store, and a CP-less host has no
|
|
96
91
|
// directory — the shared control plane that delegates here keeps it. Same posture as
|
|
97
92
|
// `recordAdmin`: nothing to write, nothing recorded to read back.
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
93
|
+
recordSwitchedOff: async () => null,
|
|
94
|
+
recordSwitchedOn: async () => null,
|
|
95
|
+
restoreSwitchRecord: noop,
|
|
101
96
|
listSystemSwitches: async () => [],
|
|
102
|
-
|
|
103
|
-
|
|
97
|
+
switchRecordsOf: async () => [],
|
|
98
|
+
switchRecordStatesOf: async () => [],
|
|
99
|
+
switchesOwedOf: async () => [],
|
|
100
|
+
clearSwitchOwed: noop,
|
|
104
101
|
// #1823: no tenant tuples here either — the platform reads them and sends `tenantHeld`.
|
|
105
|
-
|
|
102
|
+
tenantHeldOf: async () => [],
|
|
103
|
+
};
|
|
104
|
+
const unavailable = (method) => new Error(`control plane unavailable: '${method}' — this host is scope-local / CP-less ` +
|
|
105
|
+
`(docs/architecture/scope-local-permissions.md, Phase 3)`);
|
|
106
|
+
// A directory write through `reply` (#113) is named for the write it carries, not for `reply`.
|
|
107
|
+
passthrough.reply = async (method) => {
|
|
108
|
+
throw unavailable(String(method));
|
|
106
109
|
};
|
|
107
110
|
return new Proxy({}, {
|
|
108
111
|
get: (_t, prop) => typeof prop === 'string' && prop in passthrough
|
|
109
112
|
? passthrough[prop]
|
|
110
113
|
: async () => {
|
|
111
|
-
throw
|
|
112
|
-
`(docs/architecture/scope-local-permissions.md, Phase 3)`);
|
|
114
|
+
throw unavailable(String(prop));
|
|
113
115
|
},
|
|
114
116
|
});
|
|
115
117
|
}
|
|
@@ -151,19 +153,34 @@ export const SWITCH_HOLD_EXTRA_WAITS = 3;
|
|
|
151
153
|
*/
|
|
152
154
|
export const SWITCH_HOLD_PENDING_MAX_MS = 5 * 60_000;
|
|
153
155
|
/**
|
|
154
|
-
*
|
|
155
|
-
*
|
|
156
|
-
*
|
|
156
|
+
* A move the deployment serving the scope applied, or may have, without attesting what the
|
|
157
|
+
* platform needs from it (#2045 Codex r3): the switch fence (`fenced`), or for a tenant-held
|
|
158
|
+
* module's OFF the evaluator that denies a tenant-level grant (`deniesTenantGrants`, #1823).
|
|
159
|
+
*
|
|
160
|
+
* The preflight (`switchTarget`'s `attestFence`) tells a deployment built before the fence apart
|
|
161
|
+
* before anything is recorded or moved. On such a deployment only an ON is refused: the race the
|
|
162
|
+
* fence closes is an older OFF landing after a newer ON, and with every ON refused there is no
|
|
163
|
+
* newer ON to land after, so an OFF goes through unfenced (#2045 follow-up). A kill switch that
|
|
164
|
+
* cannot be pulled until every vertical is re-pushed is the outage the fence must never cause.
|
|
165
|
+
* An unfenced OFF keeps its write-ahead owed mark, so the first re-assert after the vertical is
|
|
166
|
+
* redeployed moves the scope again under the record's fence.
|
|
167
|
+
*
|
|
168
|
+
* This error fires when a deployment that attested the fence answers a move without it — a
|
|
169
|
+
* rollback to an old build between the preflight and the move — or when an ON reaches a
|
|
170
|
+
* deployment without the fence. It is NOT compensated with an opposite move: an unfenced
|
|
171
|
+
* compensation can undo a newer call's switch (Codex r3, finding 2). The call's record and its
|
|
172
|
+
* write-ahead owed mark stay, and the call is refused.
|
|
157
173
|
*/
|
|
158
|
-
class
|
|
159
|
-
outcome;
|
|
160
|
-
constructor(scopeId, moduleId, outcome) {
|
|
161
|
-
super(`the deployment serving scope ${scopeId} predates the kill switch's tenant-grant denial (#1823): ` +
|
|
162
|
-
`module '${moduleId}' holds a tenant-level grant that its OFF would leave authorizing there. ` +
|
|
163
|
-
`Redeploy the vertical, then retry.`);
|
|
164
|
-
this.outcome = outcome;
|
|
165
|
-
}
|
|
174
|
+
class UnattestedSwitch extends Error {
|
|
166
175
|
}
|
|
176
|
+
/** A fenced move answered without `fenced`: the deployment predates the fence. */
|
|
177
|
+
const unfencedMessage = (scopeId) => `the deployment serving scope ${scopeId} predates the switch fence (#2045): it cannot refuse an older ` +
|
|
178
|
+
`switch call's OFF landing after this ON, so no switch is turned on there. Switching off still works. ` +
|
|
179
|
+
`Redeploy the vertical, then retry.`;
|
|
180
|
+
/** A tenant-held module's OFF answered without `deniesTenantGrants` (#1823). */
|
|
181
|
+
const undeniedMessage = (scopeId, moduleId) => `the deployment serving scope ${scopeId} predates the kill switch's tenant-grant denial (#1823): ` +
|
|
182
|
+
`module '${moduleId}' holds a tenant-level grant that its OFF would leave authorizing there. ` +
|
|
183
|
+
`Redeploy the vertical, then retry.`;
|
|
167
184
|
/**
|
|
168
185
|
* The attachment bucket a host resolves when it was handed no resolver (#1995): the
|
|
169
186
|
* platform's per-tenant binding, read off this script's own env, named by the one shared
|
|
@@ -172,6 +189,49 @@ class UnattestedSwitchOff extends Error {
|
|
|
172
189
|
function ambientAttachmentBucket(tenantId) {
|
|
173
190
|
return ambientEnv[blobStoreBindingName(ATTACHMENT_BLOB_BINDING, tenantId)] ?? null;
|
|
174
191
|
}
|
|
192
|
+
/**
|
|
193
|
+
* #2029: scope instances whose door has read the rewind hold and found NO claim on the scope at
|
|
194
|
+
* all, so their doors skip the hold read for the rest of the instance's life. Isolate-wide, because
|
|
195
|
+
* a host lives for one request: without it, every peer call would read the deployment's one hold
|
|
196
|
+
* object — an extra hop, on a single object, on the hot path of every scope never rewound.
|
|
197
|
+
*
|
|
198
|
+
* Safe because claims are only ever added for storage a LATER instance serves. A rewind writes its
|
|
199
|
+
* claim before it arms, and arming restarts the scope, so its rewound storage is always served by a
|
|
200
|
+
* new instance with a new id, which reads the hold afresh (`SWITCH_HOLD_SETTLE_MS` puts every row of
|
|
201
|
+
* the claim before that instance existed). A join (`switchHoldJoin`) goes into a rewind's claim too,
|
|
202
|
+
* so into that same later instance. An instance that read no claim therefore never gains one; claims
|
|
203
|
+
* only leave it, by a switch move. Only a GOOD read is remembered: a read that failed open is not.
|
|
204
|
+
* Bounded: past `CLEAR_SCOPE_INSTANCES_MAX` it is emptied, which costs a re-read, nothing else.
|
|
205
|
+
*/
|
|
206
|
+
const CLEAR_SCOPE_INSTANCES = new Set();
|
|
207
|
+
const CLEAR_SCOPE_INSTANCES_MAX = 10_000;
|
|
208
|
+
const rememberClearInstance = (key) => {
|
|
209
|
+
if (CLEAR_SCOPE_INSTANCES.size >= CLEAR_SCOPE_INSTANCES_MAX)
|
|
210
|
+
CLEAR_SCOPE_INSTANCES.clear();
|
|
211
|
+
CLEAR_SCOPE_INSTANCES.add(key);
|
|
212
|
+
};
|
|
213
|
+
/**
|
|
214
|
+
* #2045: a carry's fences, by tuple subject (`RecordedOffCarry.fences`): for each recorded-off
|
|
215
|
+
* subject, the operation id of the call the record holds.
|
|
216
|
+
*/
|
|
217
|
+
const switchFencesCarried = (kinds) => {
|
|
218
|
+
const out = {};
|
|
219
|
+
for (const [kind, { keys, fences }] of kinds) {
|
|
220
|
+
for (const key of keys) {
|
|
221
|
+
const fence = fences.get(key);
|
|
222
|
+
if (fence !== undefined)
|
|
223
|
+
out[switchSubjectOf(kind, key)] = fence;
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
return out;
|
|
227
|
+
};
|
|
228
|
+
/** #2029: how a door's refusals and a re-assert's name each kind of subject. */
|
|
229
|
+
const DOOR_WORDS = {
|
|
230
|
+
system: { what: (key) => `module '${key}'`, door: 'system door', switchName: 'schedule', switched: 'schedules' },
|
|
231
|
+
peer: { what: (key) => `peer vertical '${key}'`, door: 'peer door', switchName: 'peer', switched: 'peers' },
|
|
232
|
+
};
|
|
233
|
+
/** #2029: the rewind hold's key for one subject — a module's id, bare as #1819 stored it, or `vertical:<slug>`. */
|
|
234
|
+
const holdKeyOf = (kind, key) => (kind === 'system' ? key : peerSubjectRef(key));
|
|
175
235
|
export class CloudflareScopeHost {
|
|
176
236
|
admin;
|
|
177
237
|
scopeNs;
|
|
@@ -389,6 +449,22 @@ export class CloudflareScopeHost {
|
|
|
389
449
|
},
|
|
390
450
|
};
|
|
391
451
|
}
|
|
452
|
+
/**
|
|
453
|
+
* #1184: the scope reads an executor's handler may make. Plain RPCs to the ScopeDO: the
|
|
454
|
+
* handler runs here on the coordinator, after the DO's own task has returned.
|
|
455
|
+
*/
|
|
456
|
+
executorScope(tenantId, scopeId) {
|
|
457
|
+
const stub = this.scopeStub(scopeId);
|
|
458
|
+
return {
|
|
459
|
+
history: async (entity, page) => stub.entityHistory({ entityType: entity.entityType, entityId: entity.entityId, limit: page?.limit, cursor: page?.cursor }),
|
|
460
|
+
covers: async (principal, roleKey, level) => {
|
|
461
|
+
const bound = await stub.canAssignFor(tenantId, scopeId, principalId.parse(principal), roleKey, level === 'tenant');
|
|
462
|
+
if (!bound)
|
|
463
|
+
throw unknownRoleError(roleKey);
|
|
464
|
+
return coverage.parse(bound);
|
|
465
|
+
},
|
|
466
|
+
};
|
|
467
|
+
}
|
|
392
468
|
/**
|
|
393
469
|
* Drain this scope's outbox into the registered executors (K-22 §4.2).
|
|
394
470
|
*
|
|
@@ -412,7 +488,9 @@ export class CloudflareScopeHost {
|
|
|
412
488
|
* own `invocationId` field reads null — an ambient read there would record "no
|
|
413
489
|
* call" for every attempt an operation's own tail made.
|
|
414
490
|
*/
|
|
415
|
-
invocationId
|
|
491
|
+
invocationId,
|
|
492
|
+
/** #1184: what each attempt did, for the emitting call's `onExecutorOutcomes`. */
|
|
493
|
+
outcomes) {
|
|
416
494
|
const report = {
|
|
417
495
|
attempted: 0,
|
|
418
496
|
delivered: 0,
|
|
@@ -426,27 +504,22 @@ export class CloudflareScopeHost {
|
|
|
426
504
|
// attempt runs — the DO only journals it.
|
|
427
505
|
const lines = asyncLinePass();
|
|
428
506
|
try {
|
|
429
|
-
await this.drainExecutorsPass(tenantId, scopeId, invocationId, report, lines);
|
|
507
|
+
await this.drainExecutorsPass(tenantId, scopeId, invocationId, report, lines, outcomes);
|
|
430
508
|
}
|
|
431
509
|
finally {
|
|
432
510
|
lines.end();
|
|
433
511
|
}
|
|
434
512
|
return report;
|
|
435
513
|
}
|
|
436
|
-
async drainExecutorsPass(tenantId, scopeId, invocationId, report, lines) {
|
|
514
|
+
async drainExecutorsPass(tenantId, scopeId, invocationId, report, lines, outcomes) {
|
|
437
515
|
const stub = this.scopeStub(scopeId);
|
|
516
|
+
const scope = this.executorScope(tenantId, scopeId);
|
|
438
517
|
// #2005: a non-primary scope (a fork, a snapshot, a preview of either kind) causes no
|
|
439
518
|
// outbound effects, so its deliveries are journaled terminal with the reason and no
|
|
440
|
-
// handler runs. Asked
|
|
441
|
-
//
|
|
442
|
-
// `is_copy`, #2009, set on the directory's word) — a preview and a snapshot reach a hosted
|
|
443
|
-
// vertical as restores the platform marks.
|
|
444
|
-
// Asked on the first due event, once per pass: most passes have none, and the directory
|
|
445
|
-
// is one global object.
|
|
519
|
+
// handler runs (`isInertScope`). Asked on the first due event, once per pass: most passes
|
|
520
|
+
// have none, and the directory is one global object.
|
|
446
521
|
let inert;
|
|
447
|
-
const isInert = () => (inert ??= this.
|
|
448
|
-
? stub.isCopy()
|
|
449
|
-
: this.cp.getScopeRecord(tenantId, scopeId).then((row) => !isPrimaryScopeRow(row)));
|
|
522
|
+
const isInert = () => (inert ??= this.isInertScope(tenantId, scopeId));
|
|
450
523
|
// #1713: a CP-less scope its lifecycle holds attempts nothing and journals nothing, so every
|
|
451
524
|
// due delivery stays due for the first pass after it is live again. Asked on the first due
|
|
452
525
|
// event, like `isInert`. Every caller passed `assertLive` already, so this is for the scope
|
|
@@ -477,6 +550,9 @@ export class CloudflareScopeHost {
|
|
|
477
550
|
attempt,
|
|
478
551
|
startedAt: Date.now(),
|
|
479
552
|
});
|
|
553
|
+
const outcomeOf = (event, outcome, error) => {
|
|
554
|
+
outcomes?.push(executorOutcomeOf(id, event, outcome, error));
|
|
555
|
+
};
|
|
480
556
|
for (const bad of undecodable) {
|
|
481
557
|
report.attempted += 1;
|
|
482
558
|
const attempt = await stub.recordExecutorAttempt(bad.eventId, deliveryId, bad.error, null, invocationId);
|
|
@@ -490,6 +566,7 @@ export class CloudflareScopeHost {
|
|
|
490
566
|
// No next attempt, like an undecodable row: a scope does not become primary.
|
|
491
567
|
const attempt = await stub.recordExecutorAttempt(event.id, deliveryId, INERT_SCOPE_REASON, null, invocationId);
|
|
492
568
|
report.inert = (report.inert ?? 0) + 1;
|
|
569
|
+
outcomeOf(event, 'inert', INERT_SCOPE_REASON);
|
|
493
570
|
// Its own outcome, never `delivered`: no handler ran (#2005).
|
|
494
571
|
lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'inert' });
|
|
495
572
|
continue;
|
|
@@ -507,6 +584,7 @@ export class CloudflareScopeHost {
|
|
|
507
584
|
// the at-least-once residue, as it already must in-process.
|
|
508
585
|
await stub.routeExecutorEventToPlatform(event.id, deliveryId, connectorDispatchKind(executor.provider), JSON.stringify({ executorId: id, event }), JSON.stringify({ system: 'connector-dispatch' }), invocationId);
|
|
509
586
|
report.routedToPlatform += 1;
|
|
587
|
+
outcomeOf(event, 'routed');
|
|
510
588
|
// Handed on, not run here: the platform's drain owns its attempts from now on.
|
|
511
589
|
lines.write({ ...unitOf(event.id, 1), startedAt, outcome: 'routed' });
|
|
512
590
|
}
|
|
@@ -514,13 +592,26 @@ export class CloudflareScopeHost {
|
|
|
514
592
|
await executor.handler(await this.connectorContext(tenantId, scopeId, executor.timeoutMs, event.id), event);
|
|
515
593
|
const attempt = await stub.recordExecutorAttempt(event.id, deliveryId, null, null, invocationId);
|
|
516
594
|
report.delivered += 1;
|
|
595
|
+
outcomeOf(event, 'delivered');
|
|
517
596
|
lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'delivered' });
|
|
518
597
|
}
|
|
519
598
|
else {
|
|
520
|
-
await executor.handler(this.admin, event);
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
599
|
+
const result = await executor.handler(this.admin, event, scope);
|
|
600
|
+
// #1184: a refusal is the handler's own terminal decision — journaled with its
|
|
601
|
+
// reason, no next attempt, listed by `executorDeadLetters` beside an exhausted one.
|
|
602
|
+
const refused = isDeliveryRefusal(result) ? result : null;
|
|
603
|
+
const text = refused && refusalJournalText(refused);
|
|
604
|
+
const attempt = await stub.recordExecutorAttempt(event.id, deliveryId, text, null, invocationId);
|
|
605
|
+
if (refused) {
|
|
606
|
+
report.deadLettered += 1;
|
|
607
|
+
outcomeOf(event, 'refused', refused.reason);
|
|
608
|
+
lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'dead-lettered', error: text });
|
|
609
|
+
}
|
|
610
|
+
else {
|
|
611
|
+
report.delivered += 1;
|
|
612
|
+
outcomeOf(event, 'delivered');
|
|
613
|
+
lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'delivered' });
|
|
614
|
+
}
|
|
524
615
|
}
|
|
525
616
|
}
|
|
526
617
|
catch (err) {
|
|
@@ -535,6 +626,7 @@ export class CloudflareScopeHost {
|
|
|
535
626
|
report.deadLettered += 1;
|
|
536
627
|
else
|
|
537
628
|
report.retrying += 1;
|
|
629
|
+
outcomeOf(event, exhausted ? 'dead-lettered' : 'retrying', err);
|
|
538
630
|
lines.write({
|
|
539
631
|
...unitOf(event.id, attempts),
|
|
540
632
|
startedAt,
|
|
@@ -558,13 +650,14 @@ export class CloudflareScopeHost {
|
|
|
558
650
|
// emitting operation's own tail made.
|
|
559
651
|
return this.drainExecutors(tenantId, scopeId, null);
|
|
560
652
|
}
|
|
561
|
-
registerJob(moduleId, name, handler, retry) {
|
|
653
|
+
registerJob(moduleId, name, handler, retry, options) {
|
|
562
654
|
// #1575: the kernel's own jobs are dispatched before this registry is read.
|
|
563
655
|
assertJobRegistrable(moduleId, name);
|
|
656
|
+
assertLeaseMs(options?.leaseMs);
|
|
564
657
|
const key = `${moduleId}/${name}`;
|
|
565
658
|
if (this.jobs.has(key))
|
|
566
659
|
throw new Error(`job '${key}' is already registered`);
|
|
567
|
-
this.jobs.set(key, { handler, retry });
|
|
660
|
+
this.jobs.set(key, { handler, retry, leaseMs: options?.leaseMs });
|
|
568
661
|
}
|
|
569
662
|
/**
|
|
570
663
|
* D-14's DURABLE driver: the run store is RPC to the scope DO, and nothing else.
|
|
@@ -582,13 +675,15 @@ export class CloudflareScopeHost {
|
|
|
582
675
|
const stub = this.scopeStub(scopeId);
|
|
583
676
|
return {
|
|
584
677
|
startOrJoin: (key, row) => stub.jobRunStartOrJoin(key.moduleId, key.job, key.instance, row),
|
|
585
|
-
get: (id) => stub.jobRunById(id),
|
|
586
678
|
dueKeys: (now, max) => stub.jobRunsDueKeys(now, max),
|
|
679
|
+
claim: (id, owner, leaseMs) => stub.jobRunClaim(id, owner, leaseMs),
|
|
680
|
+
begin: (id, owner, marginMs) => stub.jobRunBegin(id, owner, marginMs),
|
|
681
|
+
miss: (id, owner, note) => stub.jobRunMiss(id, owner, note),
|
|
587
682
|
list: (filter) => stub.jobRunList(filter),
|
|
588
|
-
patch: (id, patch) => stub.jobRunPatch(id, patch),
|
|
589
|
-
commitPass: (id, patch) => stub.jobCommitPass(id, patch),
|
|
590
|
-
|
|
591
|
-
recordStep: (runId, name, result, attempts, lastError, at) => stub.jobStepRecord(runId, name, result, attempts, lastError, at),
|
|
683
|
+
patch: (id, patch, owner) => stub.jobRunPatch(id, patch, owner),
|
|
684
|
+
commitPass: (id, patch, owner) => stub.jobCommitPass(id, patch, owner),
|
|
685
|
+
beginStep: (runId, name, owner, leaseMs) => stub.jobStepBegin(runId, name, owner, leaseMs),
|
|
686
|
+
recordStep: (runId, name, result, attempts, lastError, at, owner, leaseMs) => stub.jobStepRecord(runId, name, result, attempts, lastError, at, owner, leaseMs),
|
|
592
687
|
};
|
|
593
688
|
}
|
|
594
689
|
/**
|
|
@@ -710,7 +805,7 @@ export class CloudflareScopeHost {
|
|
|
710
805
|
if (!rec)
|
|
711
806
|
throw unknownScopeForTenant(tenantId, scopeId);
|
|
712
807
|
if (rec.status !== 'active' && rec.status !== 'provisioning') {
|
|
713
|
-
throw
|
|
808
|
+
throw substratError('conflict', `scope not migratable (status: ${rec.status}): ${scopeId}`);
|
|
714
809
|
}
|
|
715
810
|
const stub = this.scopeStub(scopeId);
|
|
716
811
|
try {
|
|
@@ -752,11 +847,11 @@ export class CloudflareScopeHost {
|
|
|
752
847
|
return this.scopeStub(scopeId).introspectTables();
|
|
753
848
|
}
|
|
754
849
|
async introspectScopeTable(scopeId, input) {
|
|
755
|
-
return this.scopeStub(scopeId).
|
|
850
|
+
return unwrapReply(await this.scopeStub(scopeId).introspectTableReply(input.table, input.limit, input.offset));
|
|
756
851
|
}
|
|
757
852
|
/** The SQL console's CP-less path (#219) — same trust line as the pair above. */
|
|
758
853
|
async introspectScopeQuery(scopeId, input) {
|
|
759
|
-
return this.scopeStub(scopeId).
|
|
854
|
+
return unwrapReply(await this.scopeStub(scopeId).introspectQueryReply(input.sql));
|
|
760
855
|
}
|
|
761
856
|
/**
|
|
762
857
|
* The K-35 denial log's CP-less path (#867) — same trust line as the reads above.
|
|
@@ -853,10 +948,9 @@ export class CloudflareScopeHost {
|
|
|
853
948
|
opts) {
|
|
854
949
|
if (opts?.markCopy)
|
|
855
950
|
assertCopyLineage(opts.markCopy);
|
|
951
|
+
const carry = opts ? recordedOffFromWire(opts) : undefined;
|
|
856
952
|
const load = {
|
|
857
|
-
switchOff:
|
|
858
|
-
? { moduleIds: opts.switchedOff, at: new Date().toISOString(), tenantHeld: opts.tenantHeld }
|
|
859
|
-
: undefined,
|
|
953
|
+
switchOff: carry ? { at: new Date().toISOString(), ...carry } : undefined,
|
|
860
954
|
sourceScopeId: opts?.sourceScopeId,
|
|
861
955
|
exact: opts?.exact,
|
|
862
956
|
loadStamp: opts?.loadStamp,
|
|
@@ -870,7 +964,7 @@ export class CloudflareScopeHost {
|
|
|
870
964
|
...(opts?.expect ? { expect: opts.expect } : {}),
|
|
871
965
|
});
|
|
872
966
|
if (out.refused === false) {
|
|
873
|
-
return { tables: tables.length, ...(
|
|
967
|
+
return { tables: tables.length, ...(carry ? { switchedOff: out.switchedOff } : {}) };
|
|
874
968
|
}
|
|
875
969
|
if (out.refused === 'kept')
|
|
876
970
|
throw substratError('conflict', KEPT_COPY_REFUSAL);
|
|
@@ -993,7 +1087,7 @@ export class CloudflareScopeHost {
|
|
|
993
1087
|
* safe; scope-level tuples (the restored grants) are never touched.
|
|
994
1088
|
*/
|
|
995
1089
|
async projectRolesLocal(tenantId, scopeId, roles) {
|
|
996
|
-
await this.scopeStub(scopeId).
|
|
1090
|
+
unwrapReply(await this.scopeStub(scopeId).applyProjectionReply(tenantId, roles.map((r) => ({ role_key: r.key, permissions: JSON.stringify(r.permissions), source: r.source })), []));
|
|
997
1091
|
}
|
|
998
1092
|
/**
|
|
999
1093
|
* Wipe one scope DO's storage in THIS deployment — the reap half of an orchestrated
|
|
@@ -1045,6 +1139,10 @@ export class CloudflareScopeHost {
|
|
|
1045
1139
|
async lifecycleFlowLocal(scopeId, input) {
|
|
1046
1140
|
return this.scopeStub(scopeId).lifecycleFlow(input);
|
|
1047
1141
|
}
|
|
1142
|
+
/** Business volumes per bucket (#1750) on this host's own scope — the vertical-host read. */
|
|
1143
|
+
async operationSeriesLocal(scopeId, input) {
|
|
1144
|
+
return this.scopeStub(scopeId).operationSeries(input);
|
|
1145
|
+
}
|
|
1048
1146
|
/** One event's causal chain (#1237) on this host's own scope — the vertical-host read. */
|
|
1049
1147
|
async eventCauseLocal(scopeId, input) {
|
|
1050
1148
|
return this.scopeStub(scopeId).eventCause(input);
|
|
@@ -1197,7 +1295,7 @@ export class CloudflareScopeHost {
|
|
|
1197
1295
|
const record = resolveScopeRecord(input);
|
|
1198
1296
|
// Fail-closed tenant gate throws out of the awaited cp call BEFORE migrate
|
|
1199
1297
|
// or audit, so a rejected provision creates nothing and writes no audit row.
|
|
1200
|
-
const created = await this.
|
|
1298
|
+
const created = await this.directory('provisionScope', input.tenantId, input.scopeId, record, new Date().toISOString());
|
|
1201
1299
|
// Instantiate the scope DO and trigger its lazy migration.
|
|
1202
1300
|
await this.migrateAndRecord(input.scopeId);
|
|
1203
1301
|
// Scope-local permissions: a freshly-provisioned scope evaluates from its own
|
|
@@ -1238,17 +1336,9 @@ export class CloudflareScopeHost {
|
|
|
1238
1336
|
// #1742: in the SAME unit as the seat, so no sweep can land between them; the re-assert
|
|
1239
1337
|
// after it then finds them off, and audits what the unit moved.
|
|
1240
1338
|
const ownStore = this.cpLess || record.vertical === null;
|
|
1241
|
-
const recordedOff = ownStore ? await this.cp.switchedOffModulesOf(input.tenantId, input.scopeId) : [];
|
|
1242
1339
|
const seatAt = new Date().toISOString();
|
|
1243
|
-
const
|
|
1244
|
-
|
|
1245
|
-
scopeId: input.scopeId,
|
|
1246
|
-
moduleIds: recordedOff,
|
|
1247
|
-
at: seatAt,
|
|
1248
|
-
// #1823: a recorded-off module held only by a tenant grant goes back off too.
|
|
1249
|
-
tenantHeld: await this.cp.tenantHeldSystemModules(input.tenantId, recordedOff, seatAt),
|
|
1250
|
-
}
|
|
1251
|
-
: undefined);
|
|
1340
|
+
const carry = ownStore ? await this.recordedOffCarry(input.tenantId, input.scopeId, seatAt) : undefined;
|
|
1341
|
+
const switchedOff = await this.scopeStub(input.scopeId).seatTuples(seats, carry ? { scopeId: input.scopeId, at: seatAt, ...carry } : undefined);
|
|
1252
1342
|
if (ownStore) {
|
|
1253
1343
|
await this.admin.reassertSystemSwitches(actor, { tenantId: input.tenantId, scopeId: input.scopeId }, { appliedInUnit: switchedOff });
|
|
1254
1344
|
}
|
|
@@ -1270,10 +1360,10 @@ export class CloudflareScopeHost {
|
|
|
1270
1360
|
// inside the DO's ledger write, which is the serialization point that actually holds.
|
|
1271
1361
|
const tenant = await this.cp.getTenant(input.tenantId);
|
|
1272
1362
|
if (!tenant) {
|
|
1273
|
-
throw
|
|
1363
|
+
throw substratError('not_found', `cannot provision tenant store under unknown tenant: ${input.tenantId}`);
|
|
1274
1364
|
}
|
|
1275
1365
|
if (tenant.status !== 'active') {
|
|
1276
|
-
throw
|
|
1366
|
+
throw substratError('conflict', `cannot provision tenant store under non-active tenant (status: ${tenant.status}): ${input.tenantId}`);
|
|
1277
1367
|
}
|
|
1278
1368
|
// Idempotent on (tenant, vertical, binding): a retried provision re-resolves the SAME
|
|
1279
1369
|
// store rather than minting a second database (the K-31 ready-gate retries the whole
|
|
@@ -1288,7 +1378,7 @@ export class CloudflareScopeHost {
|
|
|
1288
1378
|
// the source of truth, carrying the D1 database_id Cloudflare assigned as the ref.
|
|
1289
1379
|
const name = await tenantStoreDatabaseName(input.tenantId, input.vertical, input.binding);
|
|
1290
1380
|
const ref = await d1.create(name);
|
|
1291
|
-
const stored = await this.
|
|
1381
|
+
const stored = await this.directory('putTenantStore', {
|
|
1292
1382
|
tenantId: input.tenantId,
|
|
1293
1383
|
vertical: input.vertical,
|
|
1294
1384
|
binding: input.binding,
|
|
@@ -1339,17 +1429,17 @@ export class CloudflareScopeHost {
|
|
|
1339
1429
|
const r2 = this.requireBlobStores(`provisionBlobStore(tenant=${input.tenantId} vertical=${input.vertical} binding=${input.binding})`);
|
|
1340
1430
|
const tenant = await this.cp.getTenant(input.tenantId);
|
|
1341
1431
|
if (!tenant) {
|
|
1342
|
-
throw
|
|
1432
|
+
throw substratError('not_found', `cannot provision blob store under unknown tenant: ${input.tenantId}`);
|
|
1343
1433
|
}
|
|
1344
1434
|
if (tenant.status !== 'active') {
|
|
1345
|
-
throw
|
|
1435
|
+
throw substratError('conflict', `cannot provision blob store under non-active tenant (status: ${tenant.status}): ${input.tenantId}`);
|
|
1346
1436
|
}
|
|
1347
1437
|
const existing = await this.cp.getBlobStore(input.tenantId, input.vertical, input.binding);
|
|
1348
1438
|
if (existing)
|
|
1349
1439
|
return { binding: input.binding, kind: 'blob', ref: existing.ref };
|
|
1350
1440
|
const name = await blobStoreBucketName(input.tenantId, input.vertical, input.binding);
|
|
1351
1441
|
const ref = await r2.create(name);
|
|
1352
|
-
const stored = await this.
|
|
1442
|
+
const stored = await this.directory('putBlobStore', {
|
|
1353
1443
|
tenantId: input.tenantId,
|
|
1354
1444
|
vertical: input.vertical,
|
|
1355
1445
|
binding: input.binding,
|
|
@@ -1488,22 +1578,22 @@ export class CloudflareScopeHost {
|
|
|
1488
1578
|
// the DO only to be refused and recorded — never to the blob store.
|
|
1489
1579
|
const hash = subject.capabilitySession;
|
|
1490
1580
|
return {
|
|
1491
|
-
upload: async (input) =>
|
|
1581
|
+
upload: async (input) => unwrapReply(await stub.capabilityAttachmentRefuseWrite(hash, tenantId, scopeId, 'attachments.upload', {
|
|
1492
1582
|
entityType: input.entity.entityType,
|
|
1493
1583
|
})),
|
|
1494
|
-
list: async (entity) =>
|
|
1584
|
+
list: async (entity) => unwrapReply(await stub.capabilityAttachmentList(entity, hash, tenantId, scopeId)),
|
|
1495
1585
|
open: async (attachmentId) => {
|
|
1496
|
-
const record =
|
|
1586
|
+
const record = unwrapReply(await stub.capabilityAttachmentOpen(attachmentId, hash, tenantId, scopeId));
|
|
1497
1587
|
return record ? this.openAttachmentBytes(store, scopeId, record) : null;
|
|
1498
1588
|
},
|
|
1499
|
-
remove: async (attachmentId) =>
|
|
1589
|
+
remove: async (attachmentId) => unwrapReply(await stub.capabilityAttachmentRefuseWrite(hash, tenantId, scopeId, 'attachments.remove', {
|
|
1500
1590
|
attachmentId,
|
|
1501
1591
|
})),
|
|
1502
1592
|
search: async (term, options) => {
|
|
1503
1593
|
// Judged here too, so a short term reaches the caller as `SearchTermTooShort`
|
|
1504
1594
|
// rather than as a message that crossed the RPC boundary. The DO judges again.
|
|
1505
1595
|
searchMatchExpression(term, 'prefix');
|
|
1506
|
-
return
|
|
1596
|
+
return unwrapReply(await stub.capabilityAttachmentSearch(term, searchLimit(options?.limit), hash, tenantId, scopeId));
|
|
1507
1597
|
},
|
|
1508
1598
|
};
|
|
1509
1599
|
}
|
|
@@ -1558,7 +1648,7 @@ export class CloudflareScopeHost {
|
|
|
1558
1648
|
throw new Error('attachments.search is not available on a delivery-scoped surface');
|
|
1559
1649
|
}
|
|
1560
1650
|
searchMatchExpression(term, 'prefix');
|
|
1561
|
-
return
|
|
1651
|
+
return unwrapReply(await stub.attachmentSearch(term, searchLimit(options?.limit), asPrincipalId, tenantId, scopeId, connectionId));
|
|
1562
1652
|
},
|
|
1563
1653
|
};
|
|
1564
1654
|
}
|
|
@@ -1606,24 +1696,51 @@ export class CloudflareScopeHost {
|
|
|
1606
1696
|
// this is the scope's real store, as for the provision's seat above; a hosted scope is
|
|
1607
1697
|
// restored in its deployment, which the platform carries the same list to.
|
|
1608
1698
|
const ownStore = this.cpLess || existing.vertical === null;
|
|
1609
|
-
const recordedOff = ownStore ? await this.cp.switchedOffModulesOf(tenantId, scopeId) : [];
|
|
1610
1699
|
const replayAt = new Date().toISOString();
|
|
1700
|
+
const carry = ownStore ? await this.recordedOffCarry(tenantId, scopeId, replayAt) : undefined;
|
|
1611
1701
|
const switchedOff = await this.scopeStub(scopeId).importDump(dump.tables, scopeId, {
|
|
1612
|
-
switchOff:
|
|
1613
|
-
? {
|
|
1614
|
-
moduleIds: recordedOff,
|
|
1615
|
-
at: replayAt,
|
|
1616
|
-
// #1823: a recorded-off module held only by a tenant grant goes back off too.
|
|
1617
|
-
tenantHeld: await this.cp.tenantHeldSystemModules(tenantId, recordedOff, replayAt),
|
|
1618
|
-
}
|
|
1619
|
-
: undefined,
|
|
1702
|
+
switchOff: carry ? { at: replayAt, ...carry } : undefined,
|
|
1620
1703
|
sourceScopeId: opts?.sourceScopeId ?? dump.scopeId,
|
|
1621
1704
|
});
|
|
1622
1705
|
await this.recordAdmin(actor, 'restoreScope', { tenantId, scopeId }, null, { sourceScopeId: dump.scopeId, tables: dump.tables.length, capturedAt: dump.capturedAt });
|
|
1623
|
-
if (
|
|
1706
|
+
if (carry) {
|
|
1624
1707
|
await this.admin.reassertSystemSwitches(actor, { tenantId, scopeId }, { appliedInUnit: switchedOff });
|
|
1625
1708
|
}
|
|
1626
1709
|
}
|
|
1710
|
+
/**
|
|
1711
|
+
* #1742, #2029: what this host's own unit carries of the directory's record — the modules and
|
|
1712
|
+
* the peers recorded OFF on the scope, each with the ones held there only by a tenant-level
|
|
1713
|
+
* grant (#1823, #2030). Undefined when nothing is recorded off, so such a unit is unchanged.
|
|
1714
|
+
*/
|
|
1715
|
+
async recordedOffCarry(tenantId, scopeId, at) {
|
|
1716
|
+
const [modules, peers] = await Promise.all([
|
|
1717
|
+
this.recordedOff('system', tenantId, scopeId, at),
|
|
1718
|
+
this.recordedOff('peer', tenantId, scopeId, at),
|
|
1719
|
+
]);
|
|
1720
|
+
if (!modules.keys.length && !peers.keys.length)
|
|
1721
|
+
return undefined;
|
|
1722
|
+
return {
|
|
1723
|
+
moduleIds: modules.keys,
|
|
1724
|
+
tenantHeld: modules.tenantHeld,
|
|
1725
|
+
verticals: peers.keys,
|
|
1726
|
+
tenantHeldVerticals: peers.tenantHeld,
|
|
1727
|
+
fences: switchFencesCarried([
|
|
1728
|
+
['system', modules],
|
|
1729
|
+
['peer', peers],
|
|
1730
|
+
]),
|
|
1731
|
+
};
|
|
1732
|
+
}
|
|
1733
|
+
/** The subjects of one kind the directory records OFF on the scope, and of those the tenant-held ones. */
|
|
1734
|
+
async recordedOff(kind, tenantId, scopeId, at) {
|
|
1735
|
+
// #2045: the OFF rows and their fences in one read, so every key carried has its own row's fence.
|
|
1736
|
+
const fences = new Map((await this.cp.switchRecordStatesOf(kind, tenantId, scopeId))
|
|
1737
|
+
.filter(([, row]) => row.position === 'off')
|
|
1738
|
+
.map(([key, row]) => [key, row.fence]));
|
|
1739
|
+
const keys = [...fences.keys()];
|
|
1740
|
+
if (!keys.length)
|
|
1741
|
+
return { keys, tenantHeld: [], fences };
|
|
1742
|
+
return { keys, tenantHeld: await this.cp.tenantHeldOf(kind, tenantId, keys, at), fences };
|
|
1743
|
+
}
|
|
1627
1744
|
async snapshotScope(actor, tenantId, scopeId, opts) {
|
|
1628
1745
|
const source = await this.admin.getScopeRecord(actor, tenantId, scopeId);
|
|
1629
1746
|
if (!source)
|
|
@@ -1654,7 +1771,7 @@ export class CloudflareScopeHost {
|
|
|
1654
1771
|
if (!rec)
|
|
1655
1772
|
throw substratError('not_found', `unknown scope ${scopeId} in tenant ${tenantId}`);
|
|
1656
1773
|
if (!rec.forkedFrom && rec.kind !== 'preview') {
|
|
1657
|
-
throw
|
|
1774
|
+
throw substratError('forbidden', `scope ${scopeId} is not a fork or preview — only previews may be deleted; ` +
|
|
1658
1775
|
`archive a primary scope instead`);
|
|
1659
1776
|
}
|
|
1660
1777
|
// Storage BEFORE the directory row: a crash between the two leaves a visible row
|
|
@@ -1815,34 +1932,55 @@ export class CloudflareScopeHost {
|
|
|
1815
1932
|
/** #1834: the job pass this door is opened for; its "not now" refusals are tied to it. */
|
|
1816
1933
|
pass) {
|
|
1817
1934
|
if (!this.moduleIds.has(moduleId)) {
|
|
1818
|
-
throw
|
|
1935
|
+
throw substratError('not_found', `module not registered on this host: ${moduleId}`);
|
|
1819
1936
|
}
|
|
1820
1937
|
await this.assertLive(tenantId, scopeId);
|
|
1821
1938
|
await this.migrateAndRecord(scopeId);
|
|
1939
|
+
return this.openDoor('system', moduleId, scopeId, gated, pass);
|
|
1940
|
+
}
|
|
1941
|
+
/**
|
|
1942
|
+
* #1834: one door, for either kind of switched subject (#2029). The gate is read here unless the
|
|
1943
|
+
* caller read it (`runDueSchedules` reports a hold it could not read on its pass report); a held
|
|
1944
|
+
* subject is refused, and a call landing on another instance than the gate's re-gates, at most
|
|
1945
|
+
* `SYSTEM_DOOR_REGATES` times, then fails closed.
|
|
1946
|
+
*
|
|
1947
|
+
* The peer door (`kind: 'peer'`) is the system door's for a peer vertical. A PITR rewind to before
|
|
1948
|
+
* a peer was switched off brings its grants back live and its OFF marker gone, so the rewind holds
|
|
1949
|
+
* it outside the scope (`rewindHolding`, as for a module), and every peer entry — an invoke, a
|
|
1950
|
+
* delivery, a coverage read, an export read for it as consumer — opens this door after
|
|
1951
|
+
* `peerScopeGate`.
|
|
1952
|
+
*/
|
|
1953
|
+
async openDoor(kind, key, scopeId, gated, pass) {
|
|
1954
|
+
const words = DOOR_WORDS[kind];
|
|
1955
|
+
const what = words.what(key);
|
|
1822
1956
|
// A gate this door reads itself reports a hold it could not read; one handed in was the
|
|
1823
|
-
// caller's to report
|
|
1957
|
+
// caller's to report.
|
|
1824
1958
|
const regate = async () => {
|
|
1825
|
-
const fresh = await this.
|
|
1959
|
+
const fresh = await this.doorGate(kind, scopeId, key);
|
|
1826
1960
|
if (fresh.error)
|
|
1827
|
-
console.error(`substrat:
|
|
1961
|
+
console.error(`substrat: door of ${what} on ${scopeId}: ${fresh.error}`);
|
|
1828
1962
|
return fresh;
|
|
1829
1963
|
};
|
|
1830
1964
|
let gate = gated ?? (await regate());
|
|
1831
1965
|
return {
|
|
1832
|
-
|
|
1966
|
+
kind,
|
|
1967
|
+
key,
|
|
1833
1968
|
through: async (call) => {
|
|
1834
1969
|
for (let regates = 0;; regates += 1) {
|
|
1835
1970
|
if (gate.held) {
|
|
1836
1971
|
// The hold ends when the switch is applied again: a job run waits it out without
|
|
1837
1972
|
// spending its retries, because the driver defers on THIS refusal (`doorWait`).
|
|
1838
|
-
|
|
1839
|
-
|
|
1973
|
+
const held = substratError('forbidden', `${what} is held off on this scope: it was rewound to before its ` +
|
|
1974
|
+
`${words.switchName} switch was turned off, and it stays off until ` +
|
|
1975
|
+
'the switch is applied again', { reason: SYSTEM_DOOR_WAIT });
|
|
1976
|
+
this.heldRefusals.add(held);
|
|
1977
|
+
throw this.doorWait(pass, held);
|
|
1840
1978
|
}
|
|
1841
1979
|
const answer = await call(gate.instance);
|
|
1842
1980
|
if (!isSystemDoorMoved(answer))
|
|
1843
1981
|
return answer;
|
|
1844
1982
|
if (regates >= SYSTEM_DOOR_REGATES) {
|
|
1845
|
-
throw this.doorWait(pass, substratError('unavailable', `the scope kept restarting under the
|
|
1983
|
+
throw this.doorWait(pass, substratError('unavailable', `the scope kept restarting under the ${words.door} of ${what} ` +
|
|
1846
1984
|
`(${regates + 1} attempts); nothing ran. Retry later`, { reason: SYSTEM_DOOR_WAIT }));
|
|
1847
1985
|
}
|
|
1848
1986
|
gate = await regate();
|
|
@@ -1859,6 +1997,16 @@ export class CloudflareScopeHost {
|
|
|
1859
1997
|
* on a later pass is an ordinary failure there. Held weakly: an error nobody holds goes.
|
|
1860
1998
|
*/
|
|
1861
1999
|
doorWaits = new WeakMap();
|
|
2000
|
+
/**
|
|
2001
|
+
* #2029: the hold refusals this host's doors threw, by identity — so a peer delivery pauses its
|
|
2002
|
+
* edge, and a coverage read answers "holds nothing", on exactly those and on no error that only
|
|
2003
|
+
* looks like one. Held weakly, as `doorWaits` is.
|
|
2004
|
+
*/
|
|
2005
|
+
heldRefusals = new WeakSet();
|
|
2006
|
+
/** #2029: is `err` a hold refusal this host's door threw? */
|
|
2007
|
+
isHeldRefusal(err) {
|
|
2008
|
+
return typeof err === 'object' && err !== null && this.heldRefusals.has(err);
|
|
2009
|
+
}
|
|
1862
2010
|
/** #1834: mark this door's own refusal as "not now" for the pass it was opened for, and hand it back. */
|
|
1863
2011
|
doorWait(pass, err) {
|
|
1864
2012
|
if (pass)
|
|
@@ -1873,13 +2021,23 @@ export class CloudflareScopeHost {
|
|
|
1873
2021
|
return true;
|
|
1874
2022
|
}
|
|
1875
2023
|
/**
|
|
1876
|
-
* #1834:
|
|
1877
|
-
*
|
|
2024
|
+
* #1834: a door's gate — the scope's state read for the subject, the instance that answered it,
|
|
2025
|
+
* and then the rewind hold (#1819), for any subject the scope does not already have off. The
|
|
1878
2026
|
* hold is read AFTER the state, which is the order `switchHeld` needs.
|
|
2027
|
+
*
|
|
2028
|
+
* `ungranted` reads the hold too (#2029): a subject held on the scope only by a TENANT-level
|
|
2029
|
+
* grant (#1823, #2030) comes back from a rewind with no scope grant and no marker, and the
|
|
2030
|
+
* tenant grant still authorizes it there. A rewind captured it off, so the hold must keep it so.
|
|
1879
2031
|
*/
|
|
1880
|
-
async
|
|
1881
|
-
const
|
|
1882
|
-
const
|
|
2032
|
+
async doorGate(kind, scopeId, key) {
|
|
2033
|
+
const stub = this.scopeStub(scopeId);
|
|
2034
|
+
const { state, instance } = await (kind === 'system' ? stub.systemDoorState(key) : stub.peerDoorState(key));
|
|
2035
|
+
const cleared = `${scopeId} ${instance}`;
|
|
2036
|
+
if (state === 'off' || CLEAR_SCOPE_INSTANCES.has(cleared))
|
|
2037
|
+
return { state, instance, held: false };
|
|
2038
|
+
const hold = await this.switchHeld(scopeId, holdKeyOf(kind, key));
|
|
2039
|
+
if (hold.scopeClear)
|
|
2040
|
+
rememberClearInstance(cleared);
|
|
1883
2041
|
return { state, instance, held: hold.held, ...(hold.error ? { error: hold.error } : {}) };
|
|
1884
2042
|
}
|
|
1885
2043
|
/**
|
|
@@ -1893,8 +2051,25 @@ export class CloudflareScopeHost {
|
|
|
1893
2051
|
// The peer door's own gate (#1706): K-3's pair check, the refusal for a scope served
|
|
1894
2052
|
// elsewhere, the lifecycle check and the migration. A delivery is a write into the
|
|
1895
2053
|
// consumer's scope, so it takes the gate an invoke through that door takes.
|
|
1896
|
-
await this.peerScopeGate(tenantId, scopeId, 'deliverToPeer');
|
|
1897
|
-
|
|
2054
|
+
const record = await this.peerScopeGate(tenantId, scopeId, 'deliverToPeer');
|
|
2055
|
+
// #2004: a copy consumes no other vertical's events, at the door as in the sweep
|
|
2056
|
+
// (`scope-copy.ts`). A pause: nothing runs and the watermark stays.
|
|
2057
|
+
if (await this.isInertScope(tenantId, scopeId, record)) {
|
|
2058
|
+
return { ...emptyImportResult(batch), paused: { reason: INERT_SCOPE_REASON } };
|
|
2059
|
+
}
|
|
2060
|
+
// #2029: the producer is a peer, so the delivery goes through its door: a producer the rewind
|
|
2061
|
+
// hold keeps off pauses the edge, exactly as a switched-off one does inside the DO.
|
|
2062
|
+
const door = await this.openDoor('peer', batch.source.vertical, scopeId);
|
|
2063
|
+
let applied;
|
|
2064
|
+
try {
|
|
2065
|
+
applied = await door.through((instance) => this.scopeStub(scopeId).importApply(batch, tenantId, scopeId, instance));
|
|
2066
|
+
}
|
|
2067
|
+
catch (err) {
|
|
2068
|
+
if (!this.isHeldRefusal(err))
|
|
2069
|
+
throw err;
|
|
2070
|
+
return { ...emptyImportResult(batch), paused: { reason: err.message } };
|
|
2071
|
+
}
|
|
2072
|
+
const result = importResult.parse(applied);
|
|
1898
2073
|
if (result.delivered > 0) {
|
|
1899
2074
|
try {
|
|
1900
2075
|
await this.drainExecutors(tenantId, scopeId, null);
|
|
@@ -1938,7 +2113,9 @@ export class CloudflareScopeHost {
|
|
|
1938
2113
|
async getVerticalScope(caller, tenantId, scopeId, options) {
|
|
1939
2114
|
const parsed = verticalCaller.parse(caller);
|
|
1940
2115
|
await this.peerScopeGate(tenantId, scopeId, 'getVerticalScope');
|
|
1941
|
-
|
|
2116
|
+
// #2029: gated against the rewind hold and pinned, as the system door is.
|
|
2117
|
+
const door = await this.openDoor('peer', parsed.vertical, scopeId);
|
|
2118
|
+
return this.buildStub(tenantId, scopeId, undefined, undefined, door, options, undefined, undefined, parsed);
|
|
1942
2119
|
}
|
|
1943
2120
|
/**
|
|
1944
2121
|
* Does peer `vertical` hold each key at this scope's node now (#1706) — the ScopeDO's own
|
|
@@ -1946,9 +2123,19 @@ export class CloudflareScopeHost {
|
|
|
1946
2123
|
*/
|
|
1947
2124
|
async peerCovers(tenantId, scopeId, vertical, permissions) {
|
|
1948
2125
|
await this.peerScopeGate(tenantId, scopeId, 'peerCovers');
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
|
|
2126
|
+
const slug = verticalSlug.parse(vertical);
|
|
2127
|
+
// #2029: a peer the rewind hold keeps off holds nothing here, as a switched-off one doesn't.
|
|
2128
|
+
const door = await this.openDoor('peer', slug, scopeId);
|
|
2129
|
+
try {
|
|
2130
|
+
return peerCoverage
|
|
2131
|
+
.array()
|
|
2132
|
+
.parse(await door.through((instance) => this.scopeStub(scopeId).peerCovers(tenantId, scopeId, slug, [...permissions], instance)));
|
|
2133
|
+
}
|
|
2134
|
+
catch (err) {
|
|
2135
|
+
if (this.isHeldRefusal(err))
|
|
2136
|
+
return permissions.map((permission) => ({ permission, held: false }));
|
|
2137
|
+
throw err;
|
|
2138
|
+
}
|
|
1952
2139
|
}
|
|
1953
2140
|
/**
|
|
1954
2141
|
* `ctx.canAssign`'s bound for a principal the host names (#1931) — the ScopeDO's own role read
|
|
@@ -1994,8 +2181,9 @@ export class CloudflareScopeHost {
|
|
|
1994
2181
|
// door's own tenant-confinement test proved. A RECORD crosses intact, so the coordinator
|
|
1995
2182
|
// reads the record and types the refusal itself. `validateScopeAccess` still runs below
|
|
1996
2183
|
// and still owns the lifecycle half (a suspended tenant or scope).
|
|
2184
|
+
let record;
|
|
1997
2185
|
if (!this.cpLess) {
|
|
1998
|
-
|
|
2186
|
+
record = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
1999
2187
|
if (!record) {
|
|
2000
2188
|
throw unknownScopeForTenant(tenantId, scopeId);
|
|
2001
2189
|
}
|
|
@@ -2003,6 +2191,23 @@ export class CloudflareScopeHost {
|
|
|
2003
2191
|
}
|
|
2004
2192
|
await this.assertLive(tenantId, scopeId);
|
|
2005
2193
|
await this.migrateAndRecord(scopeId);
|
|
2194
|
+
return record;
|
|
2195
|
+
}
|
|
2196
|
+
/**
|
|
2197
|
+
* Whether this scope is held inert (#2005): a fork, a snapshot or a preview, which causes no
|
|
2198
|
+
* outbound effects and consumes no other vertical's events (#2004). The one primacy question
|
|
2199
|
+
* every such door asks. Where there is a directory its record answers (`record`, the one
|
|
2200
|
+
* `peerScopeGate` returns, when the caller has already read it). A CP-less host has none, so it asks the scope's own storage
|
|
2201
|
+
* whether it is classified a copy (`_substrat_copy_origin`'s `is_copy`, #2009, set on the
|
|
2202
|
+
* directory's word): a preview and a snapshot reach a hosted vertical as restores the
|
|
2203
|
+
* platform marks.
|
|
2204
|
+
*/
|
|
2205
|
+
isInertScope(tenantId, scopeId, record) {
|
|
2206
|
+
if (this.cpLess)
|
|
2207
|
+
return this.scopeStub(scopeId).isCopy();
|
|
2208
|
+
if (record)
|
|
2209
|
+
return Promise.resolve(!isPrimaryScopeRow(record));
|
|
2210
|
+
return this.cp.getScopeRecord(tenantId, scopeId).then((row) => !isPrimaryScopeRow(row));
|
|
2006
2211
|
}
|
|
2007
2212
|
/**
|
|
2008
2213
|
* The capability's attachment door (#1686) — `getCapabilityScope`'s gate, then the
|
|
@@ -2183,7 +2388,7 @@ export class CloudflareScopeHost {
|
|
|
2183
2388
|
// #1819: a scope rewound to before its switch was pulled answers `on`, and the rewind held
|
|
2184
2389
|
// the module outside the scope. Both are read by the system door's own gate (#1834), which
|
|
2185
2390
|
// every fire below then goes through, pinned to the instance this read came from.
|
|
2186
|
-
const gate = await this.
|
|
2391
|
+
const gate = await this.doorGate('system', scopeId, moduleId);
|
|
2187
2392
|
if (gate.state === 'ungranted')
|
|
2188
2393
|
return report;
|
|
2189
2394
|
if (gate.error)
|
|
@@ -2258,10 +2463,11 @@ export class CloudflareScopeHost {
|
|
|
2258
2463
|
/** The stub body, shared by the principal, connection, system and impersonation doors. */
|
|
2259
2464
|
buildStub(tenantId, scopeId, principal, connectionId,
|
|
2260
2465
|
/**
|
|
2261
|
-
* #1834: the
|
|
2466
|
+
* #1834: the door this stub acts through. The ONLY way to make a stub acting as
|
|
2262
2467
|
* `system:<moduleId>`: the module is the door's, and every invoke is gated and pinned by it.
|
|
2468
|
+
* #2029: a peer stub's door too (`openDoor('peer', …)`), gating and pinning each invoke the same way.
|
|
2263
2469
|
*/
|
|
2264
|
-
|
|
2470
|
+
door, options,
|
|
2265
2471
|
/**
|
|
2266
2472
|
* K-42: the session this stub acts under. Re-read from the directory on
|
|
2267
2473
|
* EVERY invoke rather than captured here — a stub is a capability and
|
|
@@ -2282,7 +2488,7 @@ export class CloudflareScopeHost {
|
|
|
2282
2488
|
*/
|
|
2283
2489
|
verticalCaller) {
|
|
2284
2490
|
const stub = this.scopeStub(scopeId);
|
|
2285
|
-
const systemModuleId =
|
|
2491
|
+
const systemModuleId = door?.kind === 'system' ? door.key : undefined;
|
|
2286
2492
|
const cp = this.cp;
|
|
2287
2493
|
const operationEntitlement = this.operationEntitlement;
|
|
2288
2494
|
// The DO needs SOME principal-shaped value for `ctx.principal`; for a
|
|
@@ -2338,10 +2544,10 @@ export class CloudflareScopeHost {
|
|
|
2338
2544
|
// K-42: resolved per invoke, so expiry and `endImpersonation` both bite.
|
|
2339
2545
|
const session = sessionId === undefined ? undefined : await this.resolveImpersonation(sessionId, tenantId, scopeId);
|
|
2340
2546
|
const send = (systemDoorInstance) => stub.invoke(operation, input, asPrincipalId, tenantId, scopeId, connectionId, requiredKey, systemModuleId, true, invokeOptions, session, capabilitySession, verticalCaller, systemDoorInstance);
|
|
2341
|
-
// #1834: through the
|
|
2342
|
-
//
|
|
2343
|
-
const envelope =
|
|
2344
|
-
? await
|
|
2547
|
+
// #1834: through the door (a module's, or #2029 a peer's), pinned to the instance its gate
|
|
2548
|
+
// read. A missed pin comes back as an answer, which `through` re-gates on.
|
|
2549
|
+
const envelope = door
|
|
2550
|
+
? await door.through(async (instance) => {
|
|
2345
2551
|
const sent = await send(instance);
|
|
2346
2552
|
return sent.systemDoorMoved === true ? SYSTEM_DOOR_MOVED : sent;
|
|
2347
2553
|
})
|
|
@@ -2394,9 +2600,9 @@ export class CloudflareScopeHost {
|
|
|
2394
2600
|
throw substratError('unavailable', `${operation} ran on a scope host that did not understand peer callers. ` +
|
|
2395
2601
|
'Retry once the scope has been migrated');
|
|
2396
2602
|
}
|
|
2397
|
-
// #1834, the same shape: a DO too old to know the
|
|
2398
|
-
//
|
|
2399
|
-
if (
|
|
2603
|
+
// #1834, the same shape: a DO too old to know the door's pin ran the call without
|
|
2604
|
+
// checking it met the storage the door's gate read.
|
|
2605
|
+
if (door !== undefined && envelope.systemDoor?.honoured !== true) {
|
|
2400
2606
|
throw substratError('unavailable', `${operation} ran on a scope host that did not understand the system door's pin. ` +
|
|
2401
2607
|
'Retry once the scope has been migrated');
|
|
2402
2608
|
}
|
|
@@ -2410,11 +2616,13 @@ export class CloudflareScopeHost {
|
|
|
2410
2616
|
// this scope's outbox — and draining anyway would run executors and
|
|
2411
2617
|
// connectors as a side effect of a session that may not have side effects.
|
|
2412
2618
|
// Whatever the outbox already held is the drain sweep's own backstop.
|
|
2619
|
+
// #1184: what this call's executors did inline, for `onExecutorOutcomes`.
|
|
2620
|
+
const executorOutcomes = invokeOptions?.onExecutorOutcomes ? [] : undefined;
|
|
2413
2621
|
const drained = session?.mode === 'read-only'
|
|
2414
2622
|
? { attempted: 0, delivered: 0, retrying: 0, deadLettered: 0, routedToPlatform: 0 }
|
|
2415
2623
|
// #1525: this drain is part of the call that emitted the events — the
|
|
2416
2624
|
// coordinator's half of the post-commit tail the DO ran the consumers in.
|
|
2417
|
-
: await this.drainExecutors(tenantId, scopeId, invokeOptions?.invocationId ?? null);
|
|
2625
|
+
: await this.drainExecutors(tenantId, scopeId, invokeOptions?.invocationId ?? null, executorOutcomes);
|
|
2418
2626
|
// #458: the operation committed having enqueued platform intents — tell the
|
|
2419
2627
|
// caller's harness so it can flag the response for the router kick (#381).
|
|
2420
2628
|
// Routed connector deliveries (#574 phase 3) count too: the inline drain just
|
|
@@ -2431,6 +2639,9 @@ export class CloudflareScopeHost {
|
|
|
2431
2639
|
invokeOptions?.onIdempotentReplay?.();
|
|
2432
2640
|
if (envelope.emitted)
|
|
2433
2641
|
invokeOptions?.onEmitted?.(envelope.emitted);
|
|
2642
|
+
if (executorOutcomes && session?.mode !== 'read-only' && !envelope.idempotency?.replayed) {
|
|
2643
|
+
invokeOptions?.onExecutorOutcomes?.(executorOutcomes);
|
|
2644
|
+
}
|
|
2434
2645
|
return envelope.result;
|
|
2435
2646
|
},
|
|
2436
2647
|
};
|
|
@@ -2523,7 +2734,7 @@ export class CloudflareScopeHost {
|
|
|
2523
2734
|
*/
|
|
2524
2735
|
const requireOrg = async (tenant, id) => {
|
|
2525
2736
|
if (!(await this.cp.readOrg(tenant, id))) {
|
|
2526
|
-
throw
|
|
2737
|
+
throw substratError('not_found', `unknown org ${id} in tenant ${tenant}`);
|
|
2527
2738
|
}
|
|
2528
2739
|
};
|
|
2529
2740
|
const mapScope = (r) => scopeSchema.parse({
|
|
@@ -2610,24 +2821,20 @@ export class CloudflareScopeHost {
|
|
|
2610
2821
|
}
|
|
2611
2822
|
};
|
|
2612
2823
|
/**
|
|
2613
|
-
*
|
|
2824
|
+
* Where one scope's kill switch of one kind lands — the ONE rule the operator's switch and
|
|
2825
|
+
* the #1674 re-assert share, so a re-assert moves exactly the switch an operator's OFF would
|
|
2826
|
+
* have. `kind` is a module's schedule switch (#1666) or a peer's (#1706, #2029); `key` is the
|
|
2827
|
+
* module id or the peer's slug.
|
|
2614
2828
|
*
|
|
2615
2829
|
* Where the write lands is the whole point of the delegation branch. The shared control
|
|
2616
|
-
* plane's own `SCOPE` namespace is the module-less placeholder, so a hosted scope's
|
|
2617
|
-
*
|
|
2618
|
-
* switch pulled while every schedule kept firing
|
|
2619
|
-
*
|
|
2830
|
+
* plane's own `SCOPE` namespace is the module-less placeholder, so a hosted scope's switch
|
|
2831
|
+
* has to be moved in the deployment serving it, over that kind's delegation; writing it here
|
|
2832
|
+
* would report a switch pulled while every schedule kept firing, or every peer call kept
|
|
2833
|
+
* being admitted. A scope bound to no vertical (#1666 review) has no deployment to delegate
|
|
2834
|
+
* to: its store is the DO here, so the switch moves (or answers `held: false`) here too.
|
|
2835
|
+
* Delegating it would throw "no deployment serving scope" instead.
|
|
2620
2836
|
*/
|
|
2621
|
-
|
|
2622
|
-
* #1706: move one PEER's kill switch on one scope — see `HostAdmin.revokeFromPeer`. The
|
|
2623
|
-
* schedule switch's audit discipline (intent first, outcome after, every attempt) and,
|
|
2624
|
-
* since part 3, its delegation: a scope served by a vertical's own deployment is switched
|
|
2625
|
-
* THERE, over `PeerSwitchDelegation`. Without one configured the scope is still refused
|
|
2626
|
-
* `unavailable` (`peerScopeGate`) rather than switched in the placeholder namespace.
|
|
2627
|
-
*/
|
|
2628
|
-
const switchPeerAt = async (actor, raw, to) => {
|
|
2629
|
-
const input = peerSwitch.parse(raw);
|
|
2630
|
-
const { tenantId, scopeId } = input.node;
|
|
2837
|
+
const switchTarget = async (kind, tenantId, scopeId) => {
|
|
2631
2838
|
let vertical = null;
|
|
2632
2839
|
if (!this.cpLess) {
|
|
2633
2840
|
const rec = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
@@ -2635,136 +2842,109 @@ export class CloudflareScopeHost {
|
|
|
2635
2842
|
throw unknownScopeForTenant(tenantId, scopeId);
|
|
2636
2843
|
vertical = rec.vertical;
|
|
2637
2844
|
}
|
|
2638
|
-
const
|
|
2639
|
-
|
|
2640
|
-
|
|
2641
|
-
|
|
2642
|
-
// placeholder namespace would tombstone nothing and report a peer cut off while every
|
|
2643
|
-
// call it makes kept being admitted. A scope bound to NO vertical has its store here,
|
|
2644
|
-
// so it switches here — delegating it would throw "no deployment serving scope".
|
|
2645
|
-
// With no delegation configured the old refusal stands, loudly, rather than a write
|
|
2646
|
-
// into the wrong namespace.
|
|
2647
|
-
const delegation = this.cpLess || vertical !== null ? this.peerSwitchDelegation : undefined;
|
|
2648
|
-
if (delegation) {
|
|
2649
|
-
// The gate's lifecycle half still runs (a suspended tenant or scope cannot be
|
|
2650
|
-
// switched); its `scopeStub` half is what the delegation replaces.
|
|
2651
|
-
await this.validateScopeAccess(tenantId, scopeId);
|
|
2652
|
-
}
|
|
2653
|
-
else {
|
|
2654
|
-
await this.peerScopeGate(tenantId, scopeId, action);
|
|
2655
|
-
}
|
|
2656
|
-
const operationId = ulid();
|
|
2657
|
-
const target = { tenantId, scopeId, vertical };
|
|
2658
|
-
const base = { operationId, vertical: input.vertical, calls: to };
|
|
2659
|
-
await this.recordAdmin(actor, action, target, null, { ...base, phase: 'intent', reason: input.reason });
|
|
2660
|
-
let outcome;
|
|
2661
|
-
try {
|
|
2662
|
-
outcome = delegation
|
|
2663
|
-
? await delegation.switch({ tenantId, scopeId, vertical: input.vertical, to })
|
|
2664
|
-
: await this.scopeStub(scopeId).switchPeer(input.vertical, scopeId, to, new Date().toISOString());
|
|
2665
|
-
}
|
|
2666
|
-
catch (err) {
|
|
2667
|
-
await this.recordAdmin(actor, action, target, null, {
|
|
2668
|
-
...base,
|
|
2669
|
-
phase: 'failed',
|
|
2670
|
-
error: err instanceof Error ? err.message : String(err),
|
|
2671
|
-
}).catch(() => undefined);
|
|
2672
|
-
throw err;
|
|
2673
|
-
}
|
|
2674
|
-
await this.recordAdmin(actor, action, target, null, {
|
|
2675
|
-
...base,
|
|
2676
|
-
phase: outcome.held ? 'applied' : 'refused',
|
|
2677
|
-
changed: outcome.changed,
|
|
2678
|
-
permissions: outcome.permissions,
|
|
2679
|
-
});
|
|
2680
|
-
if (!outcome.held) {
|
|
2681
|
-
throw substratError('not_found', `scope ${scopeId} holds no grant for peer '${input.vertical}' — nothing to switch ${to} ` +
|
|
2682
|
-
`(check the slug: it is the calling vertical's registry id, as the target's \`peers\` names it)`);
|
|
2683
|
-
}
|
|
2684
|
-
return {
|
|
2685
|
-
operationId,
|
|
2686
|
-
vertical: input.vertical,
|
|
2687
|
-
calls: to,
|
|
2688
|
-
changed: outcome.changed,
|
|
2689
|
-
permissions: outcome.permissions,
|
|
2690
|
-
};
|
|
2691
|
-
};
|
|
2692
|
-
/**
|
|
2693
|
-
* Where one scope's schedule switch lands — the ONE rule `switchSystem` and the #1674
|
|
2694
|
-
* re-assert share, so a re-assert moves exactly the switch an operator's OFF would have.
|
|
2695
|
-
* A scope bound to no vertical (#1666 review) has no deployment to delegate to: its store
|
|
2696
|
-
* is the DO here, so the switch moves (or answers `held: false`) here too. Delegating it
|
|
2697
|
-
* would throw "no deployment serving scope" instead. A scope WITH a vertical still
|
|
2698
|
-
* delegates, and still fails loudly when none serves it.
|
|
2699
|
-
*/
|
|
2700
|
-
const systemSwitchTarget = async (tenantId, scopeId) => {
|
|
2701
|
-
let vertical = null;
|
|
2702
|
-
if (!this.cpLess) {
|
|
2703
|
-
const rec = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
2704
|
-
if (!rec)
|
|
2705
|
-
throw unknownScopeForTenant(tenantId, scopeId);
|
|
2706
|
-
vertical = rec.vertical;
|
|
2707
|
-
}
|
|
2708
|
-
const delegation = this.cpLess || vertical !== null ? this.systemSwitchDelegation : undefined;
|
|
2845
|
+
const hosted = this.cpLess || vertical !== null;
|
|
2846
|
+
const systemDelegation = kind === 'system' && hosted ? this.systemSwitchDelegation : undefined;
|
|
2847
|
+
const peerDelegation = kind === 'peer' && hosted ? this.peerSwitchDelegation : undefined;
|
|
2848
|
+
const delegated = systemDelegation !== undefined || peerDelegation !== undefined;
|
|
2709
2849
|
// #1819: a scope whose store is here was rewound here, so `switchInScope` releases its
|
|
2710
2850
|
// hold here too. A delegated move releases in the deployment serving the scope.
|
|
2711
|
-
// #1823: a
|
|
2712
|
-
// the scope's storage for the switch to find, so the directory says
|
|
2713
|
-
// A caller moving several
|
|
2714
|
-
// An OFF of a tenant-held
|
|
2851
|
+
// #1823 (#2030 for a peer): a subject whose only authority on the scope is a TENANT-level
|
|
2852
|
+
// grant has nothing in the scope's storage for the switch to find, so the directory says
|
|
2853
|
+
// whether it is held. A caller moving several reads them in one call and passes each in.
|
|
2854
|
+
// An OFF of a tenant-held MODULE is refused, after the far end answers, unless that answer
|
|
2715
2855
|
// attests the evaluator that denies a tenant-level grant (`deniesTenantGrants`). A
|
|
2716
2856
|
// deployment built before #1823 drops `tenantHeld`, still answers `held: true` (the module's
|
|
2717
2857
|
// scope-level grants were there to switch), and keeps authorizing the tenant-level grant —
|
|
2718
|
-
// so its `held` alone would have the platform record a scope off that is not.
|
|
2719
|
-
//
|
|
2720
|
-
//
|
|
2721
|
-
|
|
2722
|
-
|
|
2723
|
-
|
|
2724
|
-
|
|
2725
|
-
|
|
2726
|
-
|
|
2727
|
-
|
|
2858
|
+
// so its `held` alone would have the platform record a scope off that is not. A peer needs
|
|
2859
|
+
// no attestation: its door refuses on the marker in every build that has the peer switch at
|
|
2860
|
+
// all (`admitPeer` and `switchPeer` arrived together, #1714). Every OFF the platform drives
|
|
2861
|
+
// moves through here: an operator's (`switchSubjectAt`) and a re-assert's, which every carry
|
|
2862
|
+
// ends with.
|
|
2863
|
+
//
|
|
2864
|
+
// #2045 (Codex r3): whether a delegated deployment honours the switch fence is settled
|
|
2865
|
+
// BEFORE anything moves (`attestFence`, once per target), never after the fact by putting a
|
|
2866
|
+
// move back. Without the fence an ON is refused and an OFF goes through unfenced (see
|
|
2867
|
+
// `UnattestedSwitch`). A move a fenced deployment answers without the attestation still
|
|
2868
|
+
// throws, as defence in depth against a rollback between the preflight and the move.
|
|
2869
|
+
const delegation = systemDelegation ?? peerDelegation;
|
|
2870
|
+
let attested;
|
|
2871
|
+
const attestFence = () => (attested ??= delegation ? delegation.fenceSupported({ tenantId, scopeId }) : Promise.resolve(true));
|
|
2872
|
+
const move = async (key, to, at, tenantHeld,
|
|
2873
|
+
/** #2045: the switch call's fence — its operation id, or for a re-assert the record's. */
|
|
2874
|
+
fence) => {
|
|
2875
|
+
const fenced = await attestFence();
|
|
2876
|
+
if (!fenced && to === 'on')
|
|
2877
|
+
throw new UnattestedSwitch(unfencedMessage(scopeId));
|
|
2878
|
+
const held = tenantHeld ?? (await this.cp.tenantHeldOf(kind, tenantId, [key], at)).length > 0;
|
|
2879
|
+
const outcome = await (systemDelegation
|
|
2880
|
+
? systemDelegation.switch({ tenantId, scopeId, moduleId: key, to, tenantHeld: held, fence })
|
|
2881
|
+
: peerDelegation
|
|
2882
|
+
? peerDelegation.switch({ tenantId, scopeId, vertical: key, to, tenantHeld: held, fence })
|
|
2883
|
+
: this.switchInScope(kind, scopeId, key, to, at, held, fence));
|
|
2884
|
+
// An unfenced OFF answers without `fenced`, and that is what the callers read it by.
|
|
2885
|
+
if (fenced && outcome.fenced !== true)
|
|
2886
|
+
throw new UnattestedSwitch(unfencedMessage(scopeId));
|
|
2887
|
+
if (outcome.superseded)
|
|
2888
|
+
return outcome;
|
|
2889
|
+
if (kind === 'system' && to === 'off' && held && outcome.held && outcome.deniesTenantGrants !== true) {
|
|
2890
|
+
throw new UnattestedSwitch(undeniedMessage(scopeId, key));
|
|
2728
2891
|
}
|
|
2729
2892
|
return outcome;
|
|
2730
2893
|
};
|
|
2731
|
-
|
|
2732
|
-
* Is the module switched off on the scope right now? Read from wherever `move` writes — the
|
|
2733
|
-
* deployment's status read or this scope's own — for a move that threw (#1823): the switch
|
|
2734
|
-
* may already have moved before the throw (`switchInScope` moves it, then works the rewind
|
|
2735
|
-
* hold), and then the directory's record must stay.
|
|
2736
|
-
*/
|
|
2737
|
-
const isOff = async (moduleId) => {
|
|
2738
|
-
const rows = delegation
|
|
2739
|
-
? await delegation.status({ tenantId, scopeId })
|
|
2740
|
-
: systemScheduleEntry.array().parse(await this.scopeStub(scopeId).systemGrantsStatus());
|
|
2741
|
-
return rows.some((r) => r.moduleId === moduleId && r.schedules === 'off');
|
|
2742
|
-
};
|
|
2743
|
-
return { vertical, move, isOff };
|
|
2894
|
+
return { vertical, delegated, move, attestFence };
|
|
2744
2895
|
};
|
|
2745
|
-
|
|
2746
|
-
|
|
2747
|
-
|
|
2748
|
-
|
|
2896
|
+
/**
|
|
2897
|
+
* One kill switch moved on one scope — the ONE body `revokeFromSystem` / `restoreToSystem`
|
|
2898
|
+
* (#1666) and `revokeFromPeer` / `restoreToPeer` (#1706) share, so the two levers cannot
|
|
2899
|
+
* disagree about the record, the audit, the readback or the undo (#2029). The audit row is
|
|
2900
|
+
* written HERE either way — a deployment's host is CP-less, and its `recordAdmin` is a no-op.
|
|
2901
|
+
*/
|
|
2902
|
+
const switchSubjectAt = async (actor, kind, node, key, reason, to) => {
|
|
2903
|
+
const { tenantId, scopeId } = node;
|
|
2904
|
+
const { vertical, delegated, move, attestFence } = await switchTarget(kind, tenantId, scopeId);
|
|
2905
|
+
const action = switchActionOf(kind, to);
|
|
2906
|
+
if (kind === 'peer') {
|
|
2907
|
+
// The peer door's gate (#1706): with a delegation, its lifecycle half (a suspended tenant
|
|
2908
|
+
// or scope cannot be switched), the `scopeStub` half being what the delegation replaces.
|
|
2909
|
+
// Without one, a hosted scope is refused `unavailable` rather than switched in the
|
|
2910
|
+
// placeholder namespace, where it would tombstone nothing a peer call reads.
|
|
2911
|
+
if (delegated)
|
|
2912
|
+
await this.validateScopeAccess(tenantId, scopeId);
|
|
2913
|
+
else
|
|
2914
|
+
await this.peerScopeGate(tenantId, scopeId, action);
|
|
2915
|
+
}
|
|
2749
2916
|
// AUDIT FIRST (#1666 review): the intent row lands before anything moves, and the
|
|
2750
2917
|
// outcome row after — every attempt, a repeat included. The scope's store and the
|
|
2751
2918
|
// admin log are separate, so no order makes the pair atomic; this one fails toward
|
|
2752
2919
|
// "an intent with no recorded outcome" and never toward "a switch that moved with no
|
|
2753
2920
|
// audit row". A retry after a crash re-audits even though it answers `changed: false`.
|
|
2754
2921
|
const operationId = ulid();
|
|
2755
|
-
const action = to === 'off' ? 'revokeFromSystem' : 'restoreToSystem';
|
|
2756
2922
|
const target = { tenantId, scopeId, vertical };
|
|
2757
|
-
const base = { operationId,
|
|
2758
|
-
await this.recordAdmin(actor, action, target, null, { ...base, phase: 'intent', reason
|
|
2759
|
-
// The directory's record (#1674), written BEFORE the scope moves, both ways, and undone
|
|
2760
|
-
// if the move throws or holds nothing — see `recordSystemSwitchedOn` and (#1823)
|
|
2761
|
-
// `recordSystemSwitchedOff` for why that order is the safe one.
|
|
2762
|
-
const at = new Date().toISOString();
|
|
2763
|
-
const record = { tenantId, scopeId, moduleId: input.moduleId, actor, reason: input.reason, operationId, at };
|
|
2923
|
+
const base = { operationId, ...switchAuditSubject(kind, key, to) };
|
|
2924
|
+
await this.recordAdmin(actor, action, target, null, { ...base, phase: 'intent', reason });
|
|
2764
2925
|
const errorOf = (err) => (err instanceof Error ? err.message : String(err));
|
|
2926
|
+
// #2045 (Codex r3): the deployment's fence is attested BEFORE anything is recorded or moved. An
|
|
2927
|
+
// ON on a deployment built before it is refused here, with nothing written anywhere but the
|
|
2928
|
+
// audit. An OFF goes on: the kill switch must work on every deployment (see `UnattestedSwitch`).
|
|
2929
|
+
try {
|
|
2930
|
+
if (!(await attestFence()) && to === 'on') {
|
|
2931
|
+
throw substratError('precondition_failed', `${unfencedMessage(scopeId)} Nothing was switched.`);
|
|
2932
|
+
}
|
|
2933
|
+
}
|
|
2934
|
+
catch (err) {
|
|
2935
|
+
await this.recordAdmin(actor, action, target, null, { ...base, phase: 'refused', error: errorOf(err) }).catch(() => undefined);
|
|
2936
|
+
throw err;
|
|
2937
|
+
}
|
|
2938
|
+
// The directory's record (#1674; #2029 for a peer), written BEFORE the scope moves, both
|
|
2939
|
+
// ways — see `recordSwitchedOn` and (#1823) `recordSwitchedOff` for why that order is the
|
|
2940
|
+
// safe one. #2045 (Codex r3): the same write marks the subject owed and clears the scope's
|
|
2941
|
+
// reconcile receipt, so from here on the scope converges to this record whatever happens to
|
|
2942
|
+
// the move. The mark goes only when the scope confirms the move under this call's fence.
|
|
2943
|
+
const at = new Date().toISOString();
|
|
2944
|
+
const record = { kind, tenantId, scopeId, key, actor, reason, operationId, at };
|
|
2765
2945
|
let prior;
|
|
2766
2946
|
try {
|
|
2767
|
-
prior = to === 'on' ? await this.cp.
|
|
2947
|
+
prior = to === 'on' ? await this.cp.recordSwitchedOn(record) : await this.cp.recordSwitchedOff(record);
|
|
2768
2948
|
}
|
|
2769
2949
|
catch (err) {
|
|
2770
2950
|
// Nothing has moved: fail the call here, audited. An ON must not switch a scope on
|
|
@@ -2773,92 +2953,92 @@ export class CloudflareScopeHost {
|
|
|
2773
2953
|
await this.recordAdmin(actor, action, target, null, { ...base, phase: 'failed', error: errorOf(err) }).catch(() => undefined);
|
|
2774
2954
|
throw err;
|
|
2775
2955
|
}
|
|
2776
|
-
|
|
2777
|
-
|
|
2778
|
-
|
|
2779
|
-
|
|
2780
|
-
|
|
2781
|
-
|
|
2782
|
-
|
|
2956
|
+
// #2045: a newer call on this subject has recorded its position already: this one writes
|
|
2957
|
+
// nothing, here or in the scope, and says so.
|
|
2958
|
+
if (recordWriteSuperseded(prior, record)) {
|
|
2959
|
+
await this.recordAdmin(actor, action, target, null, { ...base, phase: 'refused', superseded: true });
|
|
2960
|
+
throw substratError('conflict', switchSupersededMessage(kind, scopeId, key, to));
|
|
2961
|
+
}
|
|
2962
|
+
let outcome;
|
|
2963
|
+
try {
|
|
2783
2964
|
try {
|
|
2784
|
-
await
|
|
2785
|
-
return null;
|
|
2965
|
+
outcome = await move(key, to, at, undefined, operationId);
|
|
2786
2966
|
}
|
|
2787
|
-
catch {
|
|
2788
|
-
|
|
2789
|
-
|
|
2790
|
-
|
|
2791
|
-
|
|
2792
|
-
|
|
2793
|
-
return errorOf(err);
|
|
2794
|
-
}
|
|
2967
|
+
catch (first) {
|
|
2968
|
+
if (first instanceof UnattestedSwitch)
|
|
2969
|
+
throw first;
|
|
2970
|
+
// #2045 (Codex r2): one retry under this call's own fence — idempotent, and refused by
|
|
2971
|
+
// the scope if a newer call has moved it meanwhile.
|
|
2972
|
+
outcome = await move(key, to, new Date().toISOString(), undefined, operationId);
|
|
2795
2973
|
}
|
|
2796
|
-
};
|
|
2797
|
-
let outcome;
|
|
2798
|
-
try {
|
|
2799
|
-
outcome = await move(input.moduleId, to, at);
|
|
2800
2974
|
}
|
|
2801
2975
|
catch (err) {
|
|
2802
|
-
|
|
2803
|
-
|
|
2804
|
-
|
|
2805
|
-
|
|
2806
|
-
|
|
2807
|
-
const putBack = err.outcome.changed
|
|
2808
|
-
? await move(input.moduleId, 'on', new Date().toISOString(), true).then(() => true, () => false)
|
|
2809
|
-
: true;
|
|
2810
|
-
if (putBack) {
|
|
2811
|
-
const recordError = await undoRecord();
|
|
2812
|
-
await this.recordAdmin(actor, action, target, null, {
|
|
2813
|
-
...base,
|
|
2814
|
-
phase: 'refused',
|
|
2815
|
-
error: err.message,
|
|
2816
|
-
...(recordError ? { recordError } : {}),
|
|
2817
|
-
}).catch(() => undefined);
|
|
2818
|
-
throw substratError('precondition_failed', `${err.message} Nothing was switched` +
|
|
2819
|
-
(recordError ? `, but its directory record could not be put back (${recordError}).` : '.'));
|
|
2820
|
-
}
|
|
2821
|
-
}
|
|
2822
|
-
// A throw does not say the scope did not move: the move crosses the scope's DO and then
|
|
2823
|
-
// the rewind hold, and a throw from the second leaves the scope's switch moved. So the
|
|
2824
|
-
// record is undone only when the scope reads back in the OTHER position — an OFF that
|
|
2825
|
-
// reads not off, an ON that still reads off — and kept when it reads back as asked.
|
|
2826
|
-
// Undoing an ON that landed would put the record back to `off` beside a scope that is
|
|
2827
|
-
// on, and the next re-assert would switch the operator's ON back off. A read that fails
|
|
2828
|
-
// too counts as off, both ways, so what is left fails toward off: an OFF keeps its
|
|
2829
|
-
// record (never a scope switched off with no record — a wipe or restore would have
|
|
2830
|
-
// nothing to re-assert, and #1743's refusal nothing to read), and an ON is undone back
|
|
2831
|
-
// to `off`, which the next re-assert completes if the scope did move.
|
|
2832
|
-
const off = await isOff(input.moduleId).catch(() => true);
|
|
2833
|
-
const recordKept = off === (to === 'off');
|
|
2834
|
-
const recordError = recordKept ? null : await undoRecord();
|
|
2835
|
-
// Best effort: the original error is what the caller must see, and the intent row
|
|
2836
|
-
// already says an attempt was made.
|
|
2976
|
+
// The move threw twice, or was answered by a deployment that could not attest it. Either
|
|
2977
|
+
// way it may have landed, and an older call's move may still be in flight behind it, so no
|
|
2978
|
+
// readback can settle the scope and nothing is put back: the record is this call's position
|
|
2979
|
+
// under the newest fence, and its write-ahead mark makes the next re-assert move the scope
|
|
2980
|
+
// to it, under that fence, whichever way it lies now.
|
|
2837
2981
|
await this.recordAdmin(actor, action, target, null, {
|
|
2838
2982
|
...base,
|
|
2839
|
-
phase: 'failed',
|
|
2983
|
+
phase: err instanceof UnattestedSwitch ? 'refused' : 'failed',
|
|
2840
2984
|
error: errorOf(err),
|
|
2841
|
-
|
|
2842
|
-
|
|
2985
|
+
recordKept: true,
|
|
2986
|
+
reassertOwed: true,
|
|
2843
2987
|
}).catch(() => undefined);
|
|
2988
|
+
if (err instanceof UnattestedSwitch)
|
|
2989
|
+
throw substratError('precondition_failed', err.message);
|
|
2844
2990
|
throw err;
|
|
2845
2991
|
}
|
|
2992
|
+
// #2045: the scope has applied a newer call on this subject, so this move wrote nothing. The
|
|
2993
|
+
// record is that newer call's too (its write overwrote this one's, mark included), so nothing
|
|
2994
|
+
// is undone and no mark is this call's to clear.
|
|
2995
|
+
if (outcome.superseded) {
|
|
2996
|
+
await this.recordAdmin(actor, action, target, null, { ...base, phase: 'refused', superseded: true });
|
|
2997
|
+
throw substratError('conflict', switchSupersededMessage(kind, scopeId, key, to));
|
|
2998
|
+
}
|
|
2846
2999
|
// A call that held nothing moved nothing, so its record write is undone: left `off`, it
|
|
2847
|
-
// would switch the
|
|
2848
|
-
// a wiped scope would leave
|
|
2849
|
-
|
|
3000
|
+
// would switch the subject off the day it is installed; left `on`, the next reconcile of
|
|
3001
|
+
// a wiped scope would leave it running. Retried once; a failure of both lands on the
|
|
3002
|
+
// outcome row as `recordError`, and the call says so.
|
|
3003
|
+
//
|
|
3004
|
+
// The scope confirmed this call's move (or that there was nothing to move), so its mark goes.
|
|
3005
|
+
// Held nothing, it goes WITH the undo, in one directory transaction: the row put back carries
|
|
3006
|
+
// the prior call's older id, and no later re-assert confirms under a fence older than this
|
|
3007
|
+
// call's mark. An undo that fails twice leaves the record AND the mark this call's, and the
|
|
3008
|
+
// next re-assert confirms both under it. Held, a failed clear is not the call's: the mark only
|
|
3009
|
+
// costs the next re-assert one idempotent move under this same record.
|
|
3010
|
+
//
|
|
3011
|
+
// An unfenced OFF that held something (a deployment built before the fence) confirmed nothing
|
|
3012
|
+
// under its fence, so its mark stays for the first re-assert after the vertical is redeployed.
|
|
3013
|
+
const unfenced = outcome.held && outcome.fenced !== true;
|
|
3014
|
+
let recordError = null;
|
|
3015
|
+
let owedError = null;
|
|
3016
|
+
if (!outcome.held) {
|
|
3017
|
+
const undo = () => this.cp.restoreSwitchRecord(record, prior, true);
|
|
3018
|
+
recordError = await undo().then(() => null, () => undo().then(() => null, errorOf));
|
|
3019
|
+
}
|
|
3020
|
+
else if (!unfenced) {
|
|
3021
|
+
owedError = await this.cp.clearSwitchOwed(kind, tenantId, scopeId, key, operationId).then(() => null, errorOf);
|
|
3022
|
+
}
|
|
2850
3023
|
await this.recordAdmin(actor, action, target, null, {
|
|
2851
3024
|
...base,
|
|
2852
3025
|
phase: outcome.held ? 'applied' : 'refused',
|
|
2853
3026
|
changed: outcome.changed,
|
|
2854
3027
|
permissions: outcome.permissions,
|
|
3028
|
+
...(unfenced ? { unfenced: true, reassertOwed: true } : {}),
|
|
2855
3029
|
...(recordError ? { recordError } : {}),
|
|
3030
|
+
...(owedError ? { owedError } : {}),
|
|
2856
3031
|
});
|
|
2857
3032
|
if (!outcome.held) {
|
|
2858
|
-
throw substratError('not_found',
|
|
2859
|
-
`(check the module id: it is the module's manifest id, e.g. '@substrat-run/engine-absence')` +
|
|
3033
|
+
throw substratError('not_found', switchNotFoundMessage(kind, scopeId, key, to) +
|
|
2860
3034
|
(recordError ? `; and its directory record could not be put back (${recordError})` : ''));
|
|
2861
3035
|
}
|
|
3036
|
+
return { operationId, outcome };
|
|
3037
|
+
};
|
|
3038
|
+
/** #1666: move one module's schedule switch on one scope — see `HostAdmin.revokeFromSystem`. */
|
|
3039
|
+
const switchSystem = async (actor, raw, to) => {
|
|
3040
|
+
const input = systemSwitch.parse(raw);
|
|
3041
|
+
const { operationId, outcome } = await switchSubjectAt(actor, 'system', input.node, input.moduleId, input.reason, to);
|
|
2862
3042
|
return {
|
|
2863
3043
|
operationId,
|
|
2864
3044
|
moduleId: input.moduleId,
|
|
@@ -2867,6 +3047,18 @@ export class CloudflareScopeHost {
|
|
|
2867
3047
|
permissions: outcome.permissions,
|
|
2868
3048
|
};
|
|
2869
3049
|
};
|
|
3050
|
+
/** #1706: move one PEER's kill switch on one scope — see `HostAdmin.revokeFromPeer`. */
|
|
3051
|
+
const switchPeerAt = async (actor, raw, to) => {
|
|
3052
|
+
const input = peerSwitch.parse(raw);
|
|
3053
|
+
const { operationId, outcome } = await switchSubjectAt(actor, 'peer', input.node, input.vertical, input.reason, to);
|
|
3054
|
+
return {
|
|
3055
|
+
operationId,
|
|
3056
|
+
vertical: input.vertical,
|
|
3057
|
+
calls: to,
|
|
3058
|
+
changed: outcome.changed,
|
|
3059
|
+
permissions: outcome.permissions,
|
|
3060
|
+
};
|
|
3061
|
+
};
|
|
2870
3062
|
/**
|
|
2871
3063
|
* The status read's admin-log join (#1674): the `intent` row of the `revokeFromSystem`
|
|
2872
3064
|
* still in force for each OFF module — actor, reason, when. Paired to its `applied` row
|
|
@@ -2939,7 +3131,7 @@ export class CloudflareScopeHost {
|
|
|
2939
3131
|
delegation
|
|
2940
3132
|
? delegation.status({ tenantId, scopeId })
|
|
2941
3133
|
: this.scopeStub(scopeId).systemGrantsStatus().then((rows) => systemScheduleEntry.array().parse(rows)),
|
|
2942
|
-
this.cp.
|
|
3134
|
+
this.cp.switchRecordsOf('system', tenantId, scopeId),
|
|
2943
3135
|
]);
|
|
2944
3136
|
const offModules = new Set(states.filter((s) => s.schedules === 'off').map((s) => s.moduleId));
|
|
2945
3137
|
const explanations = offModules.size > 0
|
|
@@ -2958,71 +3150,137 @@ export class CloudflareScopeHost {
|
|
|
2958
3150
|
};
|
|
2959
3151
|
/**
|
|
2960
3152
|
* #1674: put the directory's OFF positions back into one scope — see
|
|
2961
|
-
* `HostAdmin.reassertSystemSwitches`.
|
|
2962
|
-
* so it lands wherever an operator's
|
|
2963
|
-
*
|
|
3153
|
+
* `HostAdmin.reassertSystemSwitches`. Every kind (#2029): the recorded-off modules, then the
|
|
3154
|
+
* recorded-off peers. Each moves through `switchTarget`, so it lands wherever an operator's
|
|
3155
|
+
* OFF would have — for a hosted scope, over that kind's `/internal/*` switch route.
|
|
2964
3156
|
*/
|
|
2965
3157
|
const reassertSystemSwitchesOf = async (actor, node, opts) => {
|
|
3158
|
+
const results = [];
|
|
3159
|
+
for (const kind of SWITCH_KINDS)
|
|
3160
|
+
results.push(...(await reassertKind(actor, kind, node, opts)));
|
|
3161
|
+
return results;
|
|
3162
|
+
};
|
|
3163
|
+
/** One kind's half of `reassertSystemSwitchesOf`. */
|
|
3164
|
+
const reassertKind = async (actor, kind, node, opts) => {
|
|
2966
3165
|
const { tenantId, scopeId } = node;
|
|
2967
|
-
const { vertical, move } = await
|
|
2968
|
-
const
|
|
3166
|
+
const { vertical, delegated, move, attestFence } = await switchTarget(kind, tenantId, scopeId);
|
|
3167
|
+
const action = reassertActionOf(kind);
|
|
3168
|
+
const target = { tenantId, scopeId, vertical };
|
|
3169
|
+
const recorded = new Map(await this.cp.switchRecordsOf(kind, tenantId, scopeId));
|
|
3170
|
+
// #2045: every switch call's write-ahead mark, with the newest call that wrote it.
|
|
3171
|
+
const owed = new Map(await this.cp.switchesOwedOf(kind, tenantId, scopeId));
|
|
2969
3172
|
// #1742 review: moves the deployment made from a stale list (restored ON after the list
|
|
2970
3173
|
// was read), to undo before the OFF pass below.
|
|
2971
|
-
const reverts = staleCarryReverts(recorded, opts?.appliedInUnit);
|
|
3174
|
+
const reverts = staleCarryReverts(kind, recorded, opts?.appliedInUnit);
|
|
2972
3175
|
// A hosted scope with no delegation configured (Copilot review): this host's own
|
|
2973
3176
|
// namespace is the module-less placeholder, where the switch would answer `held: false`
|
|
2974
3177
|
// quietly — a re-assert reported done, and a receipt written, while the deployment
|
|
2975
|
-
// serving the scope keeps its schedules running
|
|
2976
|
-
// read is, and only when a re-assert is owed: nothing recorded,
|
|
2977
|
-
|
|
2978
|
-
|
|
3178
|
+
// serving the scope keeps its schedules running or its peer admitted. Refused loudly
|
|
3179
|
+
// instead, as the status read is, and only when a re-assert is owed: nothing recorded,
|
|
3180
|
+
// nothing to refuse.
|
|
3181
|
+
const reassertOwed = reverts.length > 0 || owed.size > 0 || [...recorded.values()].includes('off');
|
|
3182
|
+
if (reassertOwed && !this.cpLess && vertical !== null && !delegated) {
|
|
2979
3183
|
throw substratError('unavailable', `no delegation configured for hosted scope ${scopeId} (vertical '${vertical}') — cannot re-assert ` +
|
|
2980
|
-
`its switched-off
|
|
3184
|
+
`its switched-off ${DOOR_WORDS[kind].switched} in the deployment serving it`);
|
|
2981
3185
|
}
|
|
3186
|
+
// #2045 (Codex r3): a subject's mark goes only once the scope has confirmed a move under the
|
|
3187
|
+
// record's fence — a fenced answer that was not superseded. A throw anywhere below leaves
|
|
3188
|
+
// every mark not yet confirmed for the next pass.
|
|
3189
|
+
// An unfenced OFF (a deployment built before the fence) confirms nothing, so its mark stays.
|
|
3190
|
+
const confirm = async (key, fence, outcome) => {
|
|
3191
|
+
if (owed.has(key) && !outcome.superseded && outcome.fenced === true) {
|
|
3192
|
+
await this.cp.clearSwitchOwed(kind, tenantId, scopeId, key, fence);
|
|
3193
|
+
}
|
|
3194
|
+
};
|
|
3195
|
+
// #2045 follow-up: on a deployment built before the fence no ON is sent (see
|
|
3196
|
+
// `UnattestedSwitch`). Every OFF below still moves, and the pass refuses at its end, so a
|
|
3197
|
+
// sweep records no receipt for a scope whose owed ON was not sent. Asked only when an ON is due.
|
|
3198
|
+
const onWithheld = [];
|
|
3199
|
+
const withholdOn = async (key) => {
|
|
3200
|
+
if (await attestFence())
|
|
3201
|
+
return false;
|
|
3202
|
+
onWithheld.push(key);
|
|
3203
|
+
return true;
|
|
3204
|
+
};
|
|
2982
3205
|
const at = new Date().toISOString();
|
|
2983
3206
|
const reverted = new Set();
|
|
2984
|
-
for (const
|
|
3207
|
+
for (const key of reverts) {
|
|
2985
3208
|
// Re-read immediately before the move: a staff OFF that completed since the read above
|
|
2986
|
-
// (record `off`) must not get a transient ON a due schedule could
|
|
2987
|
-
|
|
2988
|
-
|
|
3209
|
+
// (record `off`) must not get a transient ON a due schedule, or a peer call, could use.
|
|
3210
|
+
// #2045: under the fence of the ON the record holds, which is the operator's own call — read
|
|
3211
|
+
// with the position, so it is that ON's fence and no other call's.
|
|
3212
|
+
const current = new Map(await this.cp.switchRecordStatesOf(kind, tenantId, scopeId)).get(key);
|
|
3213
|
+
if (current?.position !== 'on')
|
|
2989
3214
|
continue;
|
|
2990
|
-
const
|
|
2991
|
-
|
|
3215
|
+
const fence = current.fence;
|
|
3216
|
+
if (await withholdOn(key))
|
|
3217
|
+
continue;
|
|
3218
|
+
const outcome = await move(key, 'on', at, undefined, fence);
|
|
3219
|
+
reverted.add(key);
|
|
2992
3220
|
if (outcome.changed) {
|
|
2993
|
-
await this.recordAdmin(actor,
|
|
2994
|
-
operationId: ulid(),
|
|
2995
|
-
...staleCarryRevertRow(moduleId, outcome),
|
|
2996
|
-
});
|
|
3221
|
+
await this.recordAdmin(actor, action, target, null, { operationId: ulid(), ...staleCarryRevertRow(kind, key, outcome) });
|
|
2997
3222
|
}
|
|
3223
|
+
await confirm(key, fence, outcome);
|
|
2998
3224
|
}
|
|
2999
3225
|
// Read AFTER the reverts: an OFF that landed meanwhile is switched off below.
|
|
3000
|
-
const
|
|
3226
|
+
const { keys, tenantHeld, fences } = await this.recordedOff(kind, tenantId, scopeId, at);
|
|
3001
3227
|
// #1742: what the deployment already switched off inside its own unit, audited here —
|
|
3002
|
-
// the move below answers `changed: false` for it and would write no row.
|
|
3003
|
-
//
|
|
3004
|
-
const
|
|
3005
|
-
|
|
3006
|
-
await this.recordAdmin(actor, 'reassertSystemSwitch', { tenantId, scopeId, vertical }, null, {
|
|
3007
|
-
operationId: ulid(),
|
|
3008
|
-
...row,
|
|
3009
|
-
});
|
|
3228
|
+
// the move below answers `changed: false` for it and would write no row. A move the
|
|
3229
|
+
// revert above undid is not credited as an in-unit OFF.
|
|
3230
|
+
for (const row of inUnitMovesToAudit(kind, keys, opts?.appliedInUnit, reverted)) {
|
|
3231
|
+
await this.recordAdmin(actor, action, target, null, { operationId: ulid(), ...row });
|
|
3010
3232
|
}
|
|
3233
|
+
// #2045: every move here carries the fence of the call the record holds, so a scope a newer call
|
|
3234
|
+
// has moved since refuses it. A deployment from before the fence takes the OFF unfenced.
|
|
3011
3235
|
const results = [];
|
|
3012
|
-
const
|
|
3013
|
-
for (const
|
|
3014
|
-
const
|
|
3236
|
+
const held = new Set(tenantHeld);
|
|
3237
|
+
for (const key of keys) {
|
|
3238
|
+
const fence = fences.get(key); // `recordedOff` read each key with its own row's fence
|
|
3239
|
+
const outcome = await move(key, 'off', at, held.has(key), fence);
|
|
3015
3240
|
if (outcome.changed) {
|
|
3016
|
-
await this.recordAdmin(actor,
|
|
3241
|
+
await this.recordAdmin(actor, action, target, null, {
|
|
3017
3242
|
operationId: ulid(),
|
|
3018
|
-
|
|
3019
|
-
|
|
3020
|
-
phase: 'applied',
|
|
3021
|
-
changed: true,
|
|
3022
|
-
permissions: outcome.permissions,
|
|
3243
|
+
...reassertOffRow(kind, key, outcome.permissions),
|
|
3244
|
+
...(outcome.fenced === true ? {} : { unfenced: true }),
|
|
3023
3245
|
});
|
|
3024
3246
|
}
|
|
3025
|
-
results.push(
|
|
3247
|
+
results.push(reassertEntry(kind, key, outcome));
|
|
3248
|
+
await confirm(key, fence, outcome);
|
|
3249
|
+
}
|
|
3250
|
+
// A subject marked owed converges to its record in either direction: OFF above, and here a
|
|
3251
|
+
// record of ON — sent whatever the scope reads now, because a scope that reads ON may still
|
|
3252
|
+
// be behind the record's fence, with an older call's OFF in flight that only the fence can
|
|
3253
|
+
// refuse (Codex r3, finding 1). Idempotent: an ON already in place changes nothing and moves
|
|
3254
|
+
// the fence. Every other record of ON keeps #1674's rule and turns nothing on.
|
|
3255
|
+
const settled = new Set([...keys, ...reverted]);
|
|
3256
|
+
const pending = [...owed.keys()].filter((key) => !settled.has(key));
|
|
3257
|
+
if (pending.length) {
|
|
3258
|
+
// Position and fence in one read: an ON is sent only under the fence of the ON it read.
|
|
3259
|
+
const now = new Map(await this.cp.switchRecordStatesOf(kind, tenantId, scopeId));
|
|
3260
|
+
for (const key of pending) {
|
|
3261
|
+
const row = now.get(key);
|
|
3262
|
+
if (row?.position !== 'on') {
|
|
3263
|
+
// No record (a call that held nothing, whose undo removed its row), or one that turned
|
|
3264
|
+
// OFF since the read above, which the next pass moves: nothing to send on this one. A
|
|
3265
|
+
// missing row's mark goes under its own id, so a newer call's mark stays.
|
|
3266
|
+
if (row === undefined)
|
|
3267
|
+
await this.cp.clearSwitchOwed(kind, tenantId, scopeId, key, owed.get(key));
|
|
3268
|
+
continue;
|
|
3269
|
+
}
|
|
3270
|
+
const fence = row.fence;
|
|
3271
|
+
if (await withholdOn(key))
|
|
3272
|
+
continue;
|
|
3273
|
+
const outcome = await move(key, 'on', at, undefined, fence);
|
|
3274
|
+
if (outcome.changed) {
|
|
3275
|
+
await this.recordAdmin(actor, action, target, null, { operationId: ulid(), ...reassertOnRow(kind, key, outcome.permissions) });
|
|
3276
|
+
results.push(reassertEntry(kind, key, outcome));
|
|
3277
|
+
}
|
|
3278
|
+
await confirm(key, fence, outcome);
|
|
3279
|
+
}
|
|
3280
|
+
}
|
|
3281
|
+
if (onWithheld.length) {
|
|
3282
|
+
throw substratError('precondition_failed', `${unfencedMessage(scopeId)} Its recorded OFF positions were applied; the owed ON of ` +
|
|
3283
|
+
`${onWithheld.map((k) => `'${k}'`).join(', ')} was not sent.`);
|
|
3026
3284
|
}
|
|
3027
3285
|
return results;
|
|
3028
3286
|
};
|
|
@@ -3109,18 +3367,21 @@ export class CloudflareScopeHost {
|
|
|
3109
3367
|
unassignRole: async (actor, assignment) => {
|
|
3110
3368
|
// Tombstone (K-21) — the checker skips revoked rows. A no-op returns false so
|
|
3111
3369
|
// a repeat unassign stays silent (no second audit row, no needless fan-out).
|
|
3112
|
-
const subject = `principal:${assignment.principalId}`;
|
|
3113
3370
|
const relation = `role:${assignment.roleKey}`;
|
|
3114
|
-
const
|
|
3115
|
-
|
|
3116
|
-
|
|
3117
|
-
|
|
3371
|
+
const { tenantId, scopeId } = assignment.node;
|
|
3372
|
+
if (!scopeId) {
|
|
3373
|
+
// One ControlPlaneDO method (#1184): the revoke, the removal fence (raised even when
|
|
3374
|
+
// nothing was held) and the audit row, so no add can land between any two of them.
|
|
3375
|
+
const changed = await this.cp.revokeAndFence(tenantId, assignment.principalId, relation, `tenant:${tenantId}`, this.adminEntry(actor, 'unassignRole', { tenantId, scopeId: null }, assignment, null));
|
|
3376
|
+
// A tenant-level revoke changes the projected set — the tombstone must reach scopes.
|
|
3377
|
+
if (changed)
|
|
3378
|
+
await this.fanOut(tenantId);
|
|
3379
|
+
return;
|
|
3380
|
+
}
|
|
3381
|
+
const changed = await this.scopeStub(scopeId).revokeTuple(`principal:${assignment.principalId}`, relation, `scope:${scopeId}`, new Date().toISOString());
|
|
3118
3382
|
if (!changed)
|
|
3119
3383
|
return;
|
|
3120
|
-
await this.recordAdmin(actor, 'unassignRole', { tenantId
|
|
3121
|
-
// A tenant-level revoke changes the projected set — the tombstone must reach scopes.
|
|
3122
|
-
if (!assignment.node.scopeId)
|
|
3123
|
-
await this.fanOut(assignment.node.tenantId);
|
|
3384
|
+
await this.recordAdmin(actor, 'unassignRole', { tenantId, scopeId }, assignment, null);
|
|
3124
3385
|
},
|
|
3125
3386
|
grant: async (actor, raw) => {
|
|
3126
3387
|
// Parsed like its `grantToConnection`/`grantToSystem` siblings, not taken on
|
|
@@ -3242,11 +3503,17 @@ export class CloudflareScopeHost {
|
|
|
3242
3503
|
},
|
|
3243
3504
|
tenantHeldSystemModules: async (actor, tenantId, moduleIds) => {
|
|
3244
3505
|
const held = moduleIds.length
|
|
3245
|
-
? await this.cp.
|
|
3506
|
+
? await this.cp.tenantHeldOf('system', tenantId, moduleIds, new Date().toISOString())
|
|
3246
3507
|
: [];
|
|
3247
3508
|
await this.recordAccess(actor, 'tenantHeldSystemModules', { tenantId }, { moduleIds: [...moduleIds] }, held.length);
|
|
3248
3509
|
return held;
|
|
3249
3510
|
},
|
|
3511
|
+
peerSwitchCarry: async (actor, node) => {
|
|
3512
|
+
const { keys: switchedOffPeers, tenantHeld: tenantHeldPeers, fences: all, } = await this.recordedOff('peer', node.tenantId, node.scopeId, new Date().toISOString());
|
|
3513
|
+
const fences = Object.fromEntries(switchedOffPeers.map((v) => [v, all.get(v)]));
|
|
3514
|
+
await this.recordAccess(actor, 'peerSwitchCarry', node, null, switchedOffPeers.length);
|
|
3515
|
+
return { switchedOffPeers, tenantHeldPeers, fences };
|
|
3516
|
+
},
|
|
3250
3517
|
reassertSystemSwitches: reassertSystemSwitchesOf,
|
|
3251
3518
|
// #1672 — the platform's two capability verbs. Audited AFTER the write, on both, and
|
|
3252
3519
|
// the failure each leaves is the safe one: a mint whose audit row did not land never
|
|
@@ -3312,7 +3579,7 @@ export class CloudflareScopeHost {
|
|
|
3312
3579
|
// another tenant's traffic. Exception: the holder is ARCHIVED or REAPED (a
|
|
3313
3580
|
// deleted app, storage since wiped) — it has released the name, so the rebind
|
|
3314
3581
|
// reclaims it.
|
|
3315
|
-
throw
|
|
3582
|
+
throw substratError('conflict', `hostname '${parsed.hostname}' is already bound to another scope`);
|
|
3316
3583
|
}
|
|
3317
3584
|
// Exactly one canonical per (scope, surface).
|
|
3318
3585
|
if (parsed.canonical)
|
|
@@ -3333,7 +3600,7 @@ export class CloudflareScopeHost {
|
|
|
3333
3600
|
const hostname = raw.toLowerCase(); // DNS is case-insensitive; the map is normalized
|
|
3334
3601
|
const row = await this.cp.readHostname(hostname);
|
|
3335
3602
|
if (!row)
|
|
3336
|
-
throw
|
|
3603
|
+
throw substratError('not_found', `unknown hostname '${hostname}'`);
|
|
3337
3604
|
if (row.status === status)
|
|
3338
3605
|
return; // idempotent, unaudited
|
|
3339
3606
|
await this.cp.setHostnameStatus(hostname, status, note ?? null);
|
|
@@ -3343,7 +3610,7 @@ export class CloudflareScopeHost {
|
|
|
3343
3610
|
const hostname = raw.toLowerCase(); // DNS is case-insensitive; the map is normalized
|
|
3344
3611
|
const row = await this.cp.readHostname(hostname);
|
|
3345
3612
|
if (!row)
|
|
3346
|
-
throw
|
|
3613
|
+
throw substratError('not_found', `unknown hostname '${hostname}'`);
|
|
3347
3614
|
// A poll that finds nothing changed (same status, same records, id already set)
|
|
3348
3615
|
// is not an event — skip the write and the audit entry, so the reconcile sweep
|
|
3349
3616
|
// does not flood the admin log with no-op rows every interval.
|
|
@@ -3429,7 +3696,7 @@ export class CloudflareScopeHost {
|
|
|
3429
3696
|
if (existing.owner_tenant !== parsed.ownerTenant) {
|
|
3430
3697
|
throw substratError('conflict', `vertical '${parsed.slug}' is owned by ${existing.owner_tenant ?? 'the platform'}, not ${parsed.ownerTenant ?? 'the platform'}`);
|
|
3431
3698
|
}
|
|
3432
|
-
throw
|
|
3699
|
+
throw substratError('conflict', `vertical '${parsed.slug}' is already registered as ${existing.source}`);
|
|
3433
3700
|
}
|
|
3434
3701
|
await this.cp.insertVertical(parsed.slug, parsed.name, parsed.source, parsed.ownerTenant, envSpecJson, installSpecJson, parsed.listed ? 1 : 0, new Date().toISOString());
|
|
3435
3702
|
await this.recordAdmin(actor, 'registerVertical', { tenantId: null }, null, parsed);
|
|
@@ -3610,11 +3877,11 @@ export class CloudflareScopeHost {
|
|
|
3610
3877
|
// nothing to diff against — the gate is about change, not existence.
|
|
3611
3878
|
if (outgoing) {
|
|
3612
3879
|
if (outgoing.permission_digest !== incoming.permission_digest && !ack.permissionChange) {
|
|
3613
|
-
throw
|
|
3880
|
+
throw substratError('conflict', `promotion changes the permission surface (${outgoing.permission_digest} → ` +
|
|
3614
3881
|
`${incoming.permission_digest}) — acknowledge it explicitly to promote`);
|
|
3615
3882
|
}
|
|
3616
3883
|
if (outgoing.migration_digest !== incoming.migration_digest && !ack.migrationChange) {
|
|
3617
|
-
throw
|
|
3884
|
+
throw substratError('conflict', `promotion changes migrations (${outgoing.migration_digest} → ` +
|
|
3618
3885
|
`${incoming.migration_digest}) — acknowledge it explicitly to promote`);
|
|
3619
3886
|
}
|
|
3620
3887
|
// #1705 PR 3: an export an installed consumer imports, dropped or re-versioned.
|
|
@@ -3874,7 +4141,7 @@ export class CloudflareScopeHost {
|
|
|
3874
4141
|
},
|
|
3875
4142
|
createOrg: async (actor, input) => {
|
|
3876
4143
|
const parsed = createOrgInput.parse(input);
|
|
3877
|
-
const created = await this.
|
|
4144
|
+
const created = await this.directory('createOrg', parsed.id, parsed.tenantId, parsed.slug, parsed.name, new Date().toISOString());
|
|
3878
4145
|
if (!created)
|
|
3879
4146
|
return; // idempotent, and a no-op is not audited
|
|
3880
4147
|
await this.recordAdmin(actor, 'createOrg', { tenantId: parsed.tenantId }, null, parsed);
|
|
@@ -3895,15 +4162,27 @@ export class CloudflareScopeHost {
|
|
|
3895
4162
|
await this.recordAdmin(actor, 'addMember', { tenantId }, null, { principal, orgId });
|
|
3896
4163
|
await this.fanOut(tenantId); // membership is a tenant-level tuple
|
|
3897
4164
|
},
|
|
4165
|
+
applyMembership: async (actor, change) => {
|
|
4166
|
+
// The row is minted here, where attribution and `causedBy` live, and written by the
|
|
4167
|
+
// ControlPlaneDO in the same synchronous method as the fence, the bound and the tuple
|
|
4168
|
+
// (#1184): one DO unit, so nothing lands between the check and the write.
|
|
4169
|
+
const { tenantId, principal, roleKey, op } = change;
|
|
4170
|
+
const assignment = { principalId: principal, roleKey, node: { tenantId, scopeId: null } };
|
|
4171
|
+
const result = await this.cp.applyMembership(change, op === 'add'
|
|
4172
|
+
? this.adminEntry(actor, 'assignRole', { tenantId, scopeId: null }, null, assignment)
|
|
4173
|
+
: this.adminEntry(actor, 'unassignRole', { tenantId, scopeId: null }, assignment, null));
|
|
4174
|
+
// The tenant-level tuple, or its tombstone, reaches the projections.
|
|
4175
|
+
if (result.applied && (op === 'add' || result.changed))
|
|
4176
|
+
await this.fanOut(tenantId);
|
|
4177
|
+
return result;
|
|
4178
|
+
},
|
|
3898
4179
|
removeMember: async (actor, tenantId, principal, orgId) => {
|
|
3899
4180
|
await requireOrg(tenantId, orgId);
|
|
3900
|
-
// Tombstone (K-21), never DELETE
|
|
3901
|
-
//
|
|
3902
|
-
const changed = await this.cp.
|
|
3903
|
-
if (
|
|
3904
|
-
|
|
3905
|
-
await this.recordAdmin(actor, 'removeMember', { tenantId }, { principal, orgId }, null);
|
|
3906
|
-
await this.fanOut(tenantId); // the tombstone must reach the projections
|
|
4181
|
+
// Tombstone (K-21), never DELETE, with the removal fence in the same DO method
|
|
4182
|
+
// (#1184). A repeat revoke stays a silent no-op rather than a second audit row.
|
|
4183
|
+
const changed = await this.cp.revokeAndFence(tenantId, principal, 'member', `org:${orgId}`, this.adminEntry(actor, 'removeMember', { tenantId }, { principal, orgId }, null));
|
|
4184
|
+
if (changed)
|
|
4185
|
+
await this.fanOut(tenantId); // the tombstone must reach the projections
|
|
3907
4186
|
},
|
|
3908
4187
|
listMembers: async (actor, tenantId, orgId, options) => {
|
|
3909
4188
|
await requireOrg(tenantId, orgId);
|
|
@@ -3917,20 +4196,20 @@ export class CloudflareScopeHost {
|
|
|
3917
4196
|
},
|
|
3918
4197
|
createTenant: async (actor, input) => {
|
|
3919
4198
|
const parsed = createTenantInput.parse(input);
|
|
3920
|
-
const created = await this.
|
|
4199
|
+
const created = await this.directory('createTenant', parsed.id, parsed.slug, parsed.name, new Date().toISOString(), parsed.provisionedByTenant ?? null);
|
|
3921
4200
|
// Idempotent: re-creating an existing tenant is a no-op, not audited.
|
|
3922
4201
|
if (!created)
|
|
3923
4202
|
return;
|
|
3924
4203
|
await this.recordAdmin(actor, 'createTenant', { tenantId: parsed.id }, null, created);
|
|
3925
4204
|
},
|
|
3926
4205
|
setTenantStatus: async (actor, tenantId, status) => {
|
|
3927
|
-
const before = await this.
|
|
4206
|
+
const before = await this.directory('setTenantStatus', tenantId, status);
|
|
3928
4207
|
await this.recordAdmin(actor, 'setTenantStatus', { tenantId }, { status: before }, { status });
|
|
3929
4208
|
// #1713: every hosted scope under the tenant holds or resumes its work by this.
|
|
3930
4209
|
await this.deliverLifecycles(actor, { tenantId });
|
|
3931
4210
|
},
|
|
3932
4211
|
setTenantName: async (actor, tenantId, name) => {
|
|
3933
|
-
const before = await this.
|
|
4212
|
+
const before = await this.directory('setTenantName', tenantId, name);
|
|
3934
4213
|
if (before === name)
|
|
3935
4214
|
return; // no-op is not audited — nothing changed
|
|
3936
4215
|
await this.recordAdmin(actor, 'setTenantName', { tenantId }, { name: before }, { name });
|
|
@@ -3940,7 +4219,7 @@ export class CloudflareScopeHost {
|
|
|
3940
4219
|
// first (archive-if-needed → reapScope in the vertical deployment); the DO clears
|
|
3941
4220
|
// the tenant's PII/config rows and flips the row to a `reaped` tombstone, keeping
|
|
3942
4221
|
// the row + admin log. Only a `deleting` tenant may be reaped (checked in the DO).
|
|
3943
|
-
const before = await this.
|
|
4222
|
+
const before = await this.directory('reapTenant', tenantId);
|
|
3944
4223
|
await this.recordAdmin(actor, 'reapTenant', { tenantId }, { status: before }, { status: 'reaped' });
|
|
3945
4224
|
},
|
|
3946
4225
|
listTenants: async (actor, page) => {
|
|
@@ -4024,7 +4303,7 @@ export class CloudflareScopeHost {
|
|
|
4024
4303
|
// Not `peerScopeGate`: a read must not migrate, and the phase makes this call once
|
|
4025
4304
|
// per edge per pass, where an extra round trip is a fleet-wide cost.
|
|
4026
4305
|
this.assertServedHere(await this.scopeRecordForRead(tenantId, scopeId), scopeId, 'readExportedEvents');
|
|
4027
|
-
const batch =
|
|
4306
|
+
const batch = await this.readExportsThroughDoor(input, tenantId, scopeId);
|
|
4028
4307
|
await this.recordAccess(actor, 'readExportedEvents', { tenantId, scopeId }, { consumer: input.consumer, after: input.after, limit: input.limit }, batch.events.length);
|
|
4029
4308
|
return batch;
|
|
4030
4309
|
},
|
|
@@ -4204,12 +4483,18 @@ export class CloudflareScopeHost {
|
|
|
4204
4483
|
await this.recordAccess(actor, 'lifecycleFlow', { tenantId, scopeId }, delegatedReadParams.lifecycleFlow(input), flow.observation.events);
|
|
4205
4484
|
return flow;
|
|
4206
4485
|
},
|
|
4486
|
+
operationSeries: async (actor, tenantId, scopeId, input) => {
|
|
4487
|
+
await this.scopeRecordForRead(tenantId, scopeId);
|
|
4488
|
+
const series = await this.scopeStub(scopeId).operationSeries(input);
|
|
4489
|
+
await this.recordAccess(actor, 'operationSeries', { tenantId, scopeId }, delegatedReadParams.operationSeries(input), operationSeriesCount(series));
|
|
4490
|
+
return series;
|
|
4491
|
+
},
|
|
4207
4492
|
readScopeTable: async (actor, tenantId, scopeId, input) => {
|
|
4208
4493
|
await this.scopeRecordForRead(tenantId, scopeId);
|
|
4209
4494
|
// Checked here as well as in the ScopeDO, so the refusal keeps its code across the hop.
|
|
4210
4495
|
assertRowLimit('limit', input.limit);
|
|
4211
4496
|
assertRowOffset('offset', input.offset);
|
|
4212
|
-
const page = await this.scopeStub(scopeId).
|
|
4497
|
+
const page = unwrapReply(await this.scopeStub(scopeId).introspectTableReply(input.table, input.limit, input.offset));
|
|
4213
4498
|
await this.recordAccess(actor, 'readScopeTable', { tenantId, scopeId }, { table: input.table, limit: page.limit, offset: page.offset }, page.rows.length);
|
|
4214
4499
|
return page;
|
|
4215
4500
|
},
|
|
@@ -4250,7 +4535,7 @@ export class CloudflareScopeHost {
|
|
|
4250
4535
|
},
|
|
4251
4536
|
queryScope: async (actor, tenantId, scopeId, input) => {
|
|
4252
4537
|
await this.scopeRecordForRead(tenantId, scopeId);
|
|
4253
|
-
const result = await this.scopeStub(scopeId).
|
|
4538
|
+
const result = unwrapReply(await this.scopeStub(scopeId).introspectQueryReply(input.sql));
|
|
4254
4539
|
// The statement is the logged argument: the access log is the evidence trail,
|
|
4255
4540
|
// and for a console read the SQL is the whole story.
|
|
4256
4541
|
await this.recordAccess(actor, 'queryScope', { tenantId, scopeId }, { sql: input.sql }, result.rows.length);
|
|
@@ -4343,7 +4628,7 @@ export class CloudflareScopeHost {
|
|
|
4343
4628
|
if (!rec)
|
|
4344
4629
|
throw substratError('not_found', `unknown scope ${scopeId} in tenant ${tenantId}`);
|
|
4345
4630
|
if (rec.status !== 'archived') {
|
|
4346
|
-
throw
|
|
4631
|
+
throw substratError('conflict', `scope ${scopeId} is ${rec.status}, not archived — only an archived scope may be reaped`);
|
|
4347
4632
|
}
|
|
4348
4633
|
// A serving scope always holds ≥1 bound hostname; a truly-dead one has been
|
|
4349
4634
|
// unbound. The dashboard delete path unbinds AT archive, but a bare console
|
|
@@ -4355,7 +4640,7 @@ export class CloudflareScopeHost {
|
|
|
4355
4640
|
// being released anyway; the interactive per-scope reap never sets it.
|
|
4356
4641
|
const bound = opts?.force ? [] : await this.cp.listHostnames({ scopeId, limit: 1 });
|
|
4357
4642
|
if (bound.length > 0) {
|
|
4358
|
-
throw
|
|
4643
|
+
throw substratError('conflict', `scope ${scopeId} still resolves hostname '${bound[0].hostname}' — ` +
|
|
4359
4644
|
`unbind it before reaping (reap wipes storage and cannot be undone)`);
|
|
4360
4645
|
}
|
|
4361
4646
|
await this.scopeStub(scopeId).destroyStorage();
|
|
@@ -4553,7 +4838,7 @@ export class CloudflareScopeHost {
|
|
|
4553
4838
|
},
|
|
4554
4839
|
registerIdentityPool: async (actor, input) => {
|
|
4555
4840
|
const parsed = identityPool.parse(input);
|
|
4556
|
-
const created = await this.
|
|
4841
|
+
const created = await this.directory('registerIdentityPool', parsed.provider, parsed.topology, parsed.tenantId, new Date().toISOString());
|
|
4557
4842
|
if (!created)
|
|
4558
4843
|
return; // identical registration is idempotent, unaudited
|
|
4559
4844
|
// Null tenant for a central pool: it belongs to no single tenant, which is
|
|
@@ -4570,9 +4855,9 @@ export class CloudflareScopeHost {
|
|
|
4570
4855
|
listIdentityTenants: async (actor, provider, externalId) => {
|
|
4571
4856
|
const r = await this.cp.readPool(provider);
|
|
4572
4857
|
if (!r)
|
|
4573
|
-
throw
|
|
4858
|
+
throw substratError('not_found', `identity pool '${provider}' is not registered`);
|
|
4574
4859
|
if (r.topology !== 'central') {
|
|
4575
|
-
throw
|
|
4860
|
+
throw substratError('forbidden', `identity pool '${provider}' is tenant-bound — enumerating tenants is only ` +
|
|
4576
4861
|
`meaningful on a central pool, where the same externalId is the same person`);
|
|
4577
4862
|
}
|
|
4578
4863
|
const tenants = (await this.cp.identityTenants(provider, externalId));
|
|
@@ -4589,9 +4874,9 @@ export class CloudflareScopeHost {
|
|
|
4589
4874
|
at: new Date().toISOString(),
|
|
4590
4875
|
});
|
|
4591
4876
|
if (topology === null)
|
|
4592
|
-
throw
|
|
4877
|
+
throw substratError('not_found', `identity pool '${provider}' is not registered`);
|
|
4593
4878
|
if (topology !== 'central') {
|
|
4594
|
-
throw
|
|
4879
|
+
throw substratError('forbidden', `identity pool '${provider}' is tenant-bound — enumerating tenants is only ` +
|
|
4595
4880
|
`meaningful on a central pool, where the same externalId is the same person`);
|
|
4596
4881
|
}
|
|
4597
4882
|
return memberships.map((m) => identityMembership.parse(m));
|
|
@@ -4744,14 +5029,14 @@ export class CloudflareScopeHost {
|
|
|
4744
5029
|
const parsed = identityLink.parse(input);
|
|
4745
5030
|
const pool = await this.cp.readPool(parsed.provider);
|
|
4746
5031
|
if (!pool) {
|
|
4747
|
-
throw
|
|
5032
|
+
throw substratError('conflict', `identity pool '${parsed.provider}' is not registered — a pool must declare ` +
|
|
4748
5033
|
`its topology before it may link (central vs tenant-bound decides whether ` +
|
|
4749
5034
|
`the same externalId in two tenants is one person or two)`);
|
|
4750
5035
|
}
|
|
4751
5036
|
if (pool.topology === 'tenant-bound' && pool.tenant_id !== parsed.tenantId) {
|
|
4752
|
-
throw
|
|
5037
|
+
throw substratError('conflict', `identity pool '${parsed.provider}' is bound to tenant ${pool.tenant_id} and cannot link into ${parsed.tenantId}`);
|
|
4753
5038
|
}
|
|
4754
|
-
const changed = await this.
|
|
5039
|
+
const changed = await this.directory('linkIdentity', parsed.provider, parsed.externalId, parsed.principal, parsed.tenantId, parsed.scopeId ?? null, new Date().toISOString());
|
|
4755
5040
|
// Idempotent: an identity already bound is a no-op, not audited.
|
|
4756
5041
|
if (!changed)
|
|
4757
5042
|
return;
|
|
@@ -5013,6 +5298,10 @@ export class CloudflareScopeHost {
|
|
|
5013
5298
|
};
|
|
5014
5299
|
}
|
|
5015
5300
|
// -- helpers --------------------------------------------------------------
|
|
5301
|
+
/** A directory write whose refusal must keep its code across the DO hop (#113). */
|
|
5302
|
+
async directory(method, ...args) {
|
|
5303
|
+
return unwrapReply(await this.cp.reply(method, args));
|
|
5304
|
+
}
|
|
5016
5305
|
/**
|
|
5017
5306
|
* The getScope gate (control-plane.md §4.1/§4.2) — the pair check, then the tenant's and
|
|
5018
5307
|
* the scope's lifecycle — thrown on THIS side of the RPC (#1718). The ControlPlaneDO
|
|
@@ -5032,8 +5321,11 @@ export class CloudflareScopeHost {
|
|
|
5032
5321
|
*/
|
|
5033
5322
|
async validateScopeAccess(tenantId, scopeId) {
|
|
5034
5323
|
const refusal = await this.cp.scopeAccessRefusal(tenantId, scopeId);
|
|
5035
|
-
if (refusal)
|
|
5036
|
-
throw refusal.code
|
|
5324
|
+
if (refusal) {
|
|
5325
|
+
throw refusal.code
|
|
5326
|
+
? substratError(refusal.code, refusal.message, refusal.reason ? { reason: refusal.reason } : undefined)
|
|
5327
|
+
: new Error(refusal.message);
|
|
5328
|
+
}
|
|
5037
5329
|
if (!this.cpLess)
|
|
5038
5330
|
return null;
|
|
5039
5331
|
const { verdict, lifecycle } = await this.scopeStub(scopeId).admission(tenantId);
|
|
@@ -5065,7 +5357,7 @@ export class CloudflareScopeHost {
|
|
|
5065
5357
|
async assertLive(tenantId, scopeId) {
|
|
5066
5358
|
const refusal = lifecycleRefusal(await this.validateScopeAccess(tenantId, scopeId), { tenantId, scopeId });
|
|
5067
5359
|
if (refusal)
|
|
5068
|
-
throw
|
|
5360
|
+
throw substratError('conflict', refusal, { reason: SCOPE_GATE_REASONS.notActive });
|
|
5069
5361
|
}
|
|
5070
5362
|
/**
|
|
5071
5363
|
* Deliver the directory's lifecycle to the deployments serving the matching scopes (#1713):
|
|
@@ -5262,7 +5554,7 @@ export class CloudflareScopeHost {
|
|
|
5262
5554
|
if (!row)
|
|
5263
5555
|
throw unknownScopeForTenant(tenantId, scopeId);
|
|
5264
5556
|
if (row.status === 'reaped') {
|
|
5265
|
-
throw
|
|
5557
|
+
throw substratError('conflict', `scope ${scopeId} is reaped — its storage is gone and cannot be read`);
|
|
5266
5558
|
}
|
|
5267
5559
|
return row;
|
|
5268
5560
|
}
|
|
@@ -5295,7 +5587,14 @@ export class CloudflareScopeHost {
|
|
|
5295
5587
|
});
|
|
5296
5588
|
}
|
|
5297
5589
|
async recordAdmin(actor, action, target, before, after) {
|
|
5298
|
-
await this.cp.recordAdmin(
|
|
5590
|
+
await this.cp.recordAdmin(this.adminEntry(actor, action, target, before, after));
|
|
5591
|
+
}
|
|
5592
|
+
/**
|
|
5593
|
+
* One admin-log row, minted here where attribution and `causedBy` live — for `recordAdmin`,
|
|
5594
|
+
* and for a ControlPlaneDO method that writes its row in the same unit as its effect (#1184).
|
|
5595
|
+
*/
|
|
5596
|
+
adminEntry(actor, action, target, before, after) {
|
|
5597
|
+
return {
|
|
5299
5598
|
id: ulid(),
|
|
5300
5599
|
actor,
|
|
5301
5600
|
action,
|
|
@@ -5307,7 +5606,7 @@ export class CloudflareScopeHost {
|
|
|
5307
5606
|
before: before ?? null,
|
|
5308
5607
|
after: after ?? null,
|
|
5309
5608
|
at: new Date().toISOString(),
|
|
5310
|
-
}
|
|
5609
|
+
};
|
|
5311
5610
|
}
|
|
5312
5611
|
/**
|
|
5313
5612
|
* The scope's own answer to "what may this connection do here" (#726 gap 1).
|
|
@@ -5432,6 +5731,11 @@ export class CloudflareScopeHost {
|
|
|
5432
5731
|
// rewind CLAIMS them on `SWITCH_HOLDS_NAME`, which the rewind cannot reach. A module is held
|
|
5433
5732
|
// while any claim on it exists, and `runDueSchedules` skips a held module.
|
|
5434
5733
|
//
|
|
5734
|
+
// #2029: the same for a PEER switched off (`vertical:<slug>`): the rewind claims the scope's
|
|
5735
|
+
// off peers beside its off modules, each under its hold key (`holdKeyOf`), and the peer door
|
|
5736
|
+
// refuses a held peer as the system door refuses a held module. Everything below holds for
|
|
5737
|
+
// both kinds; "module" reads as "subject".
|
|
5738
|
+
//
|
|
5435
5739
|
// A claim is released only by a switch move KNOWN to survive its rewind (`switchInScope`): a
|
|
5436
5740
|
// move on the restored storage, after the rewind armed. A move during the settle, or one the
|
|
5437
5741
|
// doomed instance applied, lands in storage the restart discards, so it releases nothing. A
|
|
@@ -5450,7 +5754,7 @@ export class CloudflareScopeHost {
|
|
|
5450
5754
|
* reopens only the gap this closes, for a rewind that lands during the outage, and the next
|
|
5451
5755
|
* reconcile still switches that module off. The error is answered, for the pass to report.
|
|
5452
5756
|
*/
|
|
5453
|
-
async switchHeld(scopeId,
|
|
5757
|
+
async switchHeld(scopeId, holdKey) {
|
|
5454
5758
|
const at = Date.now();
|
|
5455
5759
|
let error;
|
|
5456
5760
|
if (!this.holdSnapshot || at - this.holdSnapshot.at > SWITCH_HOLD_SNAPSHOT_MS) {
|
|
@@ -5473,8 +5777,11 @@ export class CloudflareScopeHost {
|
|
|
5473
5777
|
}
|
|
5474
5778
|
}
|
|
5475
5779
|
}
|
|
5476
|
-
const
|
|
5477
|
-
|
|
5780
|
+
const snapshot = this.holdSnapshot;
|
|
5781
|
+
const held = snapshot?.held.has(`${scopeId} ${holdKey}`) ?? false;
|
|
5782
|
+
// #2029: a good read naming no row for this scope at all, for `doorGate`'s instance cache.
|
|
5783
|
+
const scopeClear = !error && snapshot?.good === true && !snapshot.scopes.has(scopeId);
|
|
5784
|
+
return { held, ...(error ? { error } : {}), ...(scopeClear ? { scopeClear: true } : {}) };
|
|
5478
5785
|
}
|
|
5479
5786
|
/** One read of the whole hold into the snapshot, stamped with when it went out. Never throws. */
|
|
5480
5787
|
async refreshHolds(at) {
|
|
@@ -5482,7 +5789,12 @@ export class CloudflareScopeHost {
|
|
|
5482
5789
|
const rows = await this.switchHoldsStub().switchHoldsAll();
|
|
5483
5790
|
// A slow read that lands after a newer one never replaces it.
|
|
5484
5791
|
if (!this.holdSnapshot || at >= this.holdSnapshot.at) {
|
|
5485
|
-
this.holdSnapshot = {
|
|
5792
|
+
this.holdSnapshot = {
|
|
5793
|
+
at,
|
|
5794
|
+
held: new Set(rows.map((r) => `${r.scopeId} ${r.moduleId}`)),
|
|
5795
|
+
scopes: new Set(rows.map((r) => r.scopeId)),
|
|
5796
|
+
good: true,
|
|
5797
|
+
};
|
|
5486
5798
|
}
|
|
5487
5799
|
return {};
|
|
5488
5800
|
}
|
|
@@ -5493,8 +5805,10 @@ export class CloudflareScopeHost {
|
|
|
5493
5805
|
// than hammered per scope. For that long, other scopes reuse this possibly pre-hold set
|
|
5494
5806
|
// silently, and only the consult that sent this read reports the error: the declared
|
|
5495
5807
|
// fail-open, attributed once.
|
|
5496
|
-
if (!this.holdSnapshot || at >= this.holdSnapshot.at)
|
|
5497
|
-
|
|
5808
|
+
if (!this.holdSnapshot || at >= this.holdSnapshot.at) {
|
|
5809
|
+
const scopes = new Set([...(known ?? [])].map((k) => k.slice(0, k.indexOf(' '))));
|
|
5810
|
+
this.holdSnapshot = { at, held: known ?? new Set(), scopes, good: false };
|
|
5811
|
+
}
|
|
5498
5812
|
return {
|
|
5499
5813
|
error: known
|
|
5500
5814
|
? `switch hold unreadable (${why}); the ${known.size} hold(s) from this pass's last good read ` +
|
|
@@ -5541,7 +5855,11 @@ export class CloudflareScopeHost {
|
|
|
5541
5855
|
* its claims before the OFF's join lands. The joined row then holds the module the operator just
|
|
5542
5856
|
* turned ON, until the next surviving move of that switch releases it (another ON does).
|
|
5543
5857
|
*/
|
|
5544
|
-
async switchInScope(scopeId,
|
|
5858
|
+
async switchInScope(kind, scopeId, key, to, at, tenantHeld = false,
|
|
5859
|
+
/** #2045: the switch call's fence; see `SWITCH_FENCES_DDL`. */
|
|
5860
|
+
fence) {
|
|
5861
|
+
// #2029: a peer's claims live beside the modules', under its own hold key.
|
|
5862
|
+
const holdKey = holdKeyOf(kind, key);
|
|
5545
5863
|
let scopeClaims;
|
|
5546
5864
|
let readError;
|
|
5547
5865
|
try {
|
|
@@ -5550,23 +5868,30 @@ export class CloudflareScopeHost {
|
|
|
5550
5868
|
catch (err) {
|
|
5551
5869
|
readError = err;
|
|
5552
5870
|
}
|
|
5553
|
-
const
|
|
5871
|
+
const stub = this.scopeStub(scopeId);
|
|
5872
|
+
const { instance, ...outcome } = await (kind === 'system'
|
|
5873
|
+
? stub.switchSystemSchedules(key, scopeId, to, at, tenantHeld, fence)
|
|
5874
|
+
: stub.switchPeer(key, scopeId, to, at, tenantHeld, fence));
|
|
5554
5875
|
if (!scopeClaims)
|
|
5555
5876
|
throw readError;
|
|
5877
|
+
// #2045: a move the fence refused wrote nothing, so it is known to survive nothing and to
|
|
5878
|
+
// release nothing; the newer call that superseded it did whatever the hold needed.
|
|
5879
|
+
if (outcome.superseded)
|
|
5880
|
+
return outcome;
|
|
5556
5881
|
const now = Date.now();
|
|
5557
5882
|
const pendingLive = (c) => c.state === 'pending' && now - Date.parse(c.heldAt) <= SWITCH_HOLD_PENDING_MAX_MS;
|
|
5558
|
-
const before = scopeClaims.filter((c) => c.moduleId ===
|
|
5883
|
+
const before = scopeClaims.filter((c) => c.moduleId === holdKey);
|
|
5559
5884
|
if (to === 'on') {
|
|
5560
5885
|
// Every ON, claims or not: its tombstone is what keeps a rewind's stale read from claiming
|
|
5561
5886
|
// the module after this (#1839).
|
|
5562
5887
|
this.holdSnapshot = null;
|
|
5563
|
-
await this.switchHoldsStub().switchHoldOn(scopeId,
|
|
5888
|
+
await this.switchHoldsStub().switchHoldOn(scopeId, holdKey, before.map((c) => c.claimId));
|
|
5564
5889
|
}
|
|
5565
5890
|
else if (outcome.held && before.length > 0) {
|
|
5566
5891
|
const survived = before.filter((c) => !pendingLive(c) && c.doomed !== instance);
|
|
5567
5892
|
if (survived.length > 0) {
|
|
5568
5893
|
this.holdSnapshot = null;
|
|
5569
|
-
await this.switchHoldsStub().switchHoldRelease(scopeId,
|
|
5894
|
+
await this.switchHoldsStub().switchHoldRelease(scopeId, holdKey, survived.map((c) => c.claimId));
|
|
5570
5895
|
}
|
|
5571
5896
|
}
|
|
5572
5897
|
if (to === 'off' && outcome.changed) {
|
|
@@ -5577,7 +5902,7 @@ export class CloudflareScopeHost {
|
|
|
5577
5902
|
const joins = new Set([...scopeClaims, ...after].filter((c) => pendingLive(c) || c.doomed === instance).map((c) => c.claimId));
|
|
5578
5903
|
if (joins.size > 0) {
|
|
5579
5904
|
this.holdSnapshot = null;
|
|
5580
|
-
await this.switchHoldsStub().switchHoldJoin(scopeId,
|
|
5905
|
+
await this.switchHoldsStub().switchHoldJoin(scopeId, holdKey, [...joins]);
|
|
5581
5906
|
}
|
|
5582
5907
|
}
|
|
5583
5908
|
return outcome;
|
|
@@ -5641,11 +5966,16 @@ export class CloudflareScopeHost {
|
|
|
5641
5966
|
await holds.switchHoldArm(scopeId, claimId, result.instance ?? null).catch(() => undefined);
|
|
5642
5967
|
return { rewindingTo: result.rewindingTo };
|
|
5643
5968
|
}
|
|
5644
|
-
/**
|
|
5969
|
+
/**
|
|
5970
|
+
* The subjects this scope's own storage has switched off, as a rewind reads them before it arms:
|
|
5971
|
+
* its modules, and (#2029) its peers, each by its hold key (`holdKeyOf`).
|
|
5972
|
+
*/
|
|
5645
5973
|
async offInStorage(scopeId) {
|
|
5646
|
-
|
|
5647
|
-
|
|
5648
|
-
.map((e) => e.moduleId)
|
|
5974
|
+
const [modules, peers] = await Promise.all([this.systemGrantsStatusLocal(scopeId), this.peerGrantsStatusLocal(scopeId)]);
|
|
5975
|
+
return [
|
|
5976
|
+
...modules.filter((e) => e.schedules === 'off').map((e) => holdKeyOf('system', e.moduleId)),
|
|
5977
|
+
...peers.filter((e) => e.calls === 'off').map((e) => holdKeyOf('peer', e.vertical)),
|
|
5978
|
+
];
|
|
5649
5979
|
}
|
|
5650
5980
|
/**
|
|
5651
5981
|
* #1839: wait until every row of this rewind's claim is `SWITCH_HOLD_SETTLE_MS` old. The settle
|
|
@@ -5714,7 +6044,11 @@ export class CloudflareScopeHost {
|
|
|
5714
6044
|
* arrive as a different class of answer just because the caller asked for a socket.
|
|
5715
6045
|
*/
|
|
5716
6046
|
liveReads = {
|
|
5717
|
-
subscribe: async ({ tenantId, scopeId, principal, request }) => {
|
|
6047
|
+
subscribe: async ({ tenantId, scopeId, principal, request, within }) => {
|
|
6048
|
+
// Decided before anything else, and thrown: a `within` that is neither a plain
|
|
6049
|
+
// EntityRef nor a value `vouchedWithin` built is a caller bug, and the two ways of
|
|
6050
|
+
// guessing at it both open a feed wider than was asked for (#1853).
|
|
6051
|
+
const narrowed = liveWithinOf(within);
|
|
5718
6052
|
if (!isUpgradeRequest(request)) {
|
|
5719
6053
|
return new Response('live reads are a WebSocket surface', {
|
|
5720
6054
|
status: 426,
|
|
@@ -5772,6 +6106,10 @@ export class CloudflareScopeHost {
|
|
|
5772
6106
|
headers.set(LIVE_PRINCIPAL_HEADER, principal);
|
|
5773
6107
|
headers.set(LIVE_TENANT_HEADER, tenantId);
|
|
5774
6108
|
headers.set(LIVE_SCOPE_HEADER, scopeId);
|
|
6109
|
+
if (narrowed)
|
|
6110
|
+
headers.set(LIVE_WITHIN_HEADER, encodeLiveWithin(narrowed));
|
|
6111
|
+
else
|
|
6112
|
+
headers.delete(LIVE_WITHIN_HEADER);
|
|
5775
6113
|
const forwarded = new Request(new URL(LIVE_SUBSCRIBE_PATH, 'https://scope.substrat.internal'),
|
|
5776
6114
|
// `new Request(url, { …, headers })` rather than `new Request(request, …)`:
|
|
5777
6115
|
// the DO is addressed by its own path, not the client's, and the upgrade
|
|
@@ -5853,7 +6191,7 @@ export class CloudflareScopeHost {
|
|
|
5853
6191
|
throw unknownScopeForTenant(tenantId, scopeId);
|
|
5854
6192
|
const { roles, tuples, entitlements, identities } = await this.tenantProjection(tenantId);
|
|
5855
6193
|
const connectionKeys = await this.connectionKeyRows(tenantId, scope.vertical);
|
|
5856
|
-
await this.scopeStub(scopeId).
|
|
6194
|
+
unwrapReply(await this.scopeStub(scopeId).applyProjectionReply(tenantId, roles, tuples, entitlements, undefined, identities, connectionKeys));
|
|
5857
6195
|
}
|
|
5858
6196
|
/**
|
|
5859
6197
|
* Fan the tenant's current state out into ALL its scopes — called after any
|
|
@@ -5902,7 +6240,7 @@ export class CloudflareScopeHost {
|
|
|
5902
6240
|
if (!keysByVertical.has(vertical)) {
|
|
5903
6241
|
keysByVertical.set(vertical, this.connectionKeyRows(tenantId, s.vertical));
|
|
5904
6242
|
}
|
|
5905
|
-
await this.scopeStub(s.scope_id).
|
|
6243
|
+
unwrapReply(await this.scopeStub(s.scope_id).applyProjectionReply(tenantId, roles, tuples, entitlements, undefined, identities, await keysByVertical.get(vertical)));
|
|
5906
6244
|
}));
|
|
5907
6245
|
const failed = settled.flatMap((r, i) => (r.status === 'rejected' ? [{ scope: scopes[i].scope_id, reason: r.reason }] : []));
|
|
5908
6246
|
if (failed.length === 0)
|
|
@@ -5932,10 +6270,11 @@ export class CloudflareScopeHost {
|
|
|
5932
6270
|
* from its own storage. No tenant-level tuples, no control plane.
|
|
5933
6271
|
*/
|
|
5934
6272
|
async provisionScopeLocal(input) {
|
|
6273
|
+
const carry = recordedOffFromWire(input);
|
|
5935
6274
|
const services = await this.servicePrincipals?.(input.tenantId, input.scopeId);
|
|
5936
6275
|
const stub = this.scopeStub(input.scopeId);
|
|
5937
6276
|
await this.migrateAndRecord(input.scopeId); // create the module tables (setMigrationState no-ops on a null CP)
|
|
5938
|
-
const switchedOff = await stub.
|
|
6277
|
+
const switchedOff = unwrapReply(await stub.applyProjectionReply(input.tenantId, input.roles.map((r) => ({ role_key: r.key, permissions: JSON.stringify(r.permissions), source: r.source })), [], // no tenant-level tuples — a CP-less vertical grants at scope level only
|
|
5939
6278
|
// Only PROJECT (and thereby switch on strict enforcement) when the platform actually
|
|
5940
6279
|
// supplies entitlements. Omitting them leaves the scope un-projected and trusting-
|
|
5941
6280
|
// upstream, so a vertical whose provision path predates #304 is never denied — it
|
|
@@ -6025,12 +6364,10 @@ export class CloudflareScopeHost {
|
|
|
6025
6364
|
public_key: k.publicKey,
|
|
6026
6365
|
}))
|
|
6027
6366
|
: undefined,
|
|
6028
|
-
// #1742: the recorded-off modules, switched off after the seat in the
|
|
6029
|
-
// scope this call provisions — the one the seat's tuples name.
|
|
6030
|
-
input.
|
|
6031
|
-
|
|
6032
|
-
: undefined, services?.map((id) => `principal:${id}`));
|
|
6033
|
-
return input.switchedOff ? { switchedOff } : {};
|
|
6367
|
+
// #1742: the recorded-off modules and (#2029) peers, switched off after the seat in the
|
|
6368
|
+
// same unit, on the scope this call provisions — the one the seat's tuples name.
|
|
6369
|
+
carry ? { scopeId: input.scopeId, at: new Date().toISOString(), ...carry } : undefined, services?.map((id) => `principal:${id}`)));
|
|
6370
|
+
return carry ? { switchedOff } : {};
|
|
6034
6371
|
}
|
|
6035
6372
|
/**
|
|
6036
6373
|
* Resolve an external identity from a scope's PROJECTED links (#406) — the CP-less
|
|
@@ -6169,6 +6506,8 @@ export class CloudflareScopeHost {
|
|
|
6169
6506
|
async connectorGrantLocal(connectionId, scopeId, permission, expiresAt) {
|
|
6170
6507
|
await this.writeScopeTuple(scopeId, subjectRef({ kind: 'connection', id: connectionId }), `granted:${permission}`, `scope:${scopeId}`, expiresAt ?? null);
|
|
6171
6508
|
}
|
|
6509
|
+
/** #2045 (Codex r3): every switch mover here honours the fence — what `/internal/switch-fence` reports. */
|
|
6510
|
+
switchFenced = true;
|
|
6172
6511
|
/**
|
|
6173
6512
|
* The far end of the schedule kill switch for a scope served HERE (#1666):
|
|
6174
6513
|
* `/internal/system-switch` lands on this, from the shared control plane's
|
|
@@ -6181,7 +6520,7 @@ export class CloudflareScopeHost {
|
|
|
6181
6520
|
// Parsed on the way out: this is the wire answer the platform reads, and the DO's
|
|
6182
6521
|
// plain strings become the published shape here rather than on trust.
|
|
6183
6522
|
// #1819: `switchInScope` releases a rewind's hold on the module once the switch lands.
|
|
6184
|
-
return systemSwitchOutcome.parse(await this.switchInScope(scopeId, moduleId, to, new Date().toISOString(), opts?.tenantHeld ?? false));
|
|
6523
|
+
return systemSwitchOutcome.parse(await this.switchInScope('system', scopeId, moduleId, to, new Date().toISOString(), opts?.tenantHeld ?? false, opts?.fence));
|
|
6185
6524
|
}
|
|
6186
6525
|
/**
|
|
6187
6526
|
* The far end of the status read for a scope served HERE (#1674): `/internal/system-grants`
|
|
@@ -6216,9 +6555,12 @@ export class CloudflareScopeHost {
|
|
|
6216
6555
|
* Move one peer's kill switch on a scope served HERE (#1706) — the far end of
|
|
6217
6556
|
* `revokeFromPeer` / `restoreToPeer` for a hosted scope. Nothing is audited here; the
|
|
6218
6557
|
* platform writes the admin rows around this call, as for the schedule switch.
|
|
6558
|
+
* #2029: through `switchInScope`, so the move releases a rewind's hold on the peer once it
|
|
6559
|
+
* lands, as a module's does. #2030: `tenantHeld`, the platform's word that the peer holds a
|
|
6560
|
+
* live tenant-level grant, which this deployment has no directory to read.
|
|
6219
6561
|
*/
|
|
6220
|
-
async peerSwitchLocal(scopeId, vertical, to) {
|
|
6221
|
-
return peerSwitchOutcome.parse(await this.
|
|
6562
|
+
async peerSwitchLocal(scopeId, vertical, to, opts) {
|
|
6563
|
+
return peerSwitchOutcome.parse(await this.switchInScope('peer', scopeId, verticalSlug.parse(vertical), to, new Date().toISOString(), opts?.tenantHeld ?? false, opts?.fence));
|
|
6222
6564
|
}
|
|
6223
6565
|
// -- the cross-vertical far ends (#1705 PR 2) --------------------------------
|
|
6224
6566
|
// The shared control plane runs the cross-vertical phase for every hosted edge and reaches the
|
|
@@ -6232,7 +6574,25 @@ export class CloudflareScopeHost {
|
|
|
6232
6574
|
async exportedEventsLocal(tenantId, scopeId, raw) {
|
|
6233
6575
|
const input = exportReadInput.parse(raw);
|
|
6234
6576
|
await this.assertServesLocally(tenantId, scopeId, 'readExportedEvents');
|
|
6235
|
-
return
|
|
6577
|
+
return this.readExportsThroughDoor(input, tenantId, scopeId);
|
|
6578
|
+
}
|
|
6579
|
+
/**
|
|
6580
|
+
* #2029: a producer's export read, through the CONSUMER's peer door: the consumer's grants on
|
|
6581
|
+
* this scope decide what leaves it, so a consumer the rewind hold keeps off must hold nothing
|
|
6582
|
+
* here, exactly as a switched-off one does. Gated and pinned like every door call; held, the
|
|
6583
|
+
* producer answers every key missing, so the edge pauses rather than reading past the hold.
|
|
6584
|
+
*/
|
|
6585
|
+
async readExportsThroughDoor(input, tenantId, scopeId) {
|
|
6586
|
+
const stub = this.scopeStub(scopeId);
|
|
6587
|
+
const door = await this.openDoor('peer', input.consumer, scopeId);
|
|
6588
|
+
try {
|
|
6589
|
+
return exportedBatch.parse(await door.through((instance) => stub.exportedEventsRead(input, tenantId, scopeId, { instance })));
|
|
6590
|
+
}
|
|
6591
|
+
catch (err) {
|
|
6592
|
+
if (!this.isHeldRefusal(err))
|
|
6593
|
+
throw err;
|
|
6594
|
+
return exportedBatch.parse(await stub.exportedEventsRead(input, tenantId, scopeId, { held: true }));
|
|
6595
|
+
}
|
|
6236
6596
|
}
|
|
6237
6597
|
/**
|
|
6238
6598
|
* The consumer's imports and watermarks (#1705), for a scope served HERE. The served-here
|