@substrat-run/adapter-cloudflare 0.137.0 → 0.139.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/checker.d.ts +7 -1
- package/dist/checker.d.ts.map +1 -1
- package/dist/checker.js +2 -2
- package/dist/checker.js.map +1 -1
- package/dist/control-plane-do.d.ts +126 -35
- package/dist/control-plane-do.d.ts.map +1 -1
- package/dist/control-plane-do.js +366 -106
- package/dist/control-plane-do.js.map +1 -1
- package/dist/do-reply.d.ts +25 -0
- package/dist/do-reply.d.ts.map +1 -0
- package/dist/do-reply.js +17 -0
- package/dist/do-reply.js.map +1 -0
- package/dist/host.d.ts +141 -18
- package/dist/host.d.ts.map +1 -1
- package/dist/host.js +893 -462
- package/dist/host.js.map +1 -1
- package/dist/live-reads.d.ts +39 -1
- package/dist/live-reads.d.ts.map +1 -1
- package/dist/live-reads.js +79 -0
- package/dist/live-reads.js.map +1 -1
- package/dist/scope-do.d.ts +2 -7
- package/dist/scope-do.d.ts.map +1 -1
- package/dist/scope-do.js +455 -161
- package/dist/scope-do.js.map +1 -1
- package/dist/sql.d.ts +5 -1
- package/dist/sql.d.ts.map +1 -1
- package/dist/sql.js +6 -2
- package/dist/sql.js.map +1 -1
- package/package.json +14 -12
package/dist/host.js
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
import { env as ambientEnv } from 'cloudflare:workers';
|
|
2
2
|
import { isRewindRefusal, REWIND_REFUSED } from './rewind-refusal.js';
|
|
3
3
|
import { isSystemDoorMoved, SYSTEM_DOOR_MOVED, SYSTEM_DOOR_REGATES } from './system-door.js';
|
|
4
|
-
import { delegatedReadParams, fromWireFailure, exportReadInput, exportedBatch, importBatch, importResult, importState, importsOfManifestJson, exportsOfManifestJson, importCursorMove, importCursorMoveAt, importCursorMoved, accessLogEntry, adminLogEntry, opsFailureEntry, opsFailureFingerprint, issueEntry, sweepRunEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunsPayload, modelUsageEntry, attachmentRecord, createTenantInput, identityLink, identityPool, createOrgInput, promotionAcknowledgement, bindAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, registerVerticalInput, vertical as verticalSchema, verticalServingState, verticalChannel, verticalVersion, connection, capabilityExchange, mintedCapability, capabilityGrant, principalId, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, systemSwitch, systemSwitchOutcome, systemScheduleEntry, systemSwitchRecord, entitlementGrant, entitlementGrantInput, capabilityFilter, instant, meterReading, subjectRef, createConnectionInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, org as orgSchema, orgMembership, resolvedIdentity, identityMembership, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, delegatedReadRecord, ownerTransferAudit, copyMarkAudit, subjectShredReceipt, connectorDispatchKind, callsOfManifestJson, outboundOfManifestJson, substratError, redrainEventsInput, coverage, peerCoverage, peerGrantsEntry, peerSwitch, peerSwitchOutcome, verticalCaller, verticalResolution, verticalSlug, entityObjectRef, ATTACHMENT_BLOB_BINDING, blobStoreBindingName, lifecycleDelivery, scopeLifecycle, } from '@substrat-run/contracts';
|
|
4
|
+
import { delegatedReadParams, operationSeriesCount, fromWireFailure, exportReadInput, exportedBatch, importBatch, importResult, importState, importsOfManifestJson, exportsOfManifestJson, importCursorMove, importCursorMoveAt, importCursorMoved, accessLogEntry, adminLogEntry, opsFailureEntry, opsFailureFingerprint, issueEntry, findingEntry, findingRuleEntry, sweepRunEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunsPayload, modelUsageEntry, attachmentRecord, createTenantInput, identityLink, identityPool, createOrgInput, promotionAcknowledgement, bindAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, registerVerticalInput, vertical as verticalSchema, verticalServingState, verticalChannel, verticalVersion, connection, capabilityExchange, mintedCapability, capabilityGrant, principalId, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, systemSwitch, systemSwitchOutcome, systemScheduleEntry, systemSwitchRecord, entitlementGrant, entitlementGrantInput, capabilityFilter, instant, meterReading, subjectRef, createConnectionInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, org as orgSchema, orgMembership, resolvedIdentity, identityMembership, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, delegatedReadRecord, ownerTransferAudit, memberChangeAudit, copyMarkAudit, subjectShredReceipt, connectorDispatchKind, callsOfManifestJson, outboundOfManifestJson, SCOPE_GATE_REASONS, substratError, redrainEventsInput, coverage, peerCoverage, peerGrantsEntry, peerSwitch, peerSwitchOutcome, verticalCaller, verticalResolution, verticalSlug, entityObjectRef, ATTACHMENT_BLOB_BINDING, blobStoreBindingName, lifecycleDelivery, scopeLifecycle, } from '@substrat-run/contracts';
|
|
5
5
|
import { normalizeHostname, toRouteTarget } from './route-resolver.js';
|
|
6
|
-
import { attachmentBlobKey, attachmentSha256, entitlementDenial, requiredEntitlementFor, foldMeterReading, parseValidationRecords, resolveScopeRecord, ulid, KEPT_COPY_REFUSAL, capabilityTokenHash, checkBecomeInput, plausibleSessionToken, foldModelUsage, backoffAt, resolveRetryPolicy, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, openSealed, ImpersonationRefused, assertSessionUsable, impersonationRowValues, mapImpersonationRow, newImpersonationSession, jobRunOf, runDueJobRuns, SYSTEM_DOOR_WAIT, startJobRun, attachmentTextJob, assertAttachmentExtractors, assertJobRegistrable,
|
|
7
|
-
import {
|
|
8
|
-
import { isOrangeToOrange, isUpgradeRequest, LIVE_MODE_HEADER, LIVE_PRINCIPAL_HEADER, LIVE_SCOPE_HEADER, LIVE_SUBSCRIBE_PATH, LIVE_TENANT_HEADER, } from './live-reads.js';
|
|
6
|
+
import { attachmentBlobKey, attachmentSha256, entitlementDenial, requiredEntitlementFor, foldMeterReading, parseValidationRecords, resolveScopeRecord, ulid, KEPT_COPY_REFUSAL, capabilityTokenHash, checkBecomeInput, plausibleSessionToken, foldModelUsage, backoffAt, resolveRetryPolicy, executorOutcomeOf, isDeliveryRefusal, refusalJournalText, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, openSealed, ImpersonationRefused, assertSessionUsable, impersonationRowValues, mapImpersonationRow, newImpersonationSession, jobRunOf, assertLeaseMs, runDueJobRuns, SYSTEM_DOOR_WAIT, startJobRun, attachmentTextJob, assertAttachmentExtractors, resolveAttachmentTextBounds, assertJobRegistrable, ATTACHMENT_TEXT_BACKFILL_JOB, ATTACHMENT_TEXT_JOB, kernelJobFor, attachmentTextBackfillJob, searchLimit, searchMatchExpression, globalFetch, assertRedrainWindow, platformRequestOf, undrainedEventsOf, inUnitMovesToAudit, staleCarryRevertRow, staleCarryReverts, recordedOffFromWire, peerSubjectRef, SWITCH_KINDS, reassertActionOf, reassertEntry, reassertOffRow, reassertOnRow, switchActionOf, switchAuditSubject, switchNotFoundMessage, recordWriteSuperseded, switchSubjectOf, switchSupersededMessage, withRecorded, systemSwitchedOffMessage, tenantSystemSwitchedOffMessage, CrossVerticalRegistry, importCursorSourceOf, exportBreaksOf, exportBreakRefusal, bindExportBreaksOf, bindExportBreakRefusal, collectPeers, peerSeats, connectorCallRecord, noopConnectorCallRecorder, recordConnectorCall, settleConnectionUse, splitManifestMigrations, unknownRoleError, assertRowLimit, assertRowOffset, emptyImportResult, INERT_SCOPE_REASON, lifecycleReceipt, lifecycleRefusal, isPrimaryScope, isPrimaryScopeRow, asyncInvocationId, asyncLinePass, memberAddedAudit, } from '@substrat-run/kernel';
|
|
7
|
+
import { attributedView } from '@substrat-run/kernel';
|
|
8
|
+
import { isOrangeToOrange, isUpgradeRequest, LIVE_MODE_HEADER, LIVE_PRINCIPAL_HEADER, LIVE_SCOPE_HEADER, LIVE_SUBSCRIBE_PATH, LIVE_TENANT_HEADER, LIVE_WITHIN_HEADER, encodeLiveWithin, liveWithinOf, } from './live-reads.js';
|
|
9
9
|
import { tenantStoreDatabaseName } from './d1.js';
|
|
10
10
|
import { blobStoreBucketName, r2TenantBlobStore } from './r2.js';
|
|
11
|
+
import { unwrapReply } from './do-reply.js';
|
|
11
12
|
/** DO row → contract shape. Never reads the secrets table — that is the split. */
|
|
12
13
|
const toConnection = (r) => connection.parse({
|
|
13
14
|
id: r.id,
|
|
@@ -26,12 +27,6 @@ const toConnection = (r) => connection.parse({
|
|
|
26
27
|
createdAt: r.created_at,
|
|
27
28
|
revokedAt: r.revoked_at,
|
|
28
29
|
});
|
|
29
|
-
/** Rethrow a capability attachment verb's failure, rebuilt with its code; else its value. */
|
|
30
|
-
function unwrapCapabilityReply(reply) {
|
|
31
|
-
if (reply.failure)
|
|
32
|
-
throw fromWireFailure(reply.failure);
|
|
33
|
-
return reply.value;
|
|
34
|
-
}
|
|
35
30
|
/**
|
|
36
31
|
* How far ahead of the clock a held lifecycle epoch may be and still be learned past (#1713). An
|
|
37
32
|
* epoch is the time a directory restore minted it, so a legitimate one is behind its minter's
|
|
@@ -92,24 +87,31 @@ function nullControlPlane() {
|
|
|
92
87
|
recordAdmin: noop,
|
|
93
88
|
recordAccess: noop,
|
|
94
89
|
tenantHoldsEntitlement: async () => true,
|
|
95
|
-
// #1674: the
|
|
90
|
+
// #1674, #2029: the switches' record is a DIRECTORY store, and a CP-less host has no
|
|
96
91
|
// directory — the shared control plane that delegates here keeps it. Same posture as
|
|
97
92
|
// `recordAdmin`: nothing to write, nothing recorded to read back.
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
93
|
+
recordSwitchedOff: async () => null,
|
|
94
|
+
recordSwitchedOn: async () => null,
|
|
95
|
+
restoreSwitchRecord: noop,
|
|
101
96
|
listSystemSwitches: async () => [],
|
|
102
|
-
|
|
103
|
-
|
|
97
|
+
switchRecordsOf: async () => [],
|
|
98
|
+
switchRecordStatesOf: async () => [],
|
|
99
|
+
switchesOwedOf: async () => [],
|
|
100
|
+
clearSwitchOwed: noop,
|
|
104
101
|
// #1823: no tenant tuples here either — the platform reads them and sends `tenantHeld`.
|
|
105
|
-
|
|
102
|
+
tenantHeldOf: async () => [],
|
|
103
|
+
};
|
|
104
|
+
const unavailable = (method) => new Error(`control plane unavailable: '${method}' — this host is scope-local / CP-less ` +
|
|
105
|
+
`(docs/architecture/scope-local-permissions.md, Phase 3)`);
|
|
106
|
+
// A directory write through `reply` (#113) is named for the write it carries, not for `reply`.
|
|
107
|
+
passthrough.reply = async (method) => {
|
|
108
|
+
throw unavailable(String(method));
|
|
106
109
|
};
|
|
107
110
|
return new Proxy({}, {
|
|
108
111
|
get: (_t, prop) => typeof prop === 'string' && prop in passthrough
|
|
109
112
|
? passthrough[prop]
|
|
110
113
|
: async () => {
|
|
111
|
-
throw
|
|
112
|
-
`(docs/architecture/scope-local-permissions.md, Phase 3)`);
|
|
114
|
+
throw unavailable(String(prop));
|
|
113
115
|
},
|
|
114
116
|
});
|
|
115
117
|
}
|
|
@@ -151,19 +153,34 @@ export const SWITCH_HOLD_EXTRA_WAITS = 3;
|
|
|
151
153
|
*/
|
|
152
154
|
export const SWITCH_HOLD_PENDING_MAX_MS = 5 * 60_000;
|
|
153
155
|
/**
|
|
154
|
-
*
|
|
155
|
-
*
|
|
156
|
-
*
|
|
156
|
+
* A move the deployment serving the scope applied, or may have, without attesting what the
|
|
157
|
+
* platform needs from it (#2045 Codex r3): the switch fence (`fenced`), or for a tenant-held
|
|
158
|
+
* module's OFF the evaluator that denies a tenant-level grant (`deniesTenantGrants`, #1823).
|
|
159
|
+
*
|
|
160
|
+
* The preflight (`switchTarget`'s `attestFence`) tells a deployment built before the fence apart
|
|
161
|
+
* before anything is recorded or moved. On such a deployment only an ON is refused: the race the
|
|
162
|
+
* fence closes is an older OFF landing after a newer ON, and with every ON refused there is no
|
|
163
|
+
* newer ON to land after, so an OFF goes through unfenced (#2045 follow-up). A kill switch that
|
|
164
|
+
* cannot be pulled until every vertical is re-pushed is the outage the fence must never cause.
|
|
165
|
+
* An unfenced OFF keeps its write-ahead owed mark, so the first re-assert after the vertical is
|
|
166
|
+
* redeployed moves the scope again under the record's fence.
|
|
167
|
+
*
|
|
168
|
+
* This error fires when a deployment that attested the fence answers a move without it — a
|
|
169
|
+
* rollback to an old build between the preflight and the move — or when an ON reaches a
|
|
170
|
+
* deployment without the fence. It is NOT compensated with an opposite move: an unfenced
|
|
171
|
+
* compensation can undo a newer call's switch (Codex r3, finding 2). The call's record and its
|
|
172
|
+
* write-ahead owed mark stay, and the call is refused.
|
|
157
173
|
*/
|
|
158
|
-
class
|
|
159
|
-
outcome;
|
|
160
|
-
constructor(scopeId, moduleId, outcome) {
|
|
161
|
-
super(`the deployment serving scope ${scopeId} predates the kill switch's tenant-grant denial (#1823): ` +
|
|
162
|
-
`module '${moduleId}' holds a tenant-level grant that its OFF would leave authorizing there. ` +
|
|
163
|
-
`Redeploy the vertical, then retry.`);
|
|
164
|
-
this.outcome = outcome;
|
|
165
|
-
}
|
|
174
|
+
class UnattestedSwitch extends Error {
|
|
166
175
|
}
|
|
176
|
+
/** A fenced move answered without `fenced`: the deployment predates the fence. */
|
|
177
|
+
const unfencedMessage = (scopeId) => `the deployment serving scope ${scopeId} predates the switch fence (#2045): it cannot refuse an older ` +
|
|
178
|
+
`switch call's OFF landing after this ON, so no switch is turned on there. Switching off still works. ` +
|
|
179
|
+
`Redeploy the vertical, then retry.`;
|
|
180
|
+
/** A tenant-held module's OFF answered without `deniesTenantGrants` (#1823). */
|
|
181
|
+
const undeniedMessage = (scopeId, moduleId) => `the deployment serving scope ${scopeId} predates the kill switch's tenant-grant denial (#1823): ` +
|
|
182
|
+
`module '${moduleId}' holds a tenant-level grant that its OFF would leave authorizing there. ` +
|
|
183
|
+
`Redeploy the vertical, then retry.`;
|
|
167
184
|
/**
|
|
168
185
|
* The attachment bucket a host resolves when it was handed no resolver (#1995): the
|
|
169
186
|
* platform's per-tenant binding, read off this script's own env, named by the one shared
|
|
@@ -172,6 +189,49 @@ class UnattestedSwitchOff extends Error {
|
|
|
172
189
|
function ambientAttachmentBucket(tenantId) {
|
|
173
190
|
return ambientEnv[blobStoreBindingName(ATTACHMENT_BLOB_BINDING, tenantId)] ?? null;
|
|
174
191
|
}
|
|
192
|
+
/**
|
|
193
|
+
* #2029: scope instances whose door has read the rewind hold and found NO claim on the scope at
|
|
194
|
+
* all, so their doors skip the hold read for the rest of the instance's life. Isolate-wide, because
|
|
195
|
+
* a host lives for one request: without it, every peer call would read the deployment's one hold
|
|
196
|
+
* object — an extra hop, on a single object, on the hot path of every scope never rewound.
|
|
197
|
+
*
|
|
198
|
+
* Safe because claims are only ever added for storage a LATER instance serves. A rewind writes its
|
|
199
|
+
* claim before it arms, and arming restarts the scope, so its rewound storage is always served by a
|
|
200
|
+
* new instance with a new id, which reads the hold afresh (`SWITCH_HOLD_SETTLE_MS` puts every row of
|
|
201
|
+
* the claim before that instance existed). A join (`switchHoldJoin`) goes into a rewind's claim too,
|
|
202
|
+
* so into that same later instance. An instance that read no claim therefore never gains one; claims
|
|
203
|
+
* only leave it, by a switch move. Only a GOOD read is remembered: a read that failed open is not.
|
|
204
|
+
* Bounded: past `CLEAR_SCOPE_INSTANCES_MAX` it is emptied, which costs a re-read, nothing else.
|
|
205
|
+
*/
|
|
206
|
+
const CLEAR_SCOPE_INSTANCES = new Set();
|
|
207
|
+
const CLEAR_SCOPE_INSTANCES_MAX = 10_000;
|
|
208
|
+
const rememberClearInstance = (key) => {
|
|
209
|
+
if (CLEAR_SCOPE_INSTANCES.size >= CLEAR_SCOPE_INSTANCES_MAX)
|
|
210
|
+
CLEAR_SCOPE_INSTANCES.clear();
|
|
211
|
+
CLEAR_SCOPE_INSTANCES.add(key);
|
|
212
|
+
};
|
|
213
|
+
/**
|
|
214
|
+
* #2045: a carry's fences, by tuple subject (`RecordedOffCarry.fences`): for each recorded-off
|
|
215
|
+
* subject, the operation id of the call the record holds.
|
|
216
|
+
*/
|
|
217
|
+
const switchFencesCarried = (kinds) => {
|
|
218
|
+
const out = {};
|
|
219
|
+
for (const [kind, { keys, fences }] of kinds) {
|
|
220
|
+
for (const key of keys) {
|
|
221
|
+
const fence = fences.get(key);
|
|
222
|
+
if (fence !== undefined)
|
|
223
|
+
out[switchSubjectOf(kind, key)] = fence;
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
return out;
|
|
227
|
+
};
|
|
228
|
+
/** #2029: how a door's refusals and a re-assert's name each kind of subject. */
|
|
229
|
+
const DOOR_WORDS = {
|
|
230
|
+
system: { what: (key) => `module '${key}'`, door: 'system door', switchName: 'schedule', switched: 'schedules' },
|
|
231
|
+
peer: { what: (key) => `peer vertical '${key}'`, door: 'peer door', switchName: 'peer', switched: 'peers' },
|
|
232
|
+
};
|
|
233
|
+
/** #2029: the rewind hold's key for one subject — a module's id, bare as #1819 stored it, or `vertical:<slug>`. */
|
|
234
|
+
const holdKeyOf = (kind, key) => (kind === 'system' ? key : peerSubjectRef(key));
|
|
175
235
|
export class CloudflareScopeHost {
|
|
176
236
|
admin;
|
|
177
237
|
scopeNs;
|
|
@@ -215,6 +275,7 @@ export class CloudflareScopeHost {
|
|
|
215
275
|
attachmentBuckets;
|
|
216
276
|
/** The parsers attachment text is extracted with (K-43); the host's own, never imported here. */
|
|
217
277
|
attachmentExtractors;
|
|
278
|
+
attachmentTextBounds;
|
|
218
279
|
executors = new Map();
|
|
219
280
|
/**
|
|
220
281
|
* `<moduleId>/<job>` → the pass body and its default step policy (#1577). Host
|
|
@@ -225,10 +286,10 @@ export class CloudflareScopeHost {
|
|
|
225
286
|
*/
|
|
226
287
|
jobs = new Map();
|
|
227
288
|
/**
|
|
228
|
-
* The event
|
|
229
|
-
*
|
|
230
|
-
*
|
|
231
|
-
*
|
|
289
|
+
* The event an executor is effecting, stamped onto the admin rows it writes (K-22). Like
|
|
290
|
+
* `onBehalfOf`, only ever set on a VIEW (`causedByView`, or `attributed` with a
|
|
291
|
+
* `causedBy`), never on the host: a field set around the handler's `await` stamped every
|
|
292
|
+
* admin call the host served meanwhile — a staff call included — with that event (#2055).
|
|
232
293
|
*/
|
|
233
294
|
causedBy = null;
|
|
234
295
|
/**
|
|
@@ -282,6 +343,7 @@ export class CloudflareScopeHost {
|
|
|
282
343
|
this.attachmentBuckets = options.attachmentBuckets ?? ambientAttachmentBucket;
|
|
283
344
|
assertAttachmentExtractors(options.attachmentExtractors ?? []);
|
|
284
345
|
this.attachmentExtractors = options.attachmentExtractors ?? [];
|
|
346
|
+
this.attachmentTextBounds = resolveAttachmentTextBounds(options.attachmentTextBounds);
|
|
285
347
|
this.fetchImpl = options.fetch ?? globalFetch;
|
|
286
348
|
this.connectorCalls = options.connectorCalls ?? noopConnectorCallRecorder;
|
|
287
349
|
this.scopeLocalPermissions = options.scopeLocalPermissions ?? false;
|
|
@@ -389,6 +451,22 @@ export class CloudflareScopeHost {
|
|
|
389
451
|
},
|
|
390
452
|
};
|
|
391
453
|
}
|
|
454
|
+
/**
|
|
455
|
+
* #1184: the scope reads an executor's handler may make. Plain RPCs to the ScopeDO: the
|
|
456
|
+
* handler runs here on the coordinator, after the DO's own task has returned.
|
|
457
|
+
*/
|
|
458
|
+
executorScope(tenantId, scopeId) {
|
|
459
|
+
const stub = this.scopeStub(scopeId);
|
|
460
|
+
return {
|
|
461
|
+
history: async (entity, page) => stub.entityHistory({ entityType: entity.entityType, entityId: entity.entityId, limit: page?.limit, cursor: page?.cursor }),
|
|
462
|
+
covers: async (principal, roleKey, level) => {
|
|
463
|
+
const bound = await stub.canAssignFor(tenantId, scopeId, principalId.parse(principal), roleKey, level === 'tenant');
|
|
464
|
+
if (!bound)
|
|
465
|
+
throw unknownRoleError(roleKey);
|
|
466
|
+
return coverage.parse(bound);
|
|
467
|
+
},
|
|
468
|
+
};
|
|
469
|
+
}
|
|
392
470
|
/**
|
|
393
471
|
* Drain this scope's outbox into the registered executors (K-22 §4.2).
|
|
394
472
|
*
|
|
@@ -412,7 +490,9 @@ export class CloudflareScopeHost {
|
|
|
412
490
|
* own `invocationId` field reads null — an ambient read there would record "no
|
|
413
491
|
* call" for every attempt an operation's own tail made.
|
|
414
492
|
*/
|
|
415
|
-
invocationId
|
|
493
|
+
invocationId,
|
|
494
|
+
/** #1184: what each attempt did, for the emitting call's `onExecutorOutcomes`. */
|
|
495
|
+
outcomes) {
|
|
416
496
|
const report = {
|
|
417
497
|
attempted: 0,
|
|
418
498
|
delivered: 0,
|
|
@@ -426,27 +506,22 @@ export class CloudflareScopeHost {
|
|
|
426
506
|
// attempt runs — the DO only journals it.
|
|
427
507
|
const lines = asyncLinePass();
|
|
428
508
|
try {
|
|
429
|
-
await this.drainExecutorsPass(tenantId, scopeId, invocationId, report, lines);
|
|
509
|
+
await this.drainExecutorsPass(tenantId, scopeId, invocationId, report, lines, outcomes);
|
|
430
510
|
}
|
|
431
511
|
finally {
|
|
432
512
|
lines.end();
|
|
433
513
|
}
|
|
434
514
|
return report;
|
|
435
515
|
}
|
|
436
|
-
async drainExecutorsPass(tenantId, scopeId, invocationId, report, lines) {
|
|
516
|
+
async drainExecutorsPass(tenantId, scopeId, invocationId, report, lines, outcomes) {
|
|
437
517
|
const stub = this.scopeStub(scopeId);
|
|
518
|
+
const scope = this.executorScope(tenantId, scopeId);
|
|
438
519
|
// #2005: a non-primary scope (a fork, a snapshot, a preview of either kind) causes no
|
|
439
520
|
// outbound effects, so its deliveries are journaled terminal with the reason and no
|
|
440
|
-
// handler runs. Asked
|
|
441
|
-
//
|
|
442
|
-
// `is_copy`, #2009, set on the directory's word) — a preview and a snapshot reach a hosted
|
|
443
|
-
// vertical as restores the platform marks.
|
|
444
|
-
// Asked on the first due event, once per pass: most passes have none, and the directory
|
|
445
|
-
// is one global object.
|
|
521
|
+
// handler runs (`isInertScope`). Asked on the first due event, once per pass: most passes
|
|
522
|
+
// have none, and the directory is one global object.
|
|
446
523
|
let inert;
|
|
447
|
-
const isInert = () => (inert ??= this.
|
|
448
|
-
? stub.isCopy()
|
|
449
|
-
: this.cp.getScopeRecord(tenantId, scopeId).then((row) => !isPrimaryScopeRow(row)));
|
|
524
|
+
const isInert = () => (inert ??= this.isInertScope(tenantId, scopeId));
|
|
450
525
|
// #1713: a CP-less scope its lifecycle holds attempts nothing and journals nothing, so every
|
|
451
526
|
// due delivery stays due for the first pass after it is live again. Asked on the first due
|
|
452
527
|
// event, like `isInert`. Every caller passed `assertLive` already, so this is for the scope
|
|
@@ -477,6 +552,9 @@ export class CloudflareScopeHost {
|
|
|
477
552
|
attempt,
|
|
478
553
|
startedAt: Date.now(),
|
|
479
554
|
});
|
|
555
|
+
const outcomeOf = (event, outcome, error) => {
|
|
556
|
+
outcomes?.push(executorOutcomeOf(id, event, outcome, error));
|
|
557
|
+
};
|
|
480
558
|
for (const bad of undecodable) {
|
|
481
559
|
report.attempted += 1;
|
|
482
560
|
const attempt = await stub.recordExecutorAttempt(bad.eventId, deliveryId, bad.error, null, invocationId);
|
|
@@ -490,11 +568,13 @@ export class CloudflareScopeHost {
|
|
|
490
568
|
// No next attempt, like an undecodable row: a scope does not become primary.
|
|
491
569
|
const attempt = await stub.recordExecutorAttempt(event.id, deliveryId, INERT_SCOPE_REASON, null, invocationId);
|
|
492
570
|
report.inert = (report.inert ?? 0) + 1;
|
|
571
|
+
outcomeOf(event, 'inert', INERT_SCOPE_REASON);
|
|
493
572
|
// Its own outcome, never `delivered`: no handler ran (#2005).
|
|
494
573
|
lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'inert' });
|
|
495
574
|
continue;
|
|
496
575
|
}
|
|
497
|
-
|
|
576
|
+
// #2055: the handler writes through a view bound to its event, never through the host.
|
|
577
|
+
const caused = this.causedByView(event.id);
|
|
498
578
|
try {
|
|
499
579
|
if (executor.kind === 'connector' && this.cpLess) {
|
|
500
580
|
// #574 phase 3: this host cannot run a connector — no connection
|
|
@@ -507,20 +587,34 @@ export class CloudflareScopeHost {
|
|
|
507
587
|
// the at-least-once residue, as it already must in-process.
|
|
508
588
|
await stub.routeExecutorEventToPlatform(event.id, deliveryId, connectorDispatchKind(executor.provider), JSON.stringify({ executorId: id, event }), JSON.stringify({ system: 'connector-dispatch' }), invocationId);
|
|
509
589
|
report.routedToPlatform += 1;
|
|
590
|
+
outcomeOf(event, 'routed');
|
|
510
591
|
// Handed on, not run here: the platform's drain owns its attempts from now on.
|
|
511
592
|
lines.write({ ...unitOf(event.id, 1), startedAt, outcome: 'routed' });
|
|
512
593
|
}
|
|
513
594
|
else if (executor.kind === 'connector') {
|
|
514
|
-
await executor.handler(await
|
|
595
|
+
await executor.handler(await caused.connectorContext(tenantId, scopeId, executor.timeoutMs, event.id), event);
|
|
515
596
|
const attempt = await stub.recordExecutorAttempt(event.id, deliveryId, null, null, invocationId);
|
|
516
597
|
report.delivered += 1;
|
|
598
|
+
outcomeOf(event, 'delivered');
|
|
517
599
|
lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'delivered' });
|
|
518
600
|
}
|
|
519
601
|
else {
|
|
520
|
-
await executor.handler(
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
602
|
+
const result = await executor.handler(caused.admin, event, scope);
|
|
603
|
+
// #1184: a refusal is the handler's own terminal decision — journaled with its
|
|
604
|
+
// reason, no next attempt, listed by `executorDeadLetters` beside an exhausted one.
|
|
605
|
+
const refused = isDeliveryRefusal(result) ? result : null;
|
|
606
|
+
const text = refused && refusalJournalText(refused);
|
|
607
|
+
const attempt = await stub.recordExecutorAttempt(event.id, deliveryId, text, null, invocationId);
|
|
608
|
+
if (refused) {
|
|
609
|
+
report.deadLettered += 1;
|
|
610
|
+
outcomeOf(event, 'refused', refused.reason);
|
|
611
|
+
lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'dead-lettered', error: text });
|
|
612
|
+
}
|
|
613
|
+
else {
|
|
614
|
+
report.delivered += 1;
|
|
615
|
+
outcomeOf(event, 'delivered');
|
|
616
|
+
lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'delivered' });
|
|
617
|
+
}
|
|
524
618
|
}
|
|
525
619
|
}
|
|
526
620
|
catch (err) {
|
|
@@ -535,6 +629,7 @@ export class CloudflareScopeHost {
|
|
|
535
629
|
report.deadLettered += 1;
|
|
536
630
|
else
|
|
537
631
|
report.retrying += 1;
|
|
632
|
+
outcomeOf(event, exhausted ? 'dead-lettered' : 'retrying', err);
|
|
538
633
|
lines.write({
|
|
539
634
|
...unitOf(event.id, attempts),
|
|
540
635
|
startedAt,
|
|
@@ -542,9 +637,6 @@ export class CloudflareScopeHost {
|
|
|
542
637
|
error: err,
|
|
543
638
|
});
|
|
544
639
|
}
|
|
545
|
-
finally {
|
|
546
|
-
this.causedBy = null;
|
|
547
|
-
}
|
|
548
640
|
}
|
|
549
641
|
}
|
|
550
642
|
}
|
|
@@ -558,13 +650,14 @@ export class CloudflareScopeHost {
|
|
|
558
650
|
// emitting operation's own tail made.
|
|
559
651
|
return this.drainExecutors(tenantId, scopeId, null);
|
|
560
652
|
}
|
|
561
|
-
registerJob(moduleId, name, handler, retry) {
|
|
653
|
+
registerJob(moduleId, name, handler, retry, options) {
|
|
562
654
|
// #1575: the kernel's own jobs are dispatched before this registry is read.
|
|
563
655
|
assertJobRegistrable(moduleId, name);
|
|
656
|
+
assertLeaseMs(options?.leaseMs);
|
|
564
657
|
const key = `${moduleId}/${name}`;
|
|
565
658
|
if (this.jobs.has(key))
|
|
566
659
|
throw new Error(`job '${key}' is already registered`);
|
|
567
|
-
this.jobs.set(key, { handler, retry });
|
|
660
|
+
this.jobs.set(key, { handler, retry, leaseMs: options?.leaseMs });
|
|
568
661
|
}
|
|
569
662
|
/**
|
|
570
663
|
* D-14's DURABLE driver: the run store is RPC to the scope DO, and nothing else.
|
|
@@ -582,13 +675,15 @@ export class CloudflareScopeHost {
|
|
|
582
675
|
const stub = this.scopeStub(scopeId);
|
|
583
676
|
return {
|
|
584
677
|
startOrJoin: (key, row) => stub.jobRunStartOrJoin(key.moduleId, key.job, key.instance, row),
|
|
585
|
-
get: (id) => stub.jobRunById(id),
|
|
586
678
|
dueKeys: (now, max) => stub.jobRunsDueKeys(now, max),
|
|
679
|
+
claim: (id, owner, leaseMs) => stub.jobRunClaim(id, owner, leaseMs),
|
|
680
|
+
begin: (id, owner, marginMs) => stub.jobRunBegin(id, owner, marginMs),
|
|
681
|
+
miss: (id, owner, note) => stub.jobRunMiss(id, owner, note),
|
|
587
682
|
list: (filter) => stub.jobRunList(filter),
|
|
588
|
-
patch: (id, patch) => stub.jobRunPatch(id, patch),
|
|
589
|
-
commitPass: (id, patch) => stub.jobCommitPass(id, patch),
|
|
590
|
-
|
|
591
|
-
recordStep: (runId, name, result, attempts, lastError, at) => stub.jobStepRecord(runId, name, result, attempts, lastError, at),
|
|
683
|
+
patch: (id, patch, owner) => stub.jobRunPatch(id, patch, owner),
|
|
684
|
+
commitPass: (id, patch, owner) => stub.jobCommitPass(id, patch, owner),
|
|
685
|
+
beginStep: (runId, name, owner, leaseMs) => stub.jobStepBegin(runId, name, owner, leaseMs),
|
|
686
|
+
recordStep: (runId, name, result, attempts, lastError, at, owner, leaseMs) => stub.jobStepRecord(runId, name, result, attempts, lastError, at, owner, leaseMs),
|
|
592
687
|
};
|
|
593
688
|
}
|
|
594
689
|
/**
|
|
@@ -606,7 +701,7 @@ export class CloudflareScopeHost {
|
|
|
606
701
|
write: (attachmentId, outcome) => stub.attachmentTextRecord(attachmentId, outcome),
|
|
607
702
|
},
|
|
608
703
|
// K-43: the host's parsers, handed in — this adapter imports none.
|
|
609
|
-
this.attachmentExtractors);
|
|
704
|
+
this.attachmentExtractors, this.attachmentTextBounds);
|
|
610
705
|
}
|
|
611
706
|
async startJobRun(tenantId, scopeId, input) {
|
|
612
707
|
await this.assertLive(tenantId, scopeId);
|
|
@@ -618,12 +713,18 @@ export class CloudflareScopeHost {
|
|
|
618
713
|
// advance, and an archived one's never move again.
|
|
619
714
|
await this.assertLive(tenantId, scopeId);
|
|
620
715
|
await this.migrateAndRecord(scopeId);
|
|
621
|
-
// #1575: the kernel's extraction
|
|
622
|
-
// deployment registers
|
|
623
|
-
|
|
716
|
+
// #1575: the kernel's extraction jobs are this host's own, bound to this scope — no
|
|
717
|
+
// deployment registers them, and none can shadow them. The backfill starts here, on
|
|
718
|
+
// the drive, so a scope's first request never pays for attachments that predate it.
|
|
719
|
+
const stub = this.scopeStub(scopeId);
|
|
720
|
+
await stub.attachmentTextBackfillStart();
|
|
721
|
+
const kernelJobs = {
|
|
722
|
+
[ATTACHMENT_TEXT_JOB]: this.attachmentTextHandler(tenantId, scopeId),
|
|
723
|
+
[ATTACHMENT_TEXT_BACKFILL_JOB]: attachmentTextBackfillJob({ queueBatch: (after) => stub.attachmentTextBackfillBatch(after) }),
|
|
724
|
+
};
|
|
624
725
|
return runDueJobRuns({
|
|
625
726
|
store: this.jobStore(scopeId),
|
|
626
|
-
handlerFor: (run) =>
|
|
727
|
+
handlerFor: (run) => kernelJobFor(run, kernelJobs) ?? this.jobs.get(`${run.module_id}/${run.job}`),
|
|
627
728
|
now: () => new Date().toISOString(),
|
|
628
729
|
// #1834: the door is opened FOR this pass, so its "not now" is tied to this pass alone.
|
|
629
730
|
openScope: async (run, pass) => this.buildStub(tenantId, scopeId, undefined, undefined, await this.openSystemDoor(run.module_id, tenantId, scopeId, undefined, pass)),
|
|
@@ -645,13 +746,9 @@ export class CloudflareScopeHost {
|
|
|
645
746
|
// ran it. On a CP-less host `connectorContext` throws from the null control plane:
|
|
646
747
|
// fail closed, exactly the hole routing exists to avoid.
|
|
647
748
|
await this.assertLive(tenantId, scopeId);
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
}
|
|
652
|
-
finally {
|
|
653
|
-
this.causedBy = null;
|
|
654
|
-
}
|
|
749
|
+
// Built on a view bound to the event, so its admin rows carry it and nothing else the
|
|
750
|
+
// host serves does (#2055).
|
|
751
|
+
await handler(await this.causedByView(event.id).connectorContext(tenantId, scopeId, options?.timeoutMs ?? 30_000, event.id), event);
|
|
655
752
|
}
|
|
656
753
|
async executorDeadLetters(tenantId, scopeId) {
|
|
657
754
|
await this.validateScopeAccess(tenantId, scopeId);
|
|
@@ -710,7 +807,7 @@ export class CloudflareScopeHost {
|
|
|
710
807
|
if (!rec)
|
|
711
808
|
throw unknownScopeForTenant(tenantId, scopeId);
|
|
712
809
|
if (rec.status !== 'active' && rec.status !== 'provisioning') {
|
|
713
|
-
throw
|
|
810
|
+
throw substratError('conflict', `scope not migratable (status: ${rec.status}): ${scopeId}`);
|
|
714
811
|
}
|
|
715
812
|
const stub = this.scopeStub(scopeId);
|
|
716
813
|
try {
|
|
@@ -752,11 +849,11 @@ export class CloudflareScopeHost {
|
|
|
752
849
|
return this.scopeStub(scopeId).introspectTables();
|
|
753
850
|
}
|
|
754
851
|
async introspectScopeTable(scopeId, input) {
|
|
755
|
-
return this.scopeStub(scopeId).
|
|
852
|
+
return unwrapReply(await this.scopeStub(scopeId).introspectTableReply(input.table, input.limit, input.offset));
|
|
756
853
|
}
|
|
757
854
|
/** The SQL console's CP-less path (#219) — same trust line as the pair above. */
|
|
758
855
|
async introspectScopeQuery(scopeId, input) {
|
|
759
|
-
return this.scopeStub(scopeId).
|
|
856
|
+
return unwrapReply(await this.scopeStub(scopeId).introspectQueryReply(input.sql));
|
|
760
857
|
}
|
|
761
858
|
/**
|
|
762
859
|
* The K-35 denial log's CP-less path (#867) — same trust line as the reads above.
|
|
@@ -853,10 +950,9 @@ export class CloudflareScopeHost {
|
|
|
853
950
|
opts) {
|
|
854
951
|
if (opts?.markCopy)
|
|
855
952
|
assertCopyLineage(opts.markCopy);
|
|
953
|
+
const carry = opts ? recordedOffFromWire(opts) : undefined;
|
|
856
954
|
const load = {
|
|
857
|
-
switchOff:
|
|
858
|
-
? { moduleIds: opts.switchedOff, at: new Date().toISOString(), tenantHeld: opts.tenantHeld }
|
|
859
|
-
: undefined,
|
|
955
|
+
switchOff: carry ? { at: new Date().toISOString(), ...carry } : undefined,
|
|
860
956
|
sourceScopeId: opts?.sourceScopeId,
|
|
861
957
|
exact: opts?.exact,
|
|
862
958
|
loadStamp: opts?.loadStamp,
|
|
@@ -870,7 +966,7 @@ export class CloudflareScopeHost {
|
|
|
870
966
|
...(opts?.expect ? { expect: opts.expect } : {}),
|
|
871
967
|
});
|
|
872
968
|
if (out.refused === false) {
|
|
873
|
-
return { tables: tables.length, ...(
|
|
969
|
+
return { tables: tables.length, ...(carry ? { switchedOff: out.switchedOff } : {}) };
|
|
874
970
|
}
|
|
875
971
|
if (out.refused === 'kept')
|
|
876
972
|
throw substratError('conflict', KEPT_COPY_REFUSAL);
|
|
@@ -993,7 +1089,7 @@ export class CloudflareScopeHost {
|
|
|
993
1089
|
* safe; scope-level tuples (the restored grants) are never touched.
|
|
994
1090
|
*/
|
|
995
1091
|
async projectRolesLocal(tenantId, scopeId, roles) {
|
|
996
|
-
await this.scopeStub(scopeId).
|
|
1092
|
+
unwrapReply(await this.scopeStub(scopeId).applyProjectionReply(tenantId, roles.map((r) => ({ role_key: r.key, permissions: JSON.stringify(r.permissions), source: r.source })), []));
|
|
997
1093
|
}
|
|
998
1094
|
/**
|
|
999
1095
|
* Wipe one scope DO's storage in THIS deployment — the reap half of an orchestrated
|
|
@@ -1045,6 +1141,10 @@ export class CloudflareScopeHost {
|
|
|
1045
1141
|
async lifecycleFlowLocal(scopeId, input) {
|
|
1046
1142
|
return this.scopeStub(scopeId).lifecycleFlow(input);
|
|
1047
1143
|
}
|
|
1144
|
+
/** Business volumes per bucket (#1750) on this host's own scope — the vertical-host read. */
|
|
1145
|
+
async operationSeriesLocal(scopeId, input) {
|
|
1146
|
+
return this.scopeStub(scopeId).operationSeries(input);
|
|
1147
|
+
}
|
|
1048
1148
|
/** One event's causal chain (#1237) on this host's own scope — the vertical-host read. */
|
|
1049
1149
|
async eventCauseLocal(scopeId, input) {
|
|
1050
1150
|
return this.scopeStub(scopeId).eventCause(input);
|
|
@@ -1197,7 +1297,7 @@ export class CloudflareScopeHost {
|
|
|
1197
1297
|
const record = resolveScopeRecord(input);
|
|
1198
1298
|
// Fail-closed tenant gate throws out of the awaited cp call BEFORE migrate
|
|
1199
1299
|
// or audit, so a rejected provision creates nothing and writes no audit row.
|
|
1200
|
-
const created = await this.
|
|
1300
|
+
const created = await this.directory('provisionScope', input.tenantId, input.scopeId, record, new Date().toISOString());
|
|
1201
1301
|
// Instantiate the scope DO and trigger its lazy migration.
|
|
1202
1302
|
await this.migrateAndRecord(input.scopeId);
|
|
1203
1303
|
// Scope-local permissions: a freshly-provisioned scope evaluates from its own
|
|
@@ -1238,17 +1338,9 @@ export class CloudflareScopeHost {
|
|
|
1238
1338
|
// #1742: in the SAME unit as the seat, so no sweep can land between them; the re-assert
|
|
1239
1339
|
// after it then finds them off, and audits what the unit moved.
|
|
1240
1340
|
const ownStore = this.cpLess || record.vertical === null;
|
|
1241
|
-
const recordedOff = ownStore ? await this.cp.switchedOffModulesOf(input.tenantId, input.scopeId) : [];
|
|
1242
1341
|
const seatAt = new Date().toISOString();
|
|
1243
|
-
const
|
|
1244
|
-
|
|
1245
|
-
scopeId: input.scopeId,
|
|
1246
|
-
moduleIds: recordedOff,
|
|
1247
|
-
at: seatAt,
|
|
1248
|
-
// #1823: a recorded-off module held only by a tenant grant goes back off too.
|
|
1249
|
-
tenantHeld: await this.cp.tenantHeldSystemModules(input.tenantId, recordedOff, seatAt),
|
|
1250
|
-
}
|
|
1251
|
-
: undefined);
|
|
1342
|
+
const carry = ownStore ? await this.recordedOffCarry(input.tenantId, input.scopeId, seatAt) : undefined;
|
|
1343
|
+
const switchedOff = await this.scopeStub(input.scopeId).seatTuples(seats, carry ? { scopeId: input.scopeId, at: seatAt, ...carry } : undefined);
|
|
1252
1344
|
if (ownStore) {
|
|
1253
1345
|
await this.admin.reassertSystemSwitches(actor, { tenantId: input.tenantId, scopeId: input.scopeId }, { appliedInUnit: switchedOff });
|
|
1254
1346
|
}
|
|
@@ -1270,10 +1362,10 @@ export class CloudflareScopeHost {
|
|
|
1270
1362
|
// inside the DO's ledger write, which is the serialization point that actually holds.
|
|
1271
1363
|
const tenant = await this.cp.getTenant(input.tenantId);
|
|
1272
1364
|
if (!tenant) {
|
|
1273
|
-
throw
|
|
1365
|
+
throw substratError('not_found', `cannot provision tenant store under unknown tenant: ${input.tenantId}`);
|
|
1274
1366
|
}
|
|
1275
1367
|
if (tenant.status !== 'active') {
|
|
1276
|
-
throw
|
|
1368
|
+
throw substratError('conflict', `cannot provision tenant store under non-active tenant (status: ${tenant.status}): ${input.tenantId}`);
|
|
1277
1369
|
}
|
|
1278
1370
|
// Idempotent on (tenant, vertical, binding): a retried provision re-resolves the SAME
|
|
1279
1371
|
// store rather than minting a second database (the K-31 ready-gate retries the whole
|
|
@@ -1288,7 +1380,7 @@ export class CloudflareScopeHost {
|
|
|
1288
1380
|
// the source of truth, carrying the D1 database_id Cloudflare assigned as the ref.
|
|
1289
1381
|
const name = await tenantStoreDatabaseName(input.tenantId, input.vertical, input.binding);
|
|
1290
1382
|
const ref = await d1.create(name);
|
|
1291
|
-
const stored = await this.
|
|
1383
|
+
const stored = await this.directory('putTenantStore', {
|
|
1292
1384
|
tenantId: input.tenantId,
|
|
1293
1385
|
vertical: input.vertical,
|
|
1294
1386
|
binding: input.binding,
|
|
@@ -1339,17 +1431,17 @@ export class CloudflareScopeHost {
|
|
|
1339
1431
|
const r2 = this.requireBlobStores(`provisionBlobStore(tenant=${input.tenantId} vertical=${input.vertical} binding=${input.binding})`);
|
|
1340
1432
|
const tenant = await this.cp.getTenant(input.tenantId);
|
|
1341
1433
|
if (!tenant) {
|
|
1342
|
-
throw
|
|
1434
|
+
throw substratError('not_found', `cannot provision blob store under unknown tenant: ${input.tenantId}`);
|
|
1343
1435
|
}
|
|
1344
1436
|
if (tenant.status !== 'active') {
|
|
1345
|
-
throw
|
|
1437
|
+
throw substratError('conflict', `cannot provision blob store under non-active tenant (status: ${tenant.status}): ${input.tenantId}`);
|
|
1346
1438
|
}
|
|
1347
1439
|
const existing = await this.cp.getBlobStore(input.tenantId, input.vertical, input.binding);
|
|
1348
1440
|
if (existing)
|
|
1349
1441
|
return { binding: input.binding, kind: 'blob', ref: existing.ref };
|
|
1350
1442
|
const name = await blobStoreBucketName(input.tenantId, input.vertical, input.binding);
|
|
1351
1443
|
const ref = await r2.create(name);
|
|
1352
|
-
const stored = await this.
|
|
1444
|
+
const stored = await this.directory('putBlobStore', {
|
|
1353
1445
|
tenantId: input.tenantId,
|
|
1354
1446
|
vertical: input.vertical,
|
|
1355
1447
|
binding: input.binding,
|
|
@@ -1488,22 +1580,22 @@ export class CloudflareScopeHost {
|
|
|
1488
1580
|
// the DO only to be refused and recorded — never to the blob store.
|
|
1489
1581
|
const hash = subject.capabilitySession;
|
|
1490
1582
|
return {
|
|
1491
|
-
upload: async (input) =>
|
|
1583
|
+
upload: async (input) => unwrapReply(await stub.capabilityAttachmentRefuseWrite(hash, tenantId, scopeId, 'attachments.upload', {
|
|
1492
1584
|
entityType: input.entity.entityType,
|
|
1493
1585
|
})),
|
|
1494
|
-
list: async (entity) =>
|
|
1586
|
+
list: async (entity) => unwrapReply(await stub.capabilityAttachmentList(entity, hash, tenantId, scopeId)),
|
|
1495
1587
|
open: async (attachmentId) => {
|
|
1496
|
-
const record =
|
|
1588
|
+
const record = unwrapReply(await stub.capabilityAttachmentOpen(attachmentId, hash, tenantId, scopeId));
|
|
1497
1589
|
return record ? this.openAttachmentBytes(store, scopeId, record) : null;
|
|
1498
1590
|
},
|
|
1499
|
-
remove: async (attachmentId) =>
|
|
1591
|
+
remove: async (attachmentId) => unwrapReply(await stub.capabilityAttachmentRefuseWrite(hash, tenantId, scopeId, 'attachments.remove', {
|
|
1500
1592
|
attachmentId,
|
|
1501
1593
|
})),
|
|
1502
1594
|
search: async (term, options) => {
|
|
1503
1595
|
// Judged here too, so a short term reaches the caller as `SearchTermTooShort`
|
|
1504
1596
|
// rather than as a message that crossed the RPC boundary. The DO judges again.
|
|
1505
1597
|
searchMatchExpression(term, 'prefix');
|
|
1506
|
-
return
|
|
1598
|
+
return unwrapReply(await stub.capabilityAttachmentSearch(term, searchLimit(options?.limit), hash, tenantId, scopeId));
|
|
1507
1599
|
},
|
|
1508
1600
|
};
|
|
1509
1601
|
}
|
|
@@ -1558,7 +1650,7 @@ export class CloudflareScopeHost {
|
|
|
1558
1650
|
throw new Error('attachments.search is not available on a delivery-scoped surface');
|
|
1559
1651
|
}
|
|
1560
1652
|
searchMatchExpression(term, 'prefix');
|
|
1561
|
-
return
|
|
1653
|
+
return unwrapReply(await stub.attachmentSearch(term, searchLimit(options?.limit), asPrincipalId, tenantId, scopeId, connectionId));
|
|
1562
1654
|
},
|
|
1563
1655
|
};
|
|
1564
1656
|
}
|
|
@@ -1606,24 +1698,51 @@ export class CloudflareScopeHost {
|
|
|
1606
1698
|
// this is the scope's real store, as for the provision's seat above; a hosted scope is
|
|
1607
1699
|
// restored in its deployment, which the platform carries the same list to.
|
|
1608
1700
|
const ownStore = this.cpLess || existing.vertical === null;
|
|
1609
|
-
const recordedOff = ownStore ? await this.cp.switchedOffModulesOf(tenantId, scopeId) : [];
|
|
1610
1701
|
const replayAt = new Date().toISOString();
|
|
1702
|
+
const carry = ownStore ? await this.recordedOffCarry(tenantId, scopeId, replayAt) : undefined;
|
|
1611
1703
|
const switchedOff = await this.scopeStub(scopeId).importDump(dump.tables, scopeId, {
|
|
1612
|
-
switchOff:
|
|
1613
|
-
? {
|
|
1614
|
-
moduleIds: recordedOff,
|
|
1615
|
-
at: replayAt,
|
|
1616
|
-
// #1823: a recorded-off module held only by a tenant grant goes back off too.
|
|
1617
|
-
tenantHeld: await this.cp.tenantHeldSystemModules(tenantId, recordedOff, replayAt),
|
|
1618
|
-
}
|
|
1619
|
-
: undefined,
|
|
1704
|
+
switchOff: carry ? { at: replayAt, ...carry } : undefined,
|
|
1620
1705
|
sourceScopeId: opts?.sourceScopeId ?? dump.scopeId,
|
|
1621
1706
|
});
|
|
1622
1707
|
await this.recordAdmin(actor, 'restoreScope', { tenantId, scopeId }, null, { sourceScopeId: dump.scopeId, tables: dump.tables.length, capturedAt: dump.capturedAt });
|
|
1623
|
-
if (
|
|
1708
|
+
if (carry) {
|
|
1624
1709
|
await this.admin.reassertSystemSwitches(actor, { tenantId, scopeId }, { appliedInUnit: switchedOff });
|
|
1625
1710
|
}
|
|
1626
1711
|
}
|
|
1712
|
+
/**
|
|
1713
|
+
* #1742, #2029: what this host's own unit carries of the directory's record — the modules and
|
|
1714
|
+
* the peers recorded OFF on the scope, each with the ones held there only by a tenant-level
|
|
1715
|
+
* grant (#1823, #2030). Undefined when nothing is recorded off, so such a unit is unchanged.
|
|
1716
|
+
*/
|
|
1717
|
+
async recordedOffCarry(tenantId, scopeId, at) {
|
|
1718
|
+
const [modules, peers] = await Promise.all([
|
|
1719
|
+
this.recordedOff('system', tenantId, scopeId, at),
|
|
1720
|
+
this.recordedOff('peer', tenantId, scopeId, at),
|
|
1721
|
+
]);
|
|
1722
|
+
if (!modules.keys.length && !peers.keys.length)
|
|
1723
|
+
return undefined;
|
|
1724
|
+
return {
|
|
1725
|
+
moduleIds: modules.keys,
|
|
1726
|
+
tenantHeld: modules.tenantHeld,
|
|
1727
|
+
verticals: peers.keys,
|
|
1728
|
+
tenantHeldVerticals: peers.tenantHeld,
|
|
1729
|
+
fences: switchFencesCarried([
|
|
1730
|
+
['system', modules],
|
|
1731
|
+
['peer', peers],
|
|
1732
|
+
]),
|
|
1733
|
+
};
|
|
1734
|
+
}
|
|
1735
|
+
/** The subjects of one kind the directory records OFF on the scope, and of those the tenant-held ones. */
|
|
1736
|
+
async recordedOff(kind, tenantId, scopeId, at) {
|
|
1737
|
+
// #2045: the OFF rows and their fences in one read, so every key carried has its own row's fence.
|
|
1738
|
+
const fences = new Map((await this.cp.switchRecordStatesOf(kind, tenantId, scopeId))
|
|
1739
|
+
.filter(([, row]) => row.position === 'off')
|
|
1740
|
+
.map(([key, row]) => [key, row.fence]));
|
|
1741
|
+
const keys = [...fences.keys()];
|
|
1742
|
+
if (!keys.length)
|
|
1743
|
+
return { keys, tenantHeld: [], fences };
|
|
1744
|
+
return { keys, tenantHeld: await this.cp.tenantHeldOf(kind, tenantId, keys, at), fences };
|
|
1745
|
+
}
|
|
1627
1746
|
async snapshotScope(actor, tenantId, scopeId, opts) {
|
|
1628
1747
|
const source = await this.admin.getScopeRecord(actor, tenantId, scopeId);
|
|
1629
1748
|
if (!source)
|
|
@@ -1654,7 +1773,7 @@ export class CloudflareScopeHost {
|
|
|
1654
1773
|
if (!rec)
|
|
1655
1774
|
throw substratError('not_found', `unknown scope ${scopeId} in tenant ${tenantId}`);
|
|
1656
1775
|
if (!rec.forkedFrom && rec.kind !== 'preview') {
|
|
1657
|
-
throw
|
|
1776
|
+
throw substratError('forbidden', `scope ${scopeId} is not a fork or preview — only previews may be deleted; ` +
|
|
1658
1777
|
`archive a primary scope instead`);
|
|
1659
1778
|
}
|
|
1660
1779
|
// Storage BEFORE the directory row: a crash between the two leaves a visible row
|
|
@@ -1815,34 +1934,55 @@ export class CloudflareScopeHost {
|
|
|
1815
1934
|
/** #1834: the job pass this door is opened for; its "not now" refusals are tied to it. */
|
|
1816
1935
|
pass) {
|
|
1817
1936
|
if (!this.moduleIds.has(moduleId)) {
|
|
1818
|
-
throw
|
|
1937
|
+
throw substratError('not_found', `module not registered on this host: ${moduleId}`);
|
|
1819
1938
|
}
|
|
1820
1939
|
await this.assertLive(tenantId, scopeId);
|
|
1821
1940
|
await this.migrateAndRecord(scopeId);
|
|
1941
|
+
return this.openDoor('system', moduleId, scopeId, gated, pass);
|
|
1942
|
+
}
|
|
1943
|
+
/**
|
|
1944
|
+
* #1834: one door, for either kind of switched subject (#2029). The gate is read here unless the
|
|
1945
|
+
* caller read it (`runDueSchedules` reports a hold it could not read on its pass report); a held
|
|
1946
|
+
* subject is refused, and a call landing on another instance than the gate's re-gates, at most
|
|
1947
|
+
* `SYSTEM_DOOR_REGATES` times, then fails closed.
|
|
1948
|
+
*
|
|
1949
|
+
* The peer door (`kind: 'peer'`) is the system door's for a peer vertical. A PITR rewind to before
|
|
1950
|
+
* a peer was switched off brings its grants back live and its OFF marker gone, so the rewind holds
|
|
1951
|
+
* it outside the scope (`rewindHolding`, as for a module), and every peer entry — an invoke, a
|
|
1952
|
+
* delivery, a coverage read, an export read for it as consumer — opens this door after
|
|
1953
|
+
* `peerScopeGate`.
|
|
1954
|
+
*/
|
|
1955
|
+
async openDoor(kind, key, scopeId, gated, pass) {
|
|
1956
|
+
const words = DOOR_WORDS[kind];
|
|
1957
|
+
const what = words.what(key);
|
|
1822
1958
|
// A gate this door reads itself reports a hold it could not read; one handed in was the
|
|
1823
|
-
// caller's to report
|
|
1959
|
+
// caller's to report.
|
|
1824
1960
|
const regate = async () => {
|
|
1825
|
-
const fresh = await this.
|
|
1961
|
+
const fresh = await this.doorGate(kind, scopeId, key);
|
|
1826
1962
|
if (fresh.error)
|
|
1827
|
-
console.error(`substrat:
|
|
1963
|
+
console.error(`substrat: door of ${what} on ${scopeId}: ${fresh.error}`);
|
|
1828
1964
|
return fresh;
|
|
1829
1965
|
};
|
|
1830
1966
|
let gate = gated ?? (await regate());
|
|
1831
1967
|
return {
|
|
1832
|
-
|
|
1968
|
+
kind,
|
|
1969
|
+
key,
|
|
1833
1970
|
through: async (call) => {
|
|
1834
1971
|
for (let regates = 0;; regates += 1) {
|
|
1835
1972
|
if (gate.held) {
|
|
1836
1973
|
// The hold ends when the switch is applied again: a job run waits it out without
|
|
1837
1974
|
// spending its retries, because the driver defers on THIS refusal (`doorWait`).
|
|
1838
|
-
|
|
1839
|
-
|
|
1975
|
+
const held = substratError('forbidden', `${what} is held off on this scope: it was rewound to before its ` +
|
|
1976
|
+
`${words.switchName} switch was turned off, and it stays off until ` +
|
|
1977
|
+
'the switch is applied again', { reason: SYSTEM_DOOR_WAIT });
|
|
1978
|
+
this.heldRefusals.add(held);
|
|
1979
|
+
throw this.doorWait(pass, held);
|
|
1840
1980
|
}
|
|
1841
1981
|
const answer = await call(gate.instance);
|
|
1842
1982
|
if (!isSystemDoorMoved(answer))
|
|
1843
1983
|
return answer;
|
|
1844
1984
|
if (regates >= SYSTEM_DOOR_REGATES) {
|
|
1845
|
-
throw this.doorWait(pass, substratError('unavailable', `the scope kept restarting under the
|
|
1985
|
+
throw this.doorWait(pass, substratError('unavailable', `the scope kept restarting under the ${words.door} of ${what} ` +
|
|
1846
1986
|
`(${regates + 1} attempts); nothing ran. Retry later`, { reason: SYSTEM_DOOR_WAIT }));
|
|
1847
1987
|
}
|
|
1848
1988
|
gate = await regate();
|
|
@@ -1859,6 +1999,16 @@ export class CloudflareScopeHost {
|
|
|
1859
1999
|
* on a later pass is an ordinary failure there. Held weakly: an error nobody holds goes.
|
|
1860
2000
|
*/
|
|
1861
2001
|
doorWaits = new WeakMap();
|
|
2002
|
+
/**
|
|
2003
|
+
* #2029: the hold refusals this host's doors threw, by identity — so a peer delivery pauses its
|
|
2004
|
+
* edge, and a coverage read answers "holds nothing", on exactly those and on no error that only
|
|
2005
|
+
* looks like one. Held weakly, as `doorWaits` is.
|
|
2006
|
+
*/
|
|
2007
|
+
heldRefusals = new WeakSet();
|
|
2008
|
+
/** #2029: is `err` a hold refusal this host's door threw? */
|
|
2009
|
+
isHeldRefusal(err) {
|
|
2010
|
+
return typeof err === 'object' && err !== null && this.heldRefusals.has(err);
|
|
2011
|
+
}
|
|
1862
2012
|
/** #1834: mark this door's own refusal as "not now" for the pass it was opened for, and hand it back. */
|
|
1863
2013
|
doorWait(pass, err) {
|
|
1864
2014
|
if (pass)
|
|
@@ -1873,13 +2023,23 @@ export class CloudflareScopeHost {
|
|
|
1873
2023
|
return true;
|
|
1874
2024
|
}
|
|
1875
2025
|
/**
|
|
1876
|
-
* #1834:
|
|
1877
|
-
*
|
|
2026
|
+
* #1834: a door's gate — the scope's state read for the subject, the instance that answered it,
|
|
2027
|
+
* and then the rewind hold (#1819), for any subject the scope does not already have off. The
|
|
1878
2028
|
* hold is read AFTER the state, which is the order `switchHeld` needs.
|
|
2029
|
+
*
|
|
2030
|
+
* `ungranted` reads the hold too (#2029): a subject held on the scope only by a TENANT-level
|
|
2031
|
+
* grant (#1823, #2030) comes back from a rewind with no scope grant and no marker, and the
|
|
2032
|
+
* tenant grant still authorizes it there. A rewind captured it off, so the hold must keep it so.
|
|
1879
2033
|
*/
|
|
1880
|
-
async
|
|
1881
|
-
const
|
|
1882
|
-
const
|
|
2034
|
+
async doorGate(kind, scopeId, key) {
|
|
2035
|
+
const stub = this.scopeStub(scopeId);
|
|
2036
|
+
const { state, instance } = await (kind === 'system' ? stub.systemDoorState(key) : stub.peerDoorState(key));
|
|
2037
|
+
const cleared = `${scopeId} ${instance}`;
|
|
2038
|
+
if (state === 'off' || CLEAR_SCOPE_INSTANCES.has(cleared))
|
|
2039
|
+
return { state, instance, held: false };
|
|
2040
|
+
const hold = await this.switchHeld(scopeId, holdKeyOf(kind, key));
|
|
2041
|
+
if (hold.scopeClear)
|
|
2042
|
+
rememberClearInstance(cleared);
|
|
1883
2043
|
return { state, instance, held: hold.held, ...(hold.error ? { error: hold.error } : {}) };
|
|
1884
2044
|
}
|
|
1885
2045
|
/**
|
|
@@ -1893,8 +2053,25 @@ export class CloudflareScopeHost {
|
|
|
1893
2053
|
// The peer door's own gate (#1706): K-3's pair check, the refusal for a scope served
|
|
1894
2054
|
// elsewhere, the lifecycle check and the migration. A delivery is a write into the
|
|
1895
2055
|
// consumer's scope, so it takes the gate an invoke through that door takes.
|
|
1896
|
-
await this.peerScopeGate(tenantId, scopeId, 'deliverToPeer');
|
|
1897
|
-
|
|
2056
|
+
const record = await this.peerScopeGate(tenantId, scopeId, 'deliverToPeer');
|
|
2057
|
+
// #2004: a copy consumes no other vertical's events, at the door as in the sweep
|
|
2058
|
+
// (`scope-copy.ts`). A pause: nothing runs and the watermark stays.
|
|
2059
|
+
if (await this.isInertScope(tenantId, scopeId, record)) {
|
|
2060
|
+
return { ...emptyImportResult(batch), paused: { reason: INERT_SCOPE_REASON } };
|
|
2061
|
+
}
|
|
2062
|
+
// #2029: the producer is a peer, so the delivery goes through its door: a producer the rewind
|
|
2063
|
+
// hold keeps off pauses the edge, exactly as a switched-off one does inside the DO.
|
|
2064
|
+
const door = await this.openDoor('peer', batch.source.vertical, scopeId);
|
|
2065
|
+
let applied;
|
|
2066
|
+
try {
|
|
2067
|
+
applied = await door.through((instance) => this.scopeStub(scopeId).importApply(batch, tenantId, scopeId, instance));
|
|
2068
|
+
}
|
|
2069
|
+
catch (err) {
|
|
2070
|
+
if (!this.isHeldRefusal(err))
|
|
2071
|
+
throw err;
|
|
2072
|
+
return { ...emptyImportResult(batch), paused: { reason: err.message } };
|
|
2073
|
+
}
|
|
2074
|
+
const result = importResult.parse(applied);
|
|
1898
2075
|
if (result.delivered > 0) {
|
|
1899
2076
|
try {
|
|
1900
2077
|
await this.drainExecutors(tenantId, scopeId, null);
|
|
@@ -1938,7 +2115,9 @@ export class CloudflareScopeHost {
|
|
|
1938
2115
|
async getVerticalScope(caller, tenantId, scopeId, options) {
|
|
1939
2116
|
const parsed = verticalCaller.parse(caller);
|
|
1940
2117
|
await this.peerScopeGate(tenantId, scopeId, 'getVerticalScope');
|
|
1941
|
-
|
|
2118
|
+
// #2029: gated against the rewind hold and pinned, as the system door is.
|
|
2119
|
+
const door = await this.openDoor('peer', parsed.vertical, scopeId);
|
|
2120
|
+
return this.buildStub(tenantId, scopeId, undefined, undefined, door, options, undefined, undefined, parsed);
|
|
1942
2121
|
}
|
|
1943
2122
|
/**
|
|
1944
2123
|
* Does peer `vertical` hold each key at this scope's node now (#1706) — the ScopeDO's own
|
|
@@ -1946,9 +2125,19 @@ export class CloudflareScopeHost {
|
|
|
1946
2125
|
*/
|
|
1947
2126
|
async peerCovers(tenantId, scopeId, vertical, permissions) {
|
|
1948
2127
|
await this.peerScopeGate(tenantId, scopeId, 'peerCovers');
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
|
|
2128
|
+
const slug = verticalSlug.parse(vertical);
|
|
2129
|
+
// #2029: a peer the rewind hold keeps off holds nothing here, as a switched-off one doesn't.
|
|
2130
|
+
const door = await this.openDoor('peer', slug, scopeId);
|
|
2131
|
+
try {
|
|
2132
|
+
return peerCoverage
|
|
2133
|
+
.array()
|
|
2134
|
+
.parse(await door.through((instance) => this.scopeStub(scopeId).peerCovers(tenantId, scopeId, slug, [...permissions], instance)));
|
|
2135
|
+
}
|
|
2136
|
+
catch (err) {
|
|
2137
|
+
if (this.isHeldRefusal(err))
|
|
2138
|
+
return permissions.map((permission) => ({ permission, held: false }));
|
|
2139
|
+
throw err;
|
|
2140
|
+
}
|
|
1952
2141
|
}
|
|
1953
2142
|
/**
|
|
1954
2143
|
* `ctx.canAssign`'s bound for a principal the host names (#1931) — the ScopeDO's own role read
|
|
@@ -1980,6 +2169,32 @@ export class CloudflareScopeHost {
|
|
|
1980
2169
|
throw unknownRoleError(roleKey);
|
|
1981
2170
|
return coverage.parse(bound);
|
|
1982
2171
|
}
|
|
2172
|
+
async listScopeRoleHolders(tenantId, scopeId, principal) {
|
|
2173
|
+
await this.scopeRoleGate(tenantId, scopeId, 'listScopeRoleHolders');
|
|
2174
|
+
return this.scopeStub(scopeId).scopeRoleHoldersFor(scopeId, principal === undefined ? undefined : principalId.parse(principal));
|
|
2175
|
+
}
|
|
2176
|
+
async changeScopeRoleBounded(tenantId, scopeId, caller, principal, from, to) {
|
|
2177
|
+
await this.scopeRoleGate(tenantId, scopeId, 'changeScopeRoleBounded');
|
|
2178
|
+
const target = principalId.parse(principal);
|
|
2179
|
+
const bound = await this.scopeStub(scopeId).changeScopeRoleBoundedFor(tenantId, scopeId, principalId.parse(caller), target, from, to);
|
|
2180
|
+
if (bound === 'not-held')
|
|
2181
|
+
throw substratError('conflict', `${target} does not hold '${from}' at this scope`);
|
|
2182
|
+
if (bound === 'unknown-to')
|
|
2183
|
+
throw unknownRoleError(to);
|
|
2184
|
+
return coverage.parse(bound);
|
|
2185
|
+
}
|
|
2186
|
+
async revokeScopeRolesBounded(tenantId, scopeId, caller, principal) {
|
|
2187
|
+
await this.scopeRoleGate(tenantId, scopeId, 'revokeScopeRolesBounded');
|
|
2188
|
+
const result = await this.scopeStub(scopeId).revokeScopeRolesBoundedFor(tenantId, scopeId, principalId.parse(caller), principalId.parse(principal));
|
|
2189
|
+
return { coverage: coverage.parse(result.coverage), revoked: result.revoked };
|
|
2190
|
+
}
|
|
2191
|
+
/** The (tenant, scope) gate the scope-role verbs share — `assignScopeRoleBounded`'s two checks. */
|
|
2192
|
+
async scopeRoleGate(tenantId, scopeId, verb) {
|
|
2193
|
+
if (this.cpLess && !(await this.scopeStub(scopeId).servesTenant(tenantId))) {
|
|
2194
|
+
throw unknownScopeForTenant(tenantId, scopeId);
|
|
2195
|
+
}
|
|
2196
|
+
await this.peerScopeGate(tenantId, scopeId, verb);
|
|
2197
|
+
}
|
|
1983
2198
|
/**
|
|
1984
2199
|
* Where a peer verb (#1706) may reach a scope: its own ScopeDO, after the ordinary pair and
|
|
1985
2200
|
* lifecycle gate. Refused, loudly, on the SHARED control plane for a scope bound to a
|
|
@@ -1994,8 +2209,9 @@ export class CloudflareScopeHost {
|
|
|
1994
2209
|
// door's own tenant-confinement test proved. A RECORD crosses intact, so the coordinator
|
|
1995
2210
|
// reads the record and types the refusal itself. `validateScopeAccess` still runs below
|
|
1996
2211
|
// and still owns the lifecycle half (a suspended tenant or scope).
|
|
2212
|
+
let record;
|
|
1997
2213
|
if (!this.cpLess) {
|
|
1998
|
-
|
|
2214
|
+
record = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
1999
2215
|
if (!record) {
|
|
2000
2216
|
throw unknownScopeForTenant(tenantId, scopeId);
|
|
2001
2217
|
}
|
|
@@ -2003,6 +2219,23 @@ export class CloudflareScopeHost {
|
|
|
2003
2219
|
}
|
|
2004
2220
|
await this.assertLive(tenantId, scopeId);
|
|
2005
2221
|
await this.migrateAndRecord(scopeId);
|
|
2222
|
+
return record;
|
|
2223
|
+
}
|
|
2224
|
+
/**
|
|
2225
|
+
* Whether this scope is held inert (#2005): a fork, a snapshot or a preview, which causes no
|
|
2226
|
+
* outbound effects and consumes no other vertical's events (#2004). The one primacy question
|
|
2227
|
+
* every such door asks. Where there is a directory its record answers (`record`, the one
|
|
2228
|
+
* `peerScopeGate` returns, when the caller has already read it). A CP-less host has none, so it asks the scope's own storage
|
|
2229
|
+
* whether it is classified a copy (`_substrat_copy_origin`'s `is_copy`, #2009, set on the
|
|
2230
|
+
* directory's word): a preview and a snapshot reach a hosted vertical as restores the
|
|
2231
|
+
* platform marks.
|
|
2232
|
+
*/
|
|
2233
|
+
isInertScope(tenantId, scopeId, record) {
|
|
2234
|
+
if (this.cpLess)
|
|
2235
|
+
return this.scopeStub(scopeId).isCopy();
|
|
2236
|
+
if (record)
|
|
2237
|
+
return Promise.resolve(!isPrimaryScopeRow(record));
|
|
2238
|
+
return this.cp.getScopeRecord(tenantId, scopeId).then((row) => !isPrimaryScopeRow(row));
|
|
2006
2239
|
}
|
|
2007
2240
|
/**
|
|
2008
2241
|
* The capability's attachment door (#1686) — `getCapabilityScope`'s gate, then the
|
|
@@ -2183,7 +2416,7 @@ export class CloudflareScopeHost {
|
|
|
2183
2416
|
// #1819: a scope rewound to before its switch was pulled answers `on`, and the rewind held
|
|
2184
2417
|
// the module outside the scope. Both are read by the system door's own gate (#1834), which
|
|
2185
2418
|
// every fire below then goes through, pinned to the instance this read came from.
|
|
2186
|
-
const gate = await this.
|
|
2419
|
+
const gate = await this.doorGate('system', scopeId, moduleId);
|
|
2187
2420
|
if (gate.state === 'ungranted')
|
|
2188
2421
|
return report;
|
|
2189
2422
|
if (gate.error)
|
|
@@ -2258,10 +2491,11 @@ export class CloudflareScopeHost {
|
|
|
2258
2491
|
/** The stub body, shared by the principal, connection, system and impersonation doors. */
|
|
2259
2492
|
buildStub(tenantId, scopeId, principal, connectionId,
|
|
2260
2493
|
/**
|
|
2261
|
-
* #1834: the
|
|
2494
|
+
* #1834: the door this stub acts through. The ONLY way to make a stub acting as
|
|
2262
2495
|
* `system:<moduleId>`: the module is the door's, and every invoke is gated and pinned by it.
|
|
2496
|
+
* #2029: a peer stub's door too (`openDoor('peer', …)`), gating and pinning each invoke the same way.
|
|
2263
2497
|
*/
|
|
2264
|
-
|
|
2498
|
+
door, options,
|
|
2265
2499
|
/**
|
|
2266
2500
|
* K-42: the session this stub acts under. Re-read from the directory on
|
|
2267
2501
|
* EVERY invoke rather than captured here — a stub is a capability and
|
|
@@ -2282,7 +2516,7 @@ export class CloudflareScopeHost {
|
|
|
2282
2516
|
*/
|
|
2283
2517
|
verticalCaller) {
|
|
2284
2518
|
const stub = this.scopeStub(scopeId);
|
|
2285
|
-
const systemModuleId =
|
|
2519
|
+
const systemModuleId = door?.kind === 'system' ? door.key : undefined;
|
|
2286
2520
|
const cp = this.cp;
|
|
2287
2521
|
const operationEntitlement = this.operationEntitlement;
|
|
2288
2522
|
// The DO needs SOME principal-shaped value for `ctx.principal`; for a
|
|
@@ -2338,10 +2572,10 @@ export class CloudflareScopeHost {
|
|
|
2338
2572
|
// K-42: resolved per invoke, so expiry and `endImpersonation` both bite.
|
|
2339
2573
|
const session = sessionId === undefined ? undefined : await this.resolveImpersonation(sessionId, tenantId, scopeId);
|
|
2340
2574
|
const send = (systemDoorInstance) => stub.invoke(operation, input, asPrincipalId, tenantId, scopeId, connectionId, requiredKey, systemModuleId, true, invokeOptions, session, capabilitySession, verticalCaller, systemDoorInstance);
|
|
2341
|
-
// #1834: through the
|
|
2342
|
-
//
|
|
2343
|
-
const envelope =
|
|
2344
|
-
? await
|
|
2575
|
+
// #1834: through the door (a module's, or #2029 a peer's), pinned to the instance its gate
|
|
2576
|
+
// read. A missed pin comes back as an answer, which `through` re-gates on.
|
|
2577
|
+
const envelope = door
|
|
2578
|
+
? await door.through(async (instance) => {
|
|
2345
2579
|
const sent = await send(instance);
|
|
2346
2580
|
return sent.systemDoorMoved === true ? SYSTEM_DOOR_MOVED : sent;
|
|
2347
2581
|
})
|
|
@@ -2394,9 +2628,9 @@ export class CloudflareScopeHost {
|
|
|
2394
2628
|
throw substratError('unavailable', `${operation} ran on a scope host that did not understand peer callers. ` +
|
|
2395
2629
|
'Retry once the scope has been migrated');
|
|
2396
2630
|
}
|
|
2397
|
-
// #1834, the same shape: a DO too old to know the
|
|
2398
|
-
//
|
|
2399
|
-
if (
|
|
2631
|
+
// #1834, the same shape: a DO too old to know the door's pin ran the call without
|
|
2632
|
+
// checking it met the storage the door's gate read.
|
|
2633
|
+
if (door !== undefined && envelope.systemDoor?.honoured !== true) {
|
|
2400
2634
|
throw substratError('unavailable', `${operation} ran on a scope host that did not understand the system door's pin. ` +
|
|
2401
2635
|
'Retry once the scope has been migrated');
|
|
2402
2636
|
}
|
|
@@ -2410,11 +2644,13 @@ export class CloudflareScopeHost {
|
|
|
2410
2644
|
// this scope's outbox — and draining anyway would run executors and
|
|
2411
2645
|
// connectors as a side effect of a session that may not have side effects.
|
|
2412
2646
|
// Whatever the outbox already held is the drain sweep's own backstop.
|
|
2647
|
+
// #1184: what this call's executors did inline, for `onExecutorOutcomes`.
|
|
2648
|
+
const executorOutcomes = invokeOptions?.onExecutorOutcomes ? [] : undefined;
|
|
2413
2649
|
const drained = session?.mode === 'read-only'
|
|
2414
2650
|
? { attempted: 0, delivered: 0, retrying: 0, deadLettered: 0, routedToPlatform: 0 }
|
|
2415
2651
|
// #1525: this drain is part of the call that emitted the events — the
|
|
2416
2652
|
// coordinator's half of the post-commit tail the DO ran the consumers in.
|
|
2417
|
-
: await this.drainExecutors(tenantId, scopeId, invokeOptions?.invocationId ?? null);
|
|
2653
|
+
: await this.drainExecutors(tenantId, scopeId, invokeOptions?.invocationId ?? null, executorOutcomes);
|
|
2418
2654
|
// #458: the operation committed having enqueued platform intents — tell the
|
|
2419
2655
|
// caller's harness so it can flag the response for the router kick (#381).
|
|
2420
2656
|
// Routed connector deliveries (#574 phase 3) count too: the inline drain just
|
|
@@ -2431,6 +2667,9 @@ export class CloudflareScopeHost {
|
|
|
2431
2667
|
invokeOptions?.onIdempotentReplay?.();
|
|
2432
2668
|
if (envelope.emitted)
|
|
2433
2669
|
invokeOptions?.onEmitted?.(envelope.emitted);
|
|
2670
|
+
if (executorOutcomes && session?.mode !== 'read-only' && !envelope.idempotency?.replayed) {
|
|
2671
|
+
invokeOptions?.onExecutorOutcomes?.(executorOutcomes);
|
|
2672
|
+
}
|
|
2434
2673
|
return envelope.result;
|
|
2435
2674
|
},
|
|
2436
2675
|
};
|
|
@@ -2440,8 +2679,12 @@ export class CloudflareScopeHost {
|
|
|
2440
2679
|
}
|
|
2441
2680
|
// -- admin surface --------------------------------------------------------
|
|
2442
2681
|
/** #977: this host, with every admin row it writes naming who the actor acted for. */
|
|
2443
|
-
attributed(onBehalfOf) {
|
|
2444
|
-
return
|
|
2682
|
+
attributed(onBehalfOf, options) {
|
|
2683
|
+
return attributedView(this, { onBehalfOf, causedBy: options?.causedBy }, this.buildAdmin);
|
|
2684
|
+
}
|
|
2685
|
+
/** #2055: this host, with every admin row it writes naming `eventId` as its cause. */
|
|
2686
|
+
causedByView(eventId) {
|
|
2687
|
+
return attributedView(this, { causedBy: eventId }, this.buildAdmin);
|
|
2445
2688
|
}
|
|
2446
2689
|
buildAdmin() {
|
|
2447
2690
|
// The directory row → the `scope` contract. Parsed, not cast: the columns are
|
|
@@ -2523,7 +2766,7 @@ export class CloudflareScopeHost {
|
|
|
2523
2766
|
*/
|
|
2524
2767
|
const requireOrg = async (tenant, id) => {
|
|
2525
2768
|
if (!(await this.cp.readOrg(tenant, id))) {
|
|
2526
|
-
throw
|
|
2769
|
+
throw substratError('not_found', `unknown org ${id} in tenant ${tenant}`);
|
|
2527
2770
|
}
|
|
2528
2771
|
};
|
|
2529
2772
|
const mapScope = (r) => scopeSchema.parse({
|
|
@@ -2610,94 +2853,20 @@ export class CloudflareScopeHost {
|
|
|
2610
2853
|
}
|
|
2611
2854
|
};
|
|
2612
2855
|
/**
|
|
2613
|
-
*
|
|
2856
|
+
* Where one scope's kill switch of one kind lands — the ONE rule the operator's switch and
|
|
2857
|
+
* the #1674 re-assert share, so a re-assert moves exactly the switch an operator's OFF would
|
|
2858
|
+
* have. `kind` is a module's schedule switch (#1666) or a peer's (#1706, #2029); `key` is the
|
|
2859
|
+
* module id or the peer's slug.
|
|
2614
2860
|
*
|
|
2615
2861
|
* Where the write lands is the whole point of the delegation branch. The shared control
|
|
2616
|
-
* plane's own `SCOPE` namespace is the module-less placeholder, so a hosted scope's
|
|
2617
|
-
*
|
|
2618
|
-
* switch pulled while every schedule kept firing
|
|
2619
|
-
*
|
|
2620
|
-
|
|
2621
|
-
|
|
2622
|
-
* #1706: move one PEER's kill switch on one scope — see `HostAdmin.revokeFromPeer`. The
|
|
2623
|
-
* schedule switch's audit discipline (intent first, outcome after, every attempt) and,
|
|
2624
|
-
* since part 3, its delegation: a scope served by a vertical's own deployment is switched
|
|
2625
|
-
* THERE, over `PeerSwitchDelegation`. Without one configured the scope is still refused
|
|
2626
|
-
* `unavailable` (`peerScopeGate`) rather than switched in the placeholder namespace.
|
|
2627
|
-
*/
|
|
2628
|
-
const switchPeerAt = async (actor, raw, to) => {
|
|
2629
|
-
const input = peerSwitch.parse(raw);
|
|
2630
|
-
const { tenantId, scopeId } = input.node;
|
|
2631
|
-
let vertical = null;
|
|
2632
|
-
if (!this.cpLess) {
|
|
2633
|
-
const rec = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
2634
|
-
if (!rec)
|
|
2635
|
-
throw unknownScopeForTenant(tenantId, scopeId);
|
|
2636
|
-
vertical = rec.vertical;
|
|
2637
|
-
}
|
|
2638
|
-
const action = to === 'off' ? 'revokeFromPeer' : 'restoreToPeer';
|
|
2639
|
-
// Where the write lands is the whole point of this branch (#1706 part 3, mirroring
|
|
2640
|
-
// #1666). A scope bound to a vertical is served by that vertical's own deployment, and
|
|
2641
|
-
// its `vertical:<slug>` grants live there; switching it in the shared control plane's
|
|
2642
|
-
// placeholder namespace would tombstone nothing and report a peer cut off while every
|
|
2643
|
-
// call it makes kept being admitted. A scope bound to NO vertical has its store here,
|
|
2644
|
-
// so it switches here — delegating it would throw "no deployment serving scope".
|
|
2645
|
-
// With no delegation configured the old refusal stands, loudly, rather than a write
|
|
2646
|
-
// into the wrong namespace.
|
|
2647
|
-
const delegation = this.cpLess || vertical !== null ? this.peerSwitchDelegation : undefined;
|
|
2648
|
-
if (delegation) {
|
|
2649
|
-
// The gate's lifecycle half still runs (a suspended tenant or scope cannot be
|
|
2650
|
-
// switched); its `scopeStub` half is what the delegation replaces.
|
|
2651
|
-
await this.validateScopeAccess(tenantId, scopeId);
|
|
2652
|
-
}
|
|
2653
|
-
else {
|
|
2654
|
-
await this.peerScopeGate(tenantId, scopeId, action);
|
|
2655
|
-
}
|
|
2656
|
-
const operationId = ulid();
|
|
2657
|
-
const target = { tenantId, scopeId, vertical };
|
|
2658
|
-
const base = { operationId, vertical: input.vertical, calls: to };
|
|
2659
|
-
await this.recordAdmin(actor, action, target, null, { ...base, phase: 'intent', reason: input.reason });
|
|
2660
|
-
let outcome;
|
|
2661
|
-
try {
|
|
2662
|
-
outcome = delegation
|
|
2663
|
-
? await delegation.switch({ tenantId, scopeId, vertical: input.vertical, to })
|
|
2664
|
-
: await this.scopeStub(scopeId).switchPeer(input.vertical, scopeId, to, new Date().toISOString());
|
|
2665
|
-
}
|
|
2666
|
-
catch (err) {
|
|
2667
|
-
await this.recordAdmin(actor, action, target, null, {
|
|
2668
|
-
...base,
|
|
2669
|
-
phase: 'failed',
|
|
2670
|
-
error: err instanceof Error ? err.message : String(err),
|
|
2671
|
-
}).catch(() => undefined);
|
|
2672
|
-
throw err;
|
|
2673
|
-
}
|
|
2674
|
-
await this.recordAdmin(actor, action, target, null, {
|
|
2675
|
-
...base,
|
|
2676
|
-
phase: outcome.held ? 'applied' : 'refused',
|
|
2677
|
-
changed: outcome.changed,
|
|
2678
|
-
permissions: outcome.permissions,
|
|
2679
|
-
});
|
|
2680
|
-
if (!outcome.held) {
|
|
2681
|
-
throw substratError('not_found', `scope ${scopeId} holds no grant for peer '${input.vertical}' — nothing to switch ${to} ` +
|
|
2682
|
-
`(check the slug: it is the calling vertical's registry id, as the target's \`peers\` names it)`);
|
|
2683
|
-
}
|
|
2684
|
-
return {
|
|
2685
|
-
operationId,
|
|
2686
|
-
vertical: input.vertical,
|
|
2687
|
-
calls: to,
|
|
2688
|
-
changed: outcome.changed,
|
|
2689
|
-
permissions: outcome.permissions,
|
|
2690
|
-
};
|
|
2691
|
-
};
|
|
2692
|
-
/**
|
|
2693
|
-
* Where one scope's schedule switch lands — the ONE rule `switchSystem` and the #1674
|
|
2694
|
-
* re-assert share, so a re-assert moves exactly the switch an operator's OFF would have.
|
|
2695
|
-
* A scope bound to no vertical (#1666 review) has no deployment to delegate to: its store
|
|
2696
|
-
* is the DO here, so the switch moves (or answers `held: false`) here too. Delegating it
|
|
2697
|
-
* would throw "no deployment serving scope" instead. A scope WITH a vertical still
|
|
2698
|
-
* delegates, and still fails loudly when none serves it.
|
|
2862
|
+
* plane's own `SCOPE` namespace is the module-less placeholder, so a hosted scope's switch
|
|
2863
|
+
* has to be moved in the deployment serving it, over that kind's delegation; writing it here
|
|
2864
|
+
* would report a switch pulled while every schedule kept firing, or every peer call kept
|
|
2865
|
+
* being admitted. A scope bound to no vertical (#1666 review) has no deployment to delegate
|
|
2866
|
+
* to: its store is the DO here, so the switch moves (or answers `held: false`) here too.
|
|
2867
|
+
* Delegating it would throw "no deployment serving scope" instead.
|
|
2699
2868
|
*/
|
|
2700
|
-
const
|
|
2869
|
+
const switchTarget = async (kind, tenantId, scopeId) => {
|
|
2701
2870
|
let vertical = null;
|
|
2702
2871
|
if (!this.cpLess) {
|
|
2703
2872
|
const rec = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
@@ -2705,66 +2874,109 @@ export class CloudflareScopeHost {
|
|
|
2705
2874
|
throw unknownScopeForTenant(tenantId, scopeId);
|
|
2706
2875
|
vertical = rec.vertical;
|
|
2707
2876
|
}
|
|
2708
|
-
const
|
|
2877
|
+
const hosted = this.cpLess || vertical !== null;
|
|
2878
|
+
const systemDelegation = kind === 'system' && hosted ? this.systemSwitchDelegation : undefined;
|
|
2879
|
+
const peerDelegation = kind === 'peer' && hosted ? this.peerSwitchDelegation : undefined;
|
|
2880
|
+
const delegated = systemDelegation !== undefined || peerDelegation !== undefined;
|
|
2709
2881
|
// #1819: a scope whose store is here was rewound here, so `switchInScope` releases its
|
|
2710
2882
|
// hold here too. A delegated move releases in the deployment serving the scope.
|
|
2711
|
-
// #1823: a
|
|
2712
|
-
// the scope's storage for the switch to find, so the directory says
|
|
2713
|
-
// A caller moving several
|
|
2714
|
-
// An OFF of a tenant-held
|
|
2883
|
+
// #1823 (#2030 for a peer): a subject whose only authority on the scope is a TENANT-level
|
|
2884
|
+
// grant has nothing in the scope's storage for the switch to find, so the directory says
|
|
2885
|
+
// whether it is held. A caller moving several reads them in one call and passes each in.
|
|
2886
|
+
// An OFF of a tenant-held MODULE is refused, after the far end answers, unless that answer
|
|
2715
2887
|
// attests the evaluator that denies a tenant-level grant (`deniesTenantGrants`). A
|
|
2716
2888
|
// deployment built before #1823 drops `tenantHeld`, still answers `held: true` (the module's
|
|
2717
2889
|
// scope-level grants were there to switch), and keeps authorizing the tenant-level grant —
|
|
2718
|
-
// so its `held` alone would have the platform record a scope off that is not.
|
|
2719
|
-
//
|
|
2720
|
-
//
|
|
2721
|
-
|
|
2722
|
-
|
|
2723
|
-
|
|
2724
|
-
|
|
2725
|
-
|
|
2726
|
-
|
|
2727
|
-
|
|
2890
|
+
// so its `held` alone would have the platform record a scope off that is not. A peer needs
|
|
2891
|
+
// no attestation: its door refuses on the marker in every build that has the peer switch at
|
|
2892
|
+
// all (`admitPeer` and `switchPeer` arrived together, #1714). Every OFF the platform drives
|
|
2893
|
+
// moves through here: an operator's (`switchSubjectAt`) and a re-assert's, which every carry
|
|
2894
|
+
// ends with.
|
|
2895
|
+
//
|
|
2896
|
+
// #2045 (Codex r3): whether a delegated deployment honours the switch fence is settled
|
|
2897
|
+
// BEFORE anything moves (`attestFence`, once per target), never after the fact by putting a
|
|
2898
|
+
// move back. Without the fence an ON is refused and an OFF goes through unfenced (see
|
|
2899
|
+
// `UnattestedSwitch`). A move a fenced deployment answers without the attestation still
|
|
2900
|
+
// throws, as defence in depth against a rollback between the preflight and the move.
|
|
2901
|
+
const delegation = systemDelegation ?? peerDelegation;
|
|
2902
|
+
let attested;
|
|
2903
|
+
const attestFence = () => (attested ??= delegation ? delegation.fenceSupported({ tenantId, scopeId }) : Promise.resolve(true));
|
|
2904
|
+
const move = async (key, to, at, tenantHeld,
|
|
2905
|
+
/** #2045: the switch call's fence — its operation id, or for a re-assert the record's. */
|
|
2906
|
+
fence) => {
|
|
2907
|
+
const fenced = await attestFence();
|
|
2908
|
+
if (!fenced && to === 'on')
|
|
2909
|
+
throw new UnattestedSwitch(unfencedMessage(scopeId));
|
|
2910
|
+
const held = tenantHeld ?? (await this.cp.tenantHeldOf(kind, tenantId, [key], at)).length > 0;
|
|
2911
|
+
const outcome = await (systemDelegation
|
|
2912
|
+
? systemDelegation.switch({ tenantId, scopeId, moduleId: key, to, tenantHeld: held, fence })
|
|
2913
|
+
: peerDelegation
|
|
2914
|
+
? peerDelegation.switch({ tenantId, scopeId, vertical: key, to, tenantHeld: held, fence })
|
|
2915
|
+
: this.switchInScope(kind, scopeId, key, to, at, held, fence));
|
|
2916
|
+
// An unfenced OFF answers without `fenced`, and that is what the callers read it by.
|
|
2917
|
+
if (fenced && outcome.fenced !== true)
|
|
2918
|
+
throw new UnattestedSwitch(unfencedMessage(scopeId));
|
|
2919
|
+
if (outcome.superseded)
|
|
2920
|
+
return outcome;
|
|
2921
|
+
if (kind === 'system' && to === 'off' && held && outcome.held && outcome.deniesTenantGrants !== true) {
|
|
2922
|
+
throw new UnattestedSwitch(undeniedMessage(scopeId, key));
|
|
2728
2923
|
}
|
|
2729
2924
|
return outcome;
|
|
2730
2925
|
};
|
|
2731
|
-
|
|
2732
|
-
* Is the module switched off on the scope right now? Read from wherever `move` writes — the
|
|
2733
|
-
* deployment's status read or this scope's own — for a move that threw (#1823): the switch
|
|
2734
|
-
* may already have moved before the throw (`switchInScope` moves it, then works the rewind
|
|
2735
|
-
* hold), and then the directory's record must stay.
|
|
2736
|
-
*/
|
|
2737
|
-
const isOff = async (moduleId) => {
|
|
2738
|
-
const rows = delegation
|
|
2739
|
-
? await delegation.status({ tenantId, scopeId })
|
|
2740
|
-
: systemScheduleEntry.array().parse(await this.scopeStub(scopeId).systemGrantsStatus());
|
|
2741
|
-
return rows.some((r) => r.moduleId === moduleId && r.schedules === 'off');
|
|
2742
|
-
};
|
|
2743
|
-
return { vertical, move, isOff };
|
|
2926
|
+
return { vertical, delegated, move, attestFence };
|
|
2744
2927
|
};
|
|
2745
|
-
|
|
2746
|
-
|
|
2747
|
-
|
|
2748
|
-
|
|
2928
|
+
/**
|
|
2929
|
+
* One kill switch moved on one scope — the ONE body `revokeFromSystem` / `restoreToSystem`
|
|
2930
|
+
* (#1666) and `revokeFromPeer` / `restoreToPeer` (#1706) share, so the two levers cannot
|
|
2931
|
+
* disagree about the record, the audit, the readback or the undo (#2029). The audit row is
|
|
2932
|
+
* written HERE either way — a deployment's host is CP-less, and its `recordAdmin` is a no-op.
|
|
2933
|
+
*/
|
|
2934
|
+
const switchSubjectAt = async (actor, kind, node, key, reason, to) => {
|
|
2935
|
+
const { tenantId, scopeId } = node;
|
|
2936
|
+
const { vertical, delegated, move, attestFence } = await switchTarget(kind, tenantId, scopeId);
|
|
2937
|
+
const action = switchActionOf(kind, to);
|
|
2938
|
+
if (kind === 'peer') {
|
|
2939
|
+
// The peer door's gate (#1706): with a delegation, its lifecycle half (a suspended tenant
|
|
2940
|
+
// or scope cannot be switched), the `scopeStub` half being what the delegation replaces.
|
|
2941
|
+
// Without one, a hosted scope is refused `unavailable` rather than switched in the
|
|
2942
|
+
// placeholder namespace, where it would tombstone nothing a peer call reads.
|
|
2943
|
+
if (delegated)
|
|
2944
|
+
await this.validateScopeAccess(tenantId, scopeId);
|
|
2945
|
+
else
|
|
2946
|
+
await this.peerScopeGate(tenantId, scopeId, action);
|
|
2947
|
+
}
|
|
2749
2948
|
// AUDIT FIRST (#1666 review): the intent row lands before anything moves, and the
|
|
2750
2949
|
// outcome row after — every attempt, a repeat included. The scope's store and the
|
|
2751
2950
|
// admin log are separate, so no order makes the pair atomic; this one fails toward
|
|
2752
2951
|
// "an intent with no recorded outcome" and never toward "a switch that moved with no
|
|
2753
2952
|
// audit row". A retry after a crash re-audits even though it answers `changed: false`.
|
|
2754
2953
|
const operationId = ulid();
|
|
2755
|
-
const action = to === 'off' ? 'revokeFromSystem' : 'restoreToSystem';
|
|
2756
2954
|
const target = { tenantId, scopeId, vertical };
|
|
2757
|
-
const base = { operationId,
|
|
2758
|
-
await this.recordAdmin(actor, action, target, null, { ...base, phase: 'intent', reason
|
|
2759
|
-
// The directory's record (#1674), written BEFORE the scope moves, both ways, and undone
|
|
2760
|
-
// if the move throws or holds nothing — see `recordSystemSwitchedOn` and (#1823)
|
|
2761
|
-
// `recordSystemSwitchedOff` for why that order is the safe one.
|
|
2762
|
-
const at = new Date().toISOString();
|
|
2763
|
-
const record = { tenantId, scopeId, moduleId: input.moduleId, actor, reason: input.reason, operationId, at };
|
|
2955
|
+
const base = { operationId, ...switchAuditSubject(kind, key, to) };
|
|
2956
|
+
await this.recordAdmin(actor, action, target, null, { ...base, phase: 'intent', reason });
|
|
2764
2957
|
const errorOf = (err) => (err instanceof Error ? err.message : String(err));
|
|
2958
|
+
// #2045 (Codex r3): the deployment's fence is attested BEFORE anything is recorded or moved. An
|
|
2959
|
+
// ON on a deployment built before it is refused here, with nothing written anywhere but the
|
|
2960
|
+
// audit. An OFF goes on: the kill switch must work on every deployment (see `UnattestedSwitch`).
|
|
2961
|
+
try {
|
|
2962
|
+
if (!(await attestFence()) && to === 'on') {
|
|
2963
|
+
throw substratError('precondition_failed', `${unfencedMessage(scopeId)} Nothing was switched.`);
|
|
2964
|
+
}
|
|
2965
|
+
}
|
|
2966
|
+
catch (err) {
|
|
2967
|
+
await this.recordAdmin(actor, action, target, null, { ...base, phase: 'refused', error: errorOf(err) }).catch(() => undefined);
|
|
2968
|
+
throw err;
|
|
2969
|
+
}
|
|
2970
|
+
// The directory's record (#1674; #2029 for a peer), written BEFORE the scope moves, both
|
|
2971
|
+
// ways — see `recordSwitchedOn` and (#1823) `recordSwitchedOff` for why that order is the
|
|
2972
|
+
// safe one. #2045 (Codex r3): the same write marks the subject owed and clears the scope's
|
|
2973
|
+
// reconcile receipt, so from here on the scope converges to this record whatever happens to
|
|
2974
|
+
// the move. The mark goes only when the scope confirms the move under this call's fence.
|
|
2975
|
+
const at = new Date().toISOString();
|
|
2976
|
+
const record = { kind, tenantId, scopeId, key, actor, reason, operationId, at };
|
|
2765
2977
|
let prior;
|
|
2766
2978
|
try {
|
|
2767
|
-
prior = to === 'on' ? await this.cp.
|
|
2979
|
+
prior = to === 'on' ? await this.cp.recordSwitchedOn(record) : await this.cp.recordSwitchedOff(record);
|
|
2768
2980
|
}
|
|
2769
2981
|
catch (err) {
|
|
2770
2982
|
// Nothing has moved: fail the call here, audited. An ON must not switch a scope on
|
|
@@ -2773,92 +2985,92 @@ export class CloudflareScopeHost {
|
|
|
2773
2985
|
await this.recordAdmin(actor, action, target, null, { ...base, phase: 'failed', error: errorOf(err) }).catch(() => undefined);
|
|
2774
2986
|
throw err;
|
|
2775
2987
|
}
|
|
2776
|
-
|
|
2777
|
-
|
|
2778
|
-
|
|
2779
|
-
|
|
2780
|
-
|
|
2781
|
-
|
|
2782
|
-
|
|
2988
|
+
// #2045: a newer call on this subject has recorded its position already: this one writes
|
|
2989
|
+
// nothing, here or in the scope, and says so.
|
|
2990
|
+
if (recordWriteSuperseded(prior, record)) {
|
|
2991
|
+
await this.recordAdmin(actor, action, target, null, { ...base, phase: 'refused', superseded: true });
|
|
2992
|
+
throw substratError('conflict', switchSupersededMessage(kind, scopeId, key, to));
|
|
2993
|
+
}
|
|
2994
|
+
let outcome;
|
|
2995
|
+
try {
|
|
2783
2996
|
try {
|
|
2784
|
-
await
|
|
2785
|
-
return null;
|
|
2997
|
+
outcome = await move(key, to, at, undefined, operationId);
|
|
2786
2998
|
}
|
|
2787
|
-
catch {
|
|
2788
|
-
|
|
2789
|
-
|
|
2790
|
-
|
|
2791
|
-
|
|
2792
|
-
|
|
2793
|
-
return errorOf(err);
|
|
2794
|
-
}
|
|
2999
|
+
catch (first) {
|
|
3000
|
+
if (first instanceof UnattestedSwitch)
|
|
3001
|
+
throw first;
|
|
3002
|
+
// #2045 (Codex r2): one retry under this call's own fence — idempotent, and refused by
|
|
3003
|
+
// the scope if a newer call has moved it meanwhile.
|
|
3004
|
+
outcome = await move(key, to, new Date().toISOString(), undefined, operationId);
|
|
2795
3005
|
}
|
|
2796
|
-
};
|
|
2797
|
-
let outcome;
|
|
2798
|
-
try {
|
|
2799
|
-
outcome = await move(input.moduleId, to, at);
|
|
2800
3006
|
}
|
|
2801
3007
|
catch (err) {
|
|
2802
|
-
|
|
2803
|
-
|
|
2804
|
-
|
|
2805
|
-
|
|
2806
|
-
|
|
2807
|
-
const putBack = err.outcome.changed
|
|
2808
|
-
? await move(input.moduleId, 'on', new Date().toISOString(), true).then(() => true, () => false)
|
|
2809
|
-
: true;
|
|
2810
|
-
if (putBack) {
|
|
2811
|
-
const recordError = await undoRecord();
|
|
2812
|
-
await this.recordAdmin(actor, action, target, null, {
|
|
2813
|
-
...base,
|
|
2814
|
-
phase: 'refused',
|
|
2815
|
-
error: err.message,
|
|
2816
|
-
...(recordError ? { recordError } : {}),
|
|
2817
|
-
}).catch(() => undefined);
|
|
2818
|
-
throw substratError('precondition_failed', `${err.message} Nothing was switched` +
|
|
2819
|
-
(recordError ? `, but its directory record could not be put back (${recordError}).` : '.'));
|
|
2820
|
-
}
|
|
2821
|
-
}
|
|
2822
|
-
// A throw does not say the scope did not move: the move crosses the scope's DO and then
|
|
2823
|
-
// the rewind hold, and a throw from the second leaves the scope's switch moved. So the
|
|
2824
|
-
// record is undone only when the scope reads back in the OTHER position — an OFF that
|
|
2825
|
-
// reads not off, an ON that still reads off — and kept when it reads back as asked.
|
|
2826
|
-
// Undoing an ON that landed would put the record back to `off` beside a scope that is
|
|
2827
|
-
// on, and the next re-assert would switch the operator's ON back off. A read that fails
|
|
2828
|
-
// too counts as off, both ways, so what is left fails toward off: an OFF keeps its
|
|
2829
|
-
// record (never a scope switched off with no record — a wipe or restore would have
|
|
2830
|
-
// nothing to re-assert, and #1743's refusal nothing to read), and an ON is undone back
|
|
2831
|
-
// to `off`, which the next re-assert completes if the scope did move.
|
|
2832
|
-
const off = await isOff(input.moduleId).catch(() => true);
|
|
2833
|
-
const recordKept = off === (to === 'off');
|
|
2834
|
-
const recordError = recordKept ? null : await undoRecord();
|
|
2835
|
-
// Best effort: the original error is what the caller must see, and the intent row
|
|
2836
|
-
// already says an attempt was made.
|
|
3008
|
+
// The move threw twice, or was answered by a deployment that could not attest it. Either
|
|
3009
|
+
// way it may have landed, and an older call's move may still be in flight behind it, so no
|
|
3010
|
+
// readback can settle the scope and nothing is put back: the record is this call's position
|
|
3011
|
+
// under the newest fence, and its write-ahead mark makes the next re-assert move the scope
|
|
3012
|
+
// to it, under that fence, whichever way it lies now.
|
|
2837
3013
|
await this.recordAdmin(actor, action, target, null, {
|
|
2838
3014
|
...base,
|
|
2839
|
-
phase: 'failed',
|
|
3015
|
+
phase: err instanceof UnattestedSwitch ? 'refused' : 'failed',
|
|
2840
3016
|
error: errorOf(err),
|
|
2841
|
-
|
|
2842
|
-
|
|
3017
|
+
recordKept: true,
|
|
3018
|
+
reassertOwed: true,
|
|
2843
3019
|
}).catch(() => undefined);
|
|
3020
|
+
if (err instanceof UnattestedSwitch)
|
|
3021
|
+
throw substratError('precondition_failed', err.message);
|
|
2844
3022
|
throw err;
|
|
2845
3023
|
}
|
|
3024
|
+
// #2045: the scope has applied a newer call on this subject, so this move wrote nothing. The
|
|
3025
|
+
// record is that newer call's too (its write overwrote this one's, mark included), so nothing
|
|
3026
|
+
// is undone and no mark is this call's to clear.
|
|
3027
|
+
if (outcome.superseded) {
|
|
3028
|
+
await this.recordAdmin(actor, action, target, null, { ...base, phase: 'refused', superseded: true });
|
|
3029
|
+
throw substratError('conflict', switchSupersededMessage(kind, scopeId, key, to));
|
|
3030
|
+
}
|
|
2846
3031
|
// A call that held nothing moved nothing, so its record write is undone: left `off`, it
|
|
2847
|
-
// would switch the
|
|
2848
|
-
// a wiped scope would leave
|
|
2849
|
-
|
|
3032
|
+
// would switch the subject off the day it is installed; left `on`, the next reconcile of
|
|
3033
|
+
// a wiped scope would leave it running. Retried once; a failure of both lands on the
|
|
3034
|
+
// outcome row as `recordError`, and the call says so.
|
|
3035
|
+
//
|
|
3036
|
+
// The scope confirmed this call's move (or that there was nothing to move), so its mark goes.
|
|
3037
|
+
// Held nothing, it goes WITH the undo, in one directory transaction: the row put back carries
|
|
3038
|
+
// the prior call's older id, and no later re-assert confirms under a fence older than this
|
|
3039
|
+
// call's mark. An undo that fails twice leaves the record AND the mark this call's, and the
|
|
3040
|
+
// next re-assert confirms both under it. Held, a failed clear is not the call's: the mark only
|
|
3041
|
+
// costs the next re-assert one idempotent move under this same record.
|
|
3042
|
+
//
|
|
3043
|
+
// An unfenced OFF that held something (a deployment built before the fence) confirmed nothing
|
|
3044
|
+
// under its fence, so its mark stays for the first re-assert after the vertical is redeployed.
|
|
3045
|
+
const unfenced = outcome.held && outcome.fenced !== true;
|
|
3046
|
+
let recordError = null;
|
|
3047
|
+
let owedError = null;
|
|
3048
|
+
if (!outcome.held) {
|
|
3049
|
+
const undo = () => this.cp.restoreSwitchRecord(record, prior, true);
|
|
3050
|
+
recordError = await undo().then(() => null, () => undo().then(() => null, errorOf));
|
|
3051
|
+
}
|
|
3052
|
+
else if (!unfenced) {
|
|
3053
|
+
owedError = await this.cp.clearSwitchOwed(kind, tenantId, scopeId, key, operationId).then(() => null, errorOf);
|
|
3054
|
+
}
|
|
2850
3055
|
await this.recordAdmin(actor, action, target, null, {
|
|
2851
3056
|
...base,
|
|
2852
3057
|
phase: outcome.held ? 'applied' : 'refused',
|
|
2853
3058
|
changed: outcome.changed,
|
|
2854
3059
|
permissions: outcome.permissions,
|
|
3060
|
+
...(unfenced ? { unfenced: true, reassertOwed: true } : {}),
|
|
2855
3061
|
...(recordError ? { recordError } : {}),
|
|
3062
|
+
...(owedError ? { owedError } : {}),
|
|
2856
3063
|
});
|
|
2857
3064
|
if (!outcome.held) {
|
|
2858
|
-
throw substratError('not_found',
|
|
2859
|
-
`(check the module id: it is the module's manifest id, e.g. '@substrat-run/engine-absence')` +
|
|
3065
|
+
throw substratError('not_found', switchNotFoundMessage(kind, scopeId, key, to) +
|
|
2860
3066
|
(recordError ? `; and its directory record could not be put back (${recordError})` : ''));
|
|
2861
3067
|
}
|
|
3068
|
+
return { operationId, outcome };
|
|
3069
|
+
};
|
|
3070
|
+
/** #1666: move one module's schedule switch on one scope — see `HostAdmin.revokeFromSystem`. */
|
|
3071
|
+
const switchSystem = async (actor, raw, to) => {
|
|
3072
|
+
const input = systemSwitch.parse(raw);
|
|
3073
|
+
const { operationId, outcome } = await switchSubjectAt(actor, 'system', input.node, input.moduleId, input.reason, to);
|
|
2862
3074
|
return {
|
|
2863
3075
|
operationId,
|
|
2864
3076
|
moduleId: input.moduleId,
|
|
@@ -2867,6 +3079,18 @@ export class CloudflareScopeHost {
|
|
|
2867
3079
|
permissions: outcome.permissions,
|
|
2868
3080
|
};
|
|
2869
3081
|
};
|
|
3082
|
+
/** #1706: move one PEER's kill switch on one scope — see `HostAdmin.revokeFromPeer`. */
|
|
3083
|
+
const switchPeerAt = async (actor, raw, to) => {
|
|
3084
|
+
const input = peerSwitch.parse(raw);
|
|
3085
|
+
const { operationId, outcome } = await switchSubjectAt(actor, 'peer', input.node, input.vertical, input.reason, to);
|
|
3086
|
+
return {
|
|
3087
|
+
operationId,
|
|
3088
|
+
vertical: input.vertical,
|
|
3089
|
+
calls: to,
|
|
3090
|
+
changed: outcome.changed,
|
|
3091
|
+
permissions: outcome.permissions,
|
|
3092
|
+
};
|
|
3093
|
+
};
|
|
2870
3094
|
/**
|
|
2871
3095
|
* The status read's admin-log join (#1674): the `intent` row of the `revokeFromSystem`
|
|
2872
3096
|
* still in force for each OFF module — actor, reason, when. Paired to its `applied` row
|
|
@@ -2939,7 +3163,7 @@ export class CloudflareScopeHost {
|
|
|
2939
3163
|
delegation
|
|
2940
3164
|
? delegation.status({ tenantId, scopeId })
|
|
2941
3165
|
: this.scopeStub(scopeId).systemGrantsStatus().then((rows) => systemScheduleEntry.array().parse(rows)),
|
|
2942
|
-
this.cp.
|
|
3166
|
+
this.cp.switchRecordsOf('system', tenantId, scopeId),
|
|
2943
3167
|
]);
|
|
2944
3168
|
const offModules = new Set(states.filter((s) => s.schedules === 'off').map((s) => s.moduleId));
|
|
2945
3169
|
const explanations = offModules.size > 0
|
|
@@ -2958,71 +3182,137 @@ export class CloudflareScopeHost {
|
|
|
2958
3182
|
};
|
|
2959
3183
|
/**
|
|
2960
3184
|
* #1674: put the directory's OFF positions back into one scope — see
|
|
2961
|
-
* `HostAdmin.reassertSystemSwitches`.
|
|
2962
|
-
* so it lands wherever an operator's
|
|
2963
|
-
*
|
|
3185
|
+
* `HostAdmin.reassertSystemSwitches`. Every kind (#2029): the recorded-off modules, then the
|
|
3186
|
+
* recorded-off peers. Each moves through `switchTarget`, so it lands wherever an operator's
|
|
3187
|
+
* OFF would have — for a hosted scope, over that kind's `/internal/*` switch route.
|
|
2964
3188
|
*/
|
|
2965
3189
|
const reassertSystemSwitchesOf = async (actor, node, opts) => {
|
|
3190
|
+
const results = [];
|
|
3191
|
+
for (const kind of SWITCH_KINDS)
|
|
3192
|
+
results.push(...(await reassertKind(actor, kind, node, opts)));
|
|
3193
|
+
return results;
|
|
3194
|
+
};
|
|
3195
|
+
/** One kind's half of `reassertSystemSwitchesOf`. */
|
|
3196
|
+
const reassertKind = async (actor, kind, node, opts) => {
|
|
2966
3197
|
const { tenantId, scopeId } = node;
|
|
2967
|
-
const { vertical, move } = await
|
|
2968
|
-
const
|
|
3198
|
+
const { vertical, delegated, move, attestFence } = await switchTarget(kind, tenantId, scopeId);
|
|
3199
|
+
const action = reassertActionOf(kind);
|
|
3200
|
+
const target = { tenantId, scopeId, vertical };
|
|
3201
|
+
const recorded = new Map(await this.cp.switchRecordsOf(kind, tenantId, scopeId));
|
|
3202
|
+
// #2045: every switch call's write-ahead mark, with the newest call that wrote it.
|
|
3203
|
+
const owed = new Map(await this.cp.switchesOwedOf(kind, tenantId, scopeId));
|
|
2969
3204
|
// #1742 review: moves the deployment made from a stale list (restored ON after the list
|
|
2970
3205
|
// was read), to undo before the OFF pass below.
|
|
2971
|
-
const reverts = staleCarryReverts(recorded, opts?.appliedInUnit);
|
|
3206
|
+
const reverts = staleCarryReverts(kind, recorded, opts?.appliedInUnit);
|
|
2972
3207
|
// A hosted scope with no delegation configured (Copilot review): this host's own
|
|
2973
3208
|
// namespace is the module-less placeholder, where the switch would answer `held: false`
|
|
2974
3209
|
// quietly — a re-assert reported done, and a receipt written, while the deployment
|
|
2975
|
-
// serving the scope keeps its schedules running
|
|
2976
|
-
// read is, and only when a re-assert is owed: nothing recorded,
|
|
2977
|
-
|
|
2978
|
-
|
|
3210
|
+
// serving the scope keeps its schedules running or its peer admitted. Refused loudly
|
|
3211
|
+
// instead, as the status read is, and only when a re-assert is owed: nothing recorded,
|
|
3212
|
+
// nothing to refuse.
|
|
3213
|
+
const reassertOwed = reverts.length > 0 || owed.size > 0 || [...recorded.values()].includes('off');
|
|
3214
|
+
if (reassertOwed && !this.cpLess && vertical !== null && !delegated) {
|
|
2979
3215
|
throw substratError('unavailable', `no delegation configured for hosted scope ${scopeId} (vertical '${vertical}') — cannot re-assert ` +
|
|
2980
|
-
`its switched-off
|
|
3216
|
+
`its switched-off ${DOOR_WORDS[kind].switched} in the deployment serving it`);
|
|
2981
3217
|
}
|
|
3218
|
+
// #2045 (Codex r3): a subject's mark goes only once the scope has confirmed a move under the
|
|
3219
|
+
// record's fence — a fenced answer that was not superseded. A throw anywhere below leaves
|
|
3220
|
+
// every mark not yet confirmed for the next pass.
|
|
3221
|
+
// An unfenced OFF (a deployment built before the fence) confirms nothing, so its mark stays.
|
|
3222
|
+
const confirm = async (key, fence, outcome) => {
|
|
3223
|
+
if (owed.has(key) && !outcome.superseded && outcome.fenced === true) {
|
|
3224
|
+
await this.cp.clearSwitchOwed(kind, tenantId, scopeId, key, fence);
|
|
3225
|
+
}
|
|
3226
|
+
};
|
|
3227
|
+
// #2045 follow-up: on a deployment built before the fence no ON is sent (see
|
|
3228
|
+
// `UnattestedSwitch`). Every OFF below still moves, and the pass refuses at its end, so a
|
|
3229
|
+
// sweep records no receipt for a scope whose owed ON was not sent. Asked only when an ON is due.
|
|
3230
|
+
const onWithheld = [];
|
|
3231
|
+
const withholdOn = async (key) => {
|
|
3232
|
+
if (await attestFence())
|
|
3233
|
+
return false;
|
|
3234
|
+
onWithheld.push(key);
|
|
3235
|
+
return true;
|
|
3236
|
+
};
|
|
2982
3237
|
const at = new Date().toISOString();
|
|
2983
3238
|
const reverted = new Set();
|
|
2984
|
-
for (const
|
|
3239
|
+
for (const key of reverts) {
|
|
2985
3240
|
// Re-read immediately before the move: a staff OFF that completed since the read above
|
|
2986
|
-
// (record `off`) must not get a transient ON a due schedule could
|
|
2987
|
-
|
|
2988
|
-
|
|
3241
|
+
// (record `off`) must not get a transient ON a due schedule, or a peer call, could use.
|
|
3242
|
+
// #2045: under the fence of the ON the record holds, which is the operator's own call — read
|
|
3243
|
+
// with the position, so it is that ON's fence and no other call's.
|
|
3244
|
+
const current = new Map(await this.cp.switchRecordStatesOf(kind, tenantId, scopeId)).get(key);
|
|
3245
|
+
if (current?.position !== 'on')
|
|
2989
3246
|
continue;
|
|
2990
|
-
const
|
|
2991
|
-
|
|
3247
|
+
const fence = current.fence;
|
|
3248
|
+
if (await withholdOn(key))
|
|
3249
|
+
continue;
|
|
3250
|
+
const outcome = await move(key, 'on', at, undefined, fence);
|
|
3251
|
+
reverted.add(key);
|
|
2992
3252
|
if (outcome.changed) {
|
|
2993
|
-
await this.recordAdmin(actor,
|
|
2994
|
-
operationId: ulid(),
|
|
2995
|
-
...staleCarryRevertRow(moduleId, outcome),
|
|
2996
|
-
});
|
|
3253
|
+
await this.recordAdmin(actor, action, target, null, { operationId: ulid(), ...staleCarryRevertRow(kind, key, outcome) });
|
|
2997
3254
|
}
|
|
3255
|
+
await confirm(key, fence, outcome);
|
|
2998
3256
|
}
|
|
2999
3257
|
// Read AFTER the reverts: an OFF that landed meanwhile is switched off below.
|
|
3000
|
-
const
|
|
3258
|
+
const { keys, tenantHeld, fences } = await this.recordedOff(kind, tenantId, scopeId, at);
|
|
3001
3259
|
// #1742: what the deployment already switched off inside its own unit, audited here —
|
|
3002
|
-
// the move below answers `changed: false` for it and would write no row.
|
|
3003
|
-
//
|
|
3004
|
-
const
|
|
3005
|
-
|
|
3006
|
-
await this.recordAdmin(actor, 'reassertSystemSwitch', { tenantId, scopeId, vertical }, null, {
|
|
3007
|
-
operationId: ulid(),
|
|
3008
|
-
...row,
|
|
3009
|
-
});
|
|
3260
|
+
// the move below answers `changed: false` for it and would write no row. A move the
|
|
3261
|
+
// revert above undid is not credited as an in-unit OFF.
|
|
3262
|
+
for (const row of inUnitMovesToAudit(kind, keys, opts?.appliedInUnit, reverted)) {
|
|
3263
|
+
await this.recordAdmin(actor, action, target, null, { operationId: ulid(), ...row });
|
|
3010
3264
|
}
|
|
3265
|
+
// #2045: every move here carries the fence of the call the record holds, so a scope a newer call
|
|
3266
|
+
// has moved since refuses it. A deployment from before the fence takes the OFF unfenced.
|
|
3011
3267
|
const results = [];
|
|
3012
|
-
const
|
|
3013
|
-
for (const
|
|
3014
|
-
const
|
|
3268
|
+
const held = new Set(tenantHeld);
|
|
3269
|
+
for (const key of keys) {
|
|
3270
|
+
const fence = fences.get(key); // `recordedOff` read each key with its own row's fence
|
|
3271
|
+
const outcome = await move(key, 'off', at, held.has(key), fence);
|
|
3015
3272
|
if (outcome.changed) {
|
|
3016
|
-
await this.recordAdmin(actor,
|
|
3273
|
+
await this.recordAdmin(actor, action, target, null, {
|
|
3017
3274
|
operationId: ulid(),
|
|
3018
|
-
|
|
3019
|
-
|
|
3020
|
-
phase: 'applied',
|
|
3021
|
-
changed: true,
|
|
3022
|
-
permissions: outcome.permissions,
|
|
3275
|
+
...reassertOffRow(kind, key, outcome.permissions),
|
|
3276
|
+
...(outcome.fenced === true ? {} : { unfenced: true }),
|
|
3023
3277
|
});
|
|
3024
3278
|
}
|
|
3025
|
-
results.push(
|
|
3279
|
+
results.push(reassertEntry(kind, key, outcome));
|
|
3280
|
+
await confirm(key, fence, outcome);
|
|
3281
|
+
}
|
|
3282
|
+
// A subject marked owed converges to its record in either direction: OFF above, and here a
|
|
3283
|
+
// record of ON — sent whatever the scope reads now, because a scope that reads ON may still
|
|
3284
|
+
// be behind the record's fence, with an older call's OFF in flight that only the fence can
|
|
3285
|
+
// refuse (Codex r3, finding 1). Idempotent: an ON already in place changes nothing and moves
|
|
3286
|
+
// the fence. Every other record of ON keeps #1674's rule and turns nothing on.
|
|
3287
|
+
const settled = new Set([...keys, ...reverted]);
|
|
3288
|
+
const pending = [...owed.keys()].filter((key) => !settled.has(key));
|
|
3289
|
+
if (pending.length) {
|
|
3290
|
+
// Position and fence in one read: an ON is sent only under the fence of the ON it read.
|
|
3291
|
+
const now = new Map(await this.cp.switchRecordStatesOf(kind, tenantId, scopeId));
|
|
3292
|
+
for (const key of pending) {
|
|
3293
|
+
const row = now.get(key);
|
|
3294
|
+
if (row?.position !== 'on') {
|
|
3295
|
+
// No record (a call that held nothing, whose undo removed its row), or one that turned
|
|
3296
|
+
// OFF since the read above, which the next pass moves: nothing to send on this one. A
|
|
3297
|
+
// missing row's mark goes under its own id, so a newer call's mark stays.
|
|
3298
|
+
if (row === undefined)
|
|
3299
|
+
await this.cp.clearSwitchOwed(kind, tenantId, scopeId, key, owed.get(key));
|
|
3300
|
+
continue;
|
|
3301
|
+
}
|
|
3302
|
+
const fence = row.fence;
|
|
3303
|
+
if (await withholdOn(key))
|
|
3304
|
+
continue;
|
|
3305
|
+
const outcome = await move(key, 'on', at, undefined, fence);
|
|
3306
|
+
if (outcome.changed) {
|
|
3307
|
+
await this.recordAdmin(actor, action, target, null, { operationId: ulid(), ...reassertOnRow(kind, key, outcome.permissions) });
|
|
3308
|
+
results.push(reassertEntry(kind, key, outcome));
|
|
3309
|
+
}
|
|
3310
|
+
await confirm(key, fence, outcome);
|
|
3311
|
+
}
|
|
3312
|
+
}
|
|
3313
|
+
if (onWithheld.length) {
|
|
3314
|
+
throw substratError('precondition_failed', `${unfencedMessage(scopeId)} Its recorded OFF positions were applied; the owed ON of ` +
|
|
3315
|
+
`${onWithheld.map((k) => `'${k}'`).join(', ')} was not sent.`);
|
|
3026
3316
|
}
|
|
3027
3317
|
return results;
|
|
3028
3318
|
};
|
|
@@ -3109,18 +3399,21 @@ export class CloudflareScopeHost {
|
|
|
3109
3399
|
unassignRole: async (actor, assignment) => {
|
|
3110
3400
|
// Tombstone (K-21) — the checker skips revoked rows. A no-op returns false so
|
|
3111
3401
|
// a repeat unassign stays silent (no second audit row, no needless fan-out).
|
|
3112
|
-
const subject = `principal:${assignment.principalId}`;
|
|
3113
3402
|
const relation = `role:${assignment.roleKey}`;
|
|
3114
|
-
const
|
|
3115
|
-
|
|
3116
|
-
|
|
3117
|
-
|
|
3403
|
+
const { tenantId, scopeId } = assignment.node;
|
|
3404
|
+
if (!scopeId) {
|
|
3405
|
+
// One ControlPlaneDO method (#1184): the revoke, the removal fence (raised even when
|
|
3406
|
+
// nothing was held) and the audit row, so no add can land between any two of them.
|
|
3407
|
+
const changed = await this.cp.revokeAndFence(tenantId, assignment.principalId, relation, `tenant:${tenantId}`, this.adminEntry(actor, 'unassignRole', { tenantId, scopeId: null }, assignment, null));
|
|
3408
|
+
// A tenant-level revoke changes the projected set — the tombstone must reach scopes.
|
|
3409
|
+
if (changed)
|
|
3410
|
+
await this.fanOut(tenantId);
|
|
3411
|
+
return;
|
|
3412
|
+
}
|
|
3413
|
+
const changed = await this.scopeStub(scopeId).revokeTuple(`principal:${assignment.principalId}`, relation, `scope:${scopeId}`, new Date().toISOString());
|
|
3118
3414
|
if (!changed)
|
|
3119
3415
|
return;
|
|
3120
|
-
await this.recordAdmin(actor, 'unassignRole', { tenantId
|
|
3121
|
-
// A tenant-level revoke changes the projected set — the tombstone must reach scopes.
|
|
3122
|
-
if (!assignment.node.scopeId)
|
|
3123
|
-
await this.fanOut(assignment.node.tenantId);
|
|
3416
|
+
await this.recordAdmin(actor, 'unassignRole', { tenantId, scopeId }, assignment, null);
|
|
3124
3417
|
},
|
|
3125
3418
|
grant: async (actor, raw) => {
|
|
3126
3419
|
// Parsed like its `grantToConnection`/`grantToSystem` siblings, not taken on
|
|
@@ -3242,11 +3535,17 @@ export class CloudflareScopeHost {
|
|
|
3242
3535
|
},
|
|
3243
3536
|
tenantHeldSystemModules: async (actor, tenantId, moduleIds) => {
|
|
3244
3537
|
const held = moduleIds.length
|
|
3245
|
-
? await this.cp.
|
|
3538
|
+
? await this.cp.tenantHeldOf('system', tenantId, moduleIds, new Date().toISOString())
|
|
3246
3539
|
: [];
|
|
3247
3540
|
await this.recordAccess(actor, 'tenantHeldSystemModules', { tenantId }, { moduleIds: [...moduleIds] }, held.length);
|
|
3248
3541
|
return held;
|
|
3249
3542
|
},
|
|
3543
|
+
peerSwitchCarry: async (actor, node) => {
|
|
3544
|
+
const { keys: switchedOffPeers, tenantHeld: tenantHeldPeers, fences: all, } = await this.recordedOff('peer', node.tenantId, node.scopeId, new Date().toISOString());
|
|
3545
|
+
const fences = Object.fromEntries(switchedOffPeers.map((v) => [v, all.get(v)]));
|
|
3546
|
+
await this.recordAccess(actor, 'peerSwitchCarry', node, null, switchedOffPeers.length);
|
|
3547
|
+
return { switchedOffPeers, tenantHeldPeers, fences };
|
|
3548
|
+
},
|
|
3250
3549
|
reassertSystemSwitches: reassertSystemSwitchesOf,
|
|
3251
3550
|
// #1672 — the platform's two capability verbs. Audited AFTER the write, on both, and
|
|
3252
3551
|
// the failure each leaves is the safe one: a mint whose audit row did not land never
|
|
@@ -3312,7 +3611,7 @@ export class CloudflareScopeHost {
|
|
|
3312
3611
|
// another tenant's traffic. Exception: the holder is ARCHIVED or REAPED (a
|
|
3313
3612
|
// deleted app, storage since wiped) — it has released the name, so the rebind
|
|
3314
3613
|
// reclaims it.
|
|
3315
|
-
throw
|
|
3614
|
+
throw substratError('conflict', `hostname '${parsed.hostname}' is already bound to another scope`);
|
|
3316
3615
|
}
|
|
3317
3616
|
// Exactly one canonical per (scope, surface).
|
|
3318
3617
|
if (parsed.canonical)
|
|
@@ -3333,7 +3632,7 @@ export class CloudflareScopeHost {
|
|
|
3333
3632
|
const hostname = raw.toLowerCase(); // DNS is case-insensitive; the map is normalized
|
|
3334
3633
|
const row = await this.cp.readHostname(hostname);
|
|
3335
3634
|
if (!row)
|
|
3336
|
-
throw
|
|
3635
|
+
throw substratError('not_found', `unknown hostname '${hostname}'`);
|
|
3337
3636
|
if (row.status === status)
|
|
3338
3637
|
return; // idempotent, unaudited
|
|
3339
3638
|
await this.cp.setHostnameStatus(hostname, status, note ?? null);
|
|
@@ -3343,7 +3642,7 @@ export class CloudflareScopeHost {
|
|
|
3343
3642
|
const hostname = raw.toLowerCase(); // DNS is case-insensitive; the map is normalized
|
|
3344
3643
|
const row = await this.cp.readHostname(hostname);
|
|
3345
3644
|
if (!row)
|
|
3346
|
-
throw
|
|
3645
|
+
throw substratError('not_found', `unknown hostname '${hostname}'`);
|
|
3347
3646
|
// A poll that finds nothing changed (same status, same records, id already set)
|
|
3348
3647
|
// is not an event — skip the write and the audit entry, so the reconcile sweep
|
|
3349
3648
|
// does not flood the admin log with no-op rows every interval.
|
|
@@ -3429,7 +3728,7 @@ export class CloudflareScopeHost {
|
|
|
3429
3728
|
if (existing.owner_tenant !== parsed.ownerTenant) {
|
|
3430
3729
|
throw substratError('conflict', `vertical '${parsed.slug}' is owned by ${existing.owner_tenant ?? 'the platform'}, not ${parsed.ownerTenant ?? 'the platform'}`);
|
|
3431
3730
|
}
|
|
3432
|
-
throw
|
|
3731
|
+
throw substratError('conflict', `vertical '${parsed.slug}' is already registered as ${existing.source}`);
|
|
3433
3732
|
}
|
|
3434
3733
|
await this.cp.insertVertical(parsed.slug, parsed.name, parsed.source, parsed.ownerTenant, envSpecJson, installSpecJson, parsed.listed ? 1 : 0, new Date().toISOString());
|
|
3435
3734
|
await this.recordAdmin(actor, 'registerVertical', { tenantId: null }, null, parsed);
|
|
@@ -3610,11 +3909,11 @@ export class CloudflareScopeHost {
|
|
|
3610
3909
|
// nothing to diff against — the gate is about change, not existence.
|
|
3611
3910
|
if (outgoing) {
|
|
3612
3911
|
if (outgoing.permission_digest !== incoming.permission_digest && !ack.permissionChange) {
|
|
3613
|
-
throw
|
|
3912
|
+
throw substratError('conflict', `promotion changes the permission surface (${outgoing.permission_digest} → ` +
|
|
3614
3913
|
`${incoming.permission_digest}) — acknowledge it explicitly to promote`);
|
|
3615
3914
|
}
|
|
3616
3915
|
if (outgoing.migration_digest !== incoming.migration_digest && !ack.migrationChange) {
|
|
3617
|
-
throw
|
|
3916
|
+
throw substratError('conflict', `promotion changes migrations (${outgoing.migration_digest} → ` +
|
|
3618
3917
|
`${incoming.migration_digest}) — acknowledge it explicitly to promote`);
|
|
3619
3918
|
}
|
|
3620
3919
|
// #1705 PR 3: an export an installed consumer imports, dropped or re-versioned.
|
|
@@ -3874,7 +4173,7 @@ export class CloudflareScopeHost {
|
|
|
3874
4173
|
},
|
|
3875
4174
|
createOrg: async (actor, input) => {
|
|
3876
4175
|
const parsed = createOrgInput.parse(input);
|
|
3877
|
-
const created = await this.
|
|
4176
|
+
const created = await this.directory('createOrg', parsed.id, parsed.tenantId, parsed.slug, parsed.name, new Date().toISOString());
|
|
3878
4177
|
if (!created)
|
|
3879
4178
|
return; // idempotent, and a no-op is not audited
|
|
3880
4179
|
await this.recordAdmin(actor, 'createOrg', { tenantId: parsed.tenantId }, null, parsed);
|
|
@@ -3889,21 +4188,37 @@ export class CloudflareScopeHost {
|
|
|
3889
4188
|
await this.recordAccess(actor, 'getOrg', { tenantId }, { orgId }, r ? 1 : 0);
|
|
3890
4189
|
return r ? mapOrg(r) : undefined;
|
|
3891
4190
|
},
|
|
3892
|
-
addMember: async (actor, tenantId, principal, orgId) => {
|
|
4191
|
+
addMember: async (actor, tenantId, principal, orgId, opts) => {
|
|
3893
4192
|
await requireOrg(tenantId, orgId);
|
|
3894
|
-
|
|
3895
|
-
await this.
|
|
4193
|
+
const expiresAt = opts?.expiresAt ?? null;
|
|
4194
|
+
await this.cp.writeTenantTuple(tenantId, `principal:${principal}`, 'member', `org:${orgId}`, expiresAt);
|
|
4195
|
+
await this.recordAdmin(actor, 'addMember', { tenantId }, null, memberAddedAudit(principal, orgId, expiresAt));
|
|
3896
4196
|
await this.fanOut(tenantId); // membership is a tenant-level tuple
|
|
3897
4197
|
},
|
|
4198
|
+
applyMembership: async (actor, change) => {
|
|
4199
|
+
// The row is minted here, where attribution and `causedBy` live, and written by the
|
|
4200
|
+
// ControlPlaneDO in the same synchronous method as the fence, the bound and the tuple
|
|
4201
|
+
// (#1184): one DO unit, so nothing lands between the check and the write.
|
|
4202
|
+
const { tenantId, principal, roleKey, op, orgId } = change;
|
|
4203
|
+
const assignment = { principalId: principal, roleKey, node: { tenantId, scopeId: null } };
|
|
4204
|
+
const result = await this.cp.applyMembership(change, op === 'add'
|
|
4205
|
+
? this.adminEntry(actor, 'assignRole', { tenantId, scopeId: null }, null, assignment)
|
|
4206
|
+
: this.adminEntry(actor, 'unassignRole', { tenantId, scopeId: null }, assignment, null),
|
|
4207
|
+
// #2047: the org's own row, when the change joins or leaves one. The unit fills in its
|
|
4208
|
+
// `before`/`after` and writes it only for what it does — a join's carries the expiry.
|
|
4209
|
+
orgId ? this.adminEntry(actor, op === 'add' ? 'addMember' : 'removeMember', { tenantId }, null, null) : undefined);
|
|
4210
|
+
// The tenant-level tuple, or its tombstone, reaches the projections.
|
|
4211
|
+
if (result.applied && (op === 'add' || result.changed))
|
|
4212
|
+
await this.fanOut(tenantId);
|
|
4213
|
+
return result;
|
|
4214
|
+
},
|
|
3898
4215
|
removeMember: async (actor, tenantId, principal, orgId) => {
|
|
3899
4216
|
await requireOrg(tenantId, orgId);
|
|
3900
|
-
// Tombstone (K-21), never DELETE
|
|
3901
|
-
//
|
|
3902
|
-
const changed = await this.cp.
|
|
3903
|
-
if (
|
|
3904
|
-
|
|
3905
|
-
await this.recordAdmin(actor, 'removeMember', { tenantId }, { principal, orgId }, null);
|
|
3906
|
-
await this.fanOut(tenantId); // the tombstone must reach the projections
|
|
4217
|
+
// Tombstone (K-21), never DELETE, with the removal fence in the same DO method
|
|
4218
|
+
// (#1184). A repeat revoke stays a silent no-op rather than a second audit row.
|
|
4219
|
+
const changed = await this.cp.revokeAndFence(tenantId, principal, 'member', `org:${orgId}`, this.adminEntry(actor, 'removeMember', { tenantId }, { principal, orgId }, null));
|
|
4220
|
+
if (changed)
|
|
4221
|
+
await this.fanOut(tenantId); // the tombstone must reach the projections
|
|
3907
4222
|
},
|
|
3908
4223
|
listMembers: async (actor, tenantId, orgId, options) => {
|
|
3909
4224
|
await requireOrg(tenantId, orgId);
|
|
@@ -3913,24 +4228,25 @@ export class CloudflareScopeHost {
|
|
|
3913
4228
|
principal: r.subject.slice('principal:'.length),
|
|
3914
4229
|
orgId,
|
|
3915
4230
|
revokedAt: r.revoked_at,
|
|
4231
|
+
expiresAt: r.expires_at,
|
|
3916
4232
|
}));
|
|
3917
4233
|
},
|
|
3918
4234
|
createTenant: async (actor, input) => {
|
|
3919
4235
|
const parsed = createTenantInput.parse(input);
|
|
3920
|
-
const created = await this.
|
|
4236
|
+
const created = await this.directory('createTenant', parsed.id, parsed.slug, parsed.name, new Date().toISOString(), parsed.provisionedByTenant ?? null);
|
|
3921
4237
|
// Idempotent: re-creating an existing tenant is a no-op, not audited.
|
|
3922
4238
|
if (!created)
|
|
3923
4239
|
return;
|
|
3924
4240
|
await this.recordAdmin(actor, 'createTenant', { tenantId: parsed.id }, null, created);
|
|
3925
4241
|
},
|
|
3926
4242
|
setTenantStatus: async (actor, tenantId, status) => {
|
|
3927
|
-
const before = await this.
|
|
4243
|
+
const before = await this.directory('setTenantStatus', tenantId, status);
|
|
3928
4244
|
await this.recordAdmin(actor, 'setTenantStatus', { tenantId }, { status: before }, { status });
|
|
3929
4245
|
// #1713: every hosted scope under the tenant holds or resumes its work by this.
|
|
3930
4246
|
await this.deliverLifecycles(actor, { tenantId });
|
|
3931
4247
|
},
|
|
3932
4248
|
setTenantName: async (actor, tenantId, name) => {
|
|
3933
|
-
const before = await this.
|
|
4249
|
+
const before = await this.directory('setTenantName', tenantId, name);
|
|
3934
4250
|
if (before === name)
|
|
3935
4251
|
return; // no-op is not audited — nothing changed
|
|
3936
4252
|
await this.recordAdmin(actor, 'setTenantName', { tenantId }, { name: before }, { name });
|
|
@@ -3940,7 +4256,7 @@ export class CloudflareScopeHost {
|
|
|
3940
4256
|
// first (archive-if-needed → reapScope in the vertical deployment); the DO clears
|
|
3941
4257
|
// the tenant's PII/config rows and flips the row to a `reaped` tombstone, keeping
|
|
3942
4258
|
// the row + admin log. Only a `deleting` tenant may be reaped (checked in the DO).
|
|
3943
|
-
const before = await this.
|
|
4259
|
+
const before = await this.directory('reapTenant', tenantId);
|
|
3944
4260
|
await this.recordAdmin(actor, 'reapTenant', { tenantId }, { status: before }, { status: 'reaped' });
|
|
3945
4261
|
},
|
|
3946
4262
|
listTenants: async (actor, page) => {
|
|
@@ -4024,7 +4340,7 @@ export class CloudflareScopeHost {
|
|
|
4024
4340
|
// Not `peerScopeGate`: a read must not migrate, and the phase makes this call once
|
|
4025
4341
|
// per edge per pass, where an extra round trip is a fleet-wide cost.
|
|
4026
4342
|
this.assertServedHere(await this.scopeRecordForRead(tenantId, scopeId), scopeId, 'readExportedEvents');
|
|
4027
|
-
const batch =
|
|
4343
|
+
const batch = await this.readExportsThroughDoor(input, tenantId, scopeId);
|
|
4028
4344
|
await this.recordAccess(actor, 'readExportedEvents', { tenantId, scopeId }, { consumer: input.consumer, after: input.after, limit: input.limit }, batch.events.length);
|
|
4029
4345
|
return batch;
|
|
4030
4346
|
},
|
|
@@ -4204,12 +4520,18 @@ export class CloudflareScopeHost {
|
|
|
4204
4520
|
await this.recordAccess(actor, 'lifecycleFlow', { tenantId, scopeId }, delegatedReadParams.lifecycleFlow(input), flow.observation.events);
|
|
4205
4521
|
return flow;
|
|
4206
4522
|
},
|
|
4523
|
+
operationSeries: async (actor, tenantId, scopeId, input) => {
|
|
4524
|
+
await this.scopeRecordForRead(tenantId, scopeId);
|
|
4525
|
+
const series = await this.scopeStub(scopeId).operationSeries(input);
|
|
4526
|
+
await this.recordAccess(actor, 'operationSeries', { tenantId, scopeId }, delegatedReadParams.operationSeries(input), operationSeriesCount(series));
|
|
4527
|
+
return series;
|
|
4528
|
+
},
|
|
4207
4529
|
readScopeTable: async (actor, tenantId, scopeId, input) => {
|
|
4208
4530
|
await this.scopeRecordForRead(tenantId, scopeId);
|
|
4209
4531
|
// Checked here as well as in the ScopeDO, so the refusal keeps its code across the hop.
|
|
4210
4532
|
assertRowLimit('limit', input.limit);
|
|
4211
4533
|
assertRowOffset('offset', input.offset);
|
|
4212
|
-
const page = await this.scopeStub(scopeId).
|
|
4534
|
+
const page = unwrapReply(await this.scopeStub(scopeId).introspectTableReply(input.table, input.limit, input.offset));
|
|
4213
4535
|
await this.recordAccess(actor, 'readScopeTable', { tenantId, scopeId }, { table: input.table, limit: page.limit, offset: page.offset }, page.rows.length);
|
|
4214
4536
|
return page;
|
|
4215
4537
|
},
|
|
@@ -4250,7 +4572,7 @@ export class CloudflareScopeHost {
|
|
|
4250
4572
|
},
|
|
4251
4573
|
queryScope: async (actor, tenantId, scopeId, input) => {
|
|
4252
4574
|
await this.scopeRecordForRead(tenantId, scopeId);
|
|
4253
|
-
const result = await this.scopeStub(scopeId).
|
|
4575
|
+
const result = unwrapReply(await this.scopeStub(scopeId).introspectQueryReply(input.sql));
|
|
4254
4576
|
// The statement is the logged argument: the access log is the evidence trail,
|
|
4255
4577
|
// and for a console read the SQL is the whole story.
|
|
4256
4578
|
await this.recordAccess(actor, 'queryScope', { tenantId, scopeId }, { sql: input.sql }, result.rows.length);
|
|
@@ -4343,7 +4665,7 @@ export class CloudflareScopeHost {
|
|
|
4343
4665
|
if (!rec)
|
|
4344
4666
|
throw substratError('not_found', `unknown scope ${scopeId} in tenant ${tenantId}`);
|
|
4345
4667
|
if (rec.status !== 'archived') {
|
|
4346
|
-
throw
|
|
4668
|
+
throw substratError('conflict', `scope ${scopeId} is ${rec.status}, not archived — only an archived scope may be reaped`);
|
|
4347
4669
|
}
|
|
4348
4670
|
// A serving scope always holds ≥1 bound hostname; a truly-dead one has been
|
|
4349
4671
|
// unbound. The dashboard delete path unbinds AT archive, but a bare console
|
|
@@ -4355,7 +4677,7 @@ export class CloudflareScopeHost {
|
|
|
4355
4677
|
// being released anyway; the interactive per-scope reap never sets it.
|
|
4356
4678
|
const bound = opts?.force ? [] : await this.cp.listHostnames({ scopeId, limit: 1 });
|
|
4357
4679
|
if (bound.length > 0) {
|
|
4358
|
-
throw
|
|
4680
|
+
throw substratError('conflict', `scope ${scopeId} still resolves hostname '${bound[0].hostname}' — ` +
|
|
4359
4681
|
`unbind it before reaping (reap wipes storage and cannot be undone)`);
|
|
4360
4682
|
}
|
|
4361
4683
|
await this.scopeStub(scopeId).destroyStorage();
|
|
@@ -4553,7 +4875,7 @@ export class CloudflareScopeHost {
|
|
|
4553
4875
|
},
|
|
4554
4876
|
registerIdentityPool: async (actor, input) => {
|
|
4555
4877
|
const parsed = identityPool.parse(input);
|
|
4556
|
-
const created = await this.
|
|
4878
|
+
const created = await this.directory('registerIdentityPool', parsed.provider, parsed.topology, parsed.tenantId, new Date().toISOString());
|
|
4557
4879
|
if (!created)
|
|
4558
4880
|
return; // identical registration is idempotent, unaudited
|
|
4559
4881
|
// Null tenant for a central pool: it belongs to no single tenant, which is
|
|
@@ -4570,9 +4892,9 @@ export class CloudflareScopeHost {
|
|
|
4570
4892
|
listIdentityTenants: async (actor, provider, externalId) => {
|
|
4571
4893
|
const r = await this.cp.readPool(provider);
|
|
4572
4894
|
if (!r)
|
|
4573
|
-
throw
|
|
4895
|
+
throw substratError('not_found', `identity pool '${provider}' is not registered`);
|
|
4574
4896
|
if (r.topology !== 'central') {
|
|
4575
|
-
throw
|
|
4897
|
+
throw substratError('forbidden', `identity pool '${provider}' is tenant-bound — enumerating tenants is only ` +
|
|
4576
4898
|
`meaningful on a central pool, where the same externalId is the same person`);
|
|
4577
4899
|
}
|
|
4578
4900
|
const tenants = (await this.cp.identityTenants(provider, externalId));
|
|
@@ -4589,9 +4911,9 @@ export class CloudflareScopeHost {
|
|
|
4589
4911
|
at: new Date().toISOString(),
|
|
4590
4912
|
});
|
|
4591
4913
|
if (topology === null)
|
|
4592
|
-
throw
|
|
4914
|
+
throw substratError('not_found', `identity pool '${provider}' is not registered`);
|
|
4593
4915
|
if (topology !== 'central') {
|
|
4594
|
-
throw
|
|
4916
|
+
throw substratError('forbidden', `identity pool '${provider}' is tenant-bound — enumerating tenants is only ` +
|
|
4595
4917
|
`meaningful on a central pool, where the same externalId is the same person`);
|
|
4596
4918
|
}
|
|
4597
4919
|
return memberships.map((m) => identityMembership.parse(m));
|
|
@@ -4744,14 +5066,14 @@ export class CloudflareScopeHost {
|
|
|
4744
5066
|
const parsed = identityLink.parse(input);
|
|
4745
5067
|
const pool = await this.cp.readPool(parsed.provider);
|
|
4746
5068
|
if (!pool) {
|
|
4747
|
-
throw
|
|
5069
|
+
throw substratError('conflict', `identity pool '${parsed.provider}' is not registered — a pool must declare ` +
|
|
4748
5070
|
`its topology before it may link (central vs tenant-bound decides whether ` +
|
|
4749
5071
|
`the same externalId in two tenants is one person or two)`);
|
|
4750
5072
|
}
|
|
4751
5073
|
if (pool.topology === 'tenant-bound' && pool.tenant_id !== parsed.tenantId) {
|
|
4752
|
-
throw
|
|
5074
|
+
throw substratError('conflict', `identity pool '${parsed.provider}' is bound to tenant ${pool.tenant_id} and cannot link into ${parsed.tenantId}`);
|
|
4753
5075
|
}
|
|
4754
|
-
const changed = await this.
|
|
5076
|
+
const changed = await this.directory('linkIdentity', parsed.provider, parsed.externalId, parsed.principal, parsed.tenantId, parsed.scopeId ?? null, new Date().toISOString());
|
|
4755
5077
|
// Idempotent: an identity already bound is a no-op, not audited.
|
|
4756
5078
|
if (!changed)
|
|
4757
5079
|
return;
|
|
@@ -4800,6 +5122,11 @@ export class CloudflareScopeHost {
|
|
|
4800
5122
|
const { tenantId, scopeId, ...after } = ownerTransferAudit.parse(entry);
|
|
4801
5123
|
await this.recordAdmin(actor, 'transferOwner', { tenantId, scopeId }, null, after);
|
|
4802
5124
|
},
|
|
5125
|
+
/** #1150: one phase of a dashboard member change, written around the vertical's own. */
|
|
5126
|
+
recordMemberChange: async (actor, entry) => {
|
|
5127
|
+
const { tenantId, scopeId, ...after } = memberChangeAudit.parse(entry);
|
|
5128
|
+
await this.recordAdmin(actor, 'manageScopeMember', { tenantId, scopeId }, null, after);
|
|
5129
|
+
},
|
|
4803
5130
|
/** #2005: one change to a scope's copy marker, written around the vertical's own change. */
|
|
4804
5131
|
recordCopyMark: async (actor, entry) => {
|
|
4805
5132
|
const { tenantId, scopeId, action, ...after } = copyMarkAudit.parse(entry);
|
|
@@ -4956,6 +5283,8 @@ export class CloudflareScopeHost {
|
|
|
4956
5283
|
},
|
|
4957
5284
|
// Checked here as well as in the directory, so the refusal keeps its code across the hop.
|
|
4958
5285
|
pruneTelemetry: async (_actor, limit) => this.cp.pruneTelemetry(assertRowLimit('limit', limit)),
|
|
5286
|
+
// #1748: the DO writes each stale resolution's audit row in the same unit, from this one.
|
|
5287
|
+
pruneFindings: async (actor, limit) => this.cp.pruneFindings(assertRowLimit('limit', limit), this.adminEntry(actor, 'resolveStaleFinding', { tenantId: null }, null, null)),
|
|
4959
5288
|
listIssues: async (actor, filter) => {
|
|
4960
5289
|
const rows = await this.cp.listIssues({
|
|
4961
5290
|
status: filter?.status,
|
|
@@ -4977,6 +5306,33 @@ export class CloudflareScopeHost {
|
|
|
4977
5306
|
await this.recordAdmin(actor, 'setIssueStatus', { tenantId: null, vertical: after.lastVertical }, { fingerprint, status: before.status }, { fingerprint, status: after.status });
|
|
4978
5307
|
return after;
|
|
4979
5308
|
},
|
|
5309
|
+
listFindings: async (actor, filter) => {
|
|
5310
|
+
const rows = await this.cp.listFindings({ ...filter });
|
|
5311
|
+
await this.recordAccess(actor, 'listFindings', { tenantId: filter?.tenantId ?? null }, filter, rows.length);
|
|
5312
|
+
return rows.map((r) => findingEntry.parse(r));
|
|
5313
|
+
},
|
|
5314
|
+
// #1748: each findings mutation writes its audit row in the DO's own unit, from the row
|
|
5315
|
+
// minted here (actor, attribution) — so a failed audit write rolls the mutation back.
|
|
5316
|
+
setFindingStatus: async (actor, tenantId, id, status) => {
|
|
5317
|
+
const audit = this.adminEntry(actor, 'setFindingStatus', { tenantId }, null, null);
|
|
5318
|
+
const change = await this.cp.setFindingStatus(tenantId, id, status, new Date().toISOString(), audit);
|
|
5319
|
+
return change ? findingEntry.parse(change.after) : undefined;
|
|
5320
|
+
},
|
|
5321
|
+
createFindingRule: async (actor, tenantId, input) => {
|
|
5322
|
+
const audit = this.adminEntry(actor, 'createFindingRule', { tenantId }, null, null);
|
|
5323
|
+
const created = await this.directory('createFindingRule', tenantId, input, actor, new Date().toISOString(), audit);
|
|
5324
|
+
return { rule: findingRuleEntry.parse(created.rule), suppressed: created.suppressed.length };
|
|
5325
|
+
},
|
|
5326
|
+
revokeFindingRule: async (actor, tenantId, ruleId) => {
|
|
5327
|
+
const audit = this.adminEntry(actor, 'revokeFindingRule', { tenantId }, null, null);
|
|
5328
|
+
const change = await this.cp.revokeFindingRule(tenantId, ruleId, new Date().toISOString(), audit);
|
|
5329
|
+
return change ? findingRuleEntry.parse(change.after) : undefined;
|
|
5330
|
+
},
|
|
5331
|
+
listFindingRules: async (actor, tenantId, filter) => {
|
|
5332
|
+
const rows = await this.cp.listFindingRules(tenantId, filter?.active ? new Date().toISOString() : undefined, filter?.limit);
|
|
5333
|
+
await this.recordAccess(actor, 'listFindingRules', { tenantId }, filter, rows.length);
|
|
5334
|
+
return rows.map((r) => findingRuleEntry.parse(r));
|
|
5335
|
+
},
|
|
4980
5336
|
recordModelUsage: async (input) => {
|
|
4981
5337
|
const l = input.line;
|
|
4982
5338
|
return this.cp.recordModelUsage({
|
|
@@ -5013,6 +5369,10 @@ export class CloudflareScopeHost {
|
|
|
5013
5369
|
};
|
|
5014
5370
|
}
|
|
5015
5371
|
// -- helpers --------------------------------------------------------------
|
|
5372
|
+
/** A directory write whose refusal must keep its code across the DO hop (#113). */
|
|
5373
|
+
async directory(method, ...args) {
|
|
5374
|
+
return unwrapReply(await this.cp.reply(method, args));
|
|
5375
|
+
}
|
|
5016
5376
|
/**
|
|
5017
5377
|
* The getScope gate (control-plane.md §4.1/§4.2) — the pair check, then the tenant's and
|
|
5018
5378
|
* the scope's lifecycle — thrown on THIS side of the RPC (#1718). The ControlPlaneDO
|
|
@@ -5032,8 +5392,11 @@ export class CloudflareScopeHost {
|
|
|
5032
5392
|
*/
|
|
5033
5393
|
async validateScopeAccess(tenantId, scopeId) {
|
|
5034
5394
|
const refusal = await this.cp.scopeAccessRefusal(tenantId, scopeId);
|
|
5035
|
-
if (refusal)
|
|
5036
|
-
throw refusal.code
|
|
5395
|
+
if (refusal) {
|
|
5396
|
+
throw refusal.code
|
|
5397
|
+
? substratError(refusal.code, refusal.message, refusal.reason ? { reason: refusal.reason } : undefined)
|
|
5398
|
+
: new Error(refusal.message);
|
|
5399
|
+
}
|
|
5037
5400
|
if (!this.cpLess)
|
|
5038
5401
|
return null;
|
|
5039
5402
|
const { verdict, lifecycle } = await this.scopeStub(scopeId).admission(tenantId);
|
|
@@ -5065,7 +5428,7 @@ export class CloudflareScopeHost {
|
|
|
5065
5428
|
async assertLive(tenantId, scopeId) {
|
|
5066
5429
|
const refusal = lifecycleRefusal(await this.validateScopeAccess(tenantId, scopeId), { tenantId, scopeId });
|
|
5067
5430
|
if (refusal)
|
|
5068
|
-
throw
|
|
5431
|
+
throw substratError('conflict', refusal, { reason: SCOPE_GATE_REASONS.notActive });
|
|
5069
5432
|
}
|
|
5070
5433
|
/**
|
|
5071
5434
|
* Deliver the directory's lifecycle to the deployments serving the matching scopes (#1713):
|
|
@@ -5262,7 +5625,7 @@ export class CloudflareScopeHost {
|
|
|
5262
5625
|
if (!row)
|
|
5263
5626
|
throw unknownScopeForTenant(tenantId, scopeId);
|
|
5264
5627
|
if (row.status === 'reaped') {
|
|
5265
|
-
throw
|
|
5628
|
+
throw substratError('conflict', `scope ${scopeId} is reaped — its storage is gone and cannot be read`);
|
|
5266
5629
|
}
|
|
5267
5630
|
return row;
|
|
5268
5631
|
}
|
|
@@ -5295,7 +5658,14 @@ export class CloudflareScopeHost {
|
|
|
5295
5658
|
});
|
|
5296
5659
|
}
|
|
5297
5660
|
async recordAdmin(actor, action, target, before, after) {
|
|
5298
|
-
await this.cp.recordAdmin(
|
|
5661
|
+
await this.cp.recordAdmin(this.adminEntry(actor, action, target, before, after));
|
|
5662
|
+
}
|
|
5663
|
+
/**
|
|
5664
|
+
* One admin-log row, minted here where attribution and `causedBy` live — for `recordAdmin`,
|
|
5665
|
+
* and for a ControlPlaneDO method that writes its row in the same unit as its effect (#1184).
|
|
5666
|
+
*/
|
|
5667
|
+
adminEntry(actor, action, target, before, after) {
|
|
5668
|
+
return {
|
|
5299
5669
|
id: ulid(),
|
|
5300
5670
|
actor,
|
|
5301
5671
|
action,
|
|
@@ -5307,7 +5677,7 @@ export class CloudflareScopeHost {
|
|
|
5307
5677
|
before: before ?? null,
|
|
5308
5678
|
after: after ?? null,
|
|
5309
5679
|
at: new Date().toISOString(),
|
|
5310
|
-
}
|
|
5680
|
+
};
|
|
5311
5681
|
}
|
|
5312
5682
|
/**
|
|
5313
5683
|
* The scope's own answer to "what may this connection do here" (#726 gap 1).
|
|
@@ -5432,6 +5802,11 @@ export class CloudflareScopeHost {
|
|
|
5432
5802
|
// rewind CLAIMS them on `SWITCH_HOLDS_NAME`, which the rewind cannot reach. A module is held
|
|
5433
5803
|
// while any claim on it exists, and `runDueSchedules` skips a held module.
|
|
5434
5804
|
//
|
|
5805
|
+
// #2029: the same for a PEER switched off (`vertical:<slug>`): the rewind claims the scope's
|
|
5806
|
+
// off peers beside its off modules, each under its hold key (`holdKeyOf`), and the peer door
|
|
5807
|
+
// refuses a held peer as the system door refuses a held module. Everything below holds for
|
|
5808
|
+
// both kinds; "module" reads as "subject".
|
|
5809
|
+
//
|
|
5435
5810
|
// A claim is released only by a switch move KNOWN to survive its rewind (`switchInScope`): a
|
|
5436
5811
|
// move on the restored storage, after the rewind armed. A move during the settle, or one the
|
|
5437
5812
|
// doomed instance applied, lands in storage the restart discards, so it releases nothing. A
|
|
@@ -5450,7 +5825,7 @@ export class CloudflareScopeHost {
|
|
|
5450
5825
|
* reopens only the gap this closes, for a rewind that lands during the outage, and the next
|
|
5451
5826
|
* reconcile still switches that module off. The error is answered, for the pass to report.
|
|
5452
5827
|
*/
|
|
5453
|
-
async switchHeld(scopeId,
|
|
5828
|
+
async switchHeld(scopeId, holdKey) {
|
|
5454
5829
|
const at = Date.now();
|
|
5455
5830
|
let error;
|
|
5456
5831
|
if (!this.holdSnapshot || at - this.holdSnapshot.at > SWITCH_HOLD_SNAPSHOT_MS) {
|
|
@@ -5473,8 +5848,11 @@ export class CloudflareScopeHost {
|
|
|
5473
5848
|
}
|
|
5474
5849
|
}
|
|
5475
5850
|
}
|
|
5476
|
-
const
|
|
5477
|
-
|
|
5851
|
+
const snapshot = this.holdSnapshot;
|
|
5852
|
+
const held = snapshot?.held.has(`${scopeId} ${holdKey}`) ?? false;
|
|
5853
|
+
// #2029: a good read naming no row for this scope at all, for `doorGate`'s instance cache.
|
|
5854
|
+
const scopeClear = !error && snapshot?.good === true && !snapshot.scopes.has(scopeId);
|
|
5855
|
+
return { held, ...(error ? { error } : {}), ...(scopeClear ? { scopeClear: true } : {}) };
|
|
5478
5856
|
}
|
|
5479
5857
|
/** One read of the whole hold into the snapshot, stamped with when it went out. Never throws. */
|
|
5480
5858
|
async refreshHolds(at) {
|
|
@@ -5482,7 +5860,12 @@ export class CloudflareScopeHost {
|
|
|
5482
5860
|
const rows = await this.switchHoldsStub().switchHoldsAll();
|
|
5483
5861
|
// A slow read that lands after a newer one never replaces it.
|
|
5484
5862
|
if (!this.holdSnapshot || at >= this.holdSnapshot.at) {
|
|
5485
|
-
this.holdSnapshot = {
|
|
5863
|
+
this.holdSnapshot = {
|
|
5864
|
+
at,
|
|
5865
|
+
held: new Set(rows.map((r) => `${r.scopeId} ${r.moduleId}`)),
|
|
5866
|
+
scopes: new Set(rows.map((r) => r.scopeId)),
|
|
5867
|
+
good: true,
|
|
5868
|
+
};
|
|
5486
5869
|
}
|
|
5487
5870
|
return {};
|
|
5488
5871
|
}
|
|
@@ -5493,8 +5876,10 @@ export class CloudflareScopeHost {
|
|
|
5493
5876
|
// than hammered per scope. For that long, other scopes reuse this possibly pre-hold set
|
|
5494
5877
|
// silently, and only the consult that sent this read reports the error: the declared
|
|
5495
5878
|
// fail-open, attributed once.
|
|
5496
|
-
if (!this.holdSnapshot || at >= this.holdSnapshot.at)
|
|
5497
|
-
|
|
5879
|
+
if (!this.holdSnapshot || at >= this.holdSnapshot.at) {
|
|
5880
|
+
const scopes = new Set([...(known ?? [])].map((k) => k.slice(0, k.indexOf(' '))));
|
|
5881
|
+
this.holdSnapshot = { at, held: known ?? new Set(), scopes, good: false };
|
|
5882
|
+
}
|
|
5498
5883
|
return {
|
|
5499
5884
|
error: known
|
|
5500
5885
|
? `switch hold unreadable (${why}); the ${known.size} hold(s) from this pass's last good read ` +
|
|
@@ -5541,7 +5926,11 @@ export class CloudflareScopeHost {
|
|
|
5541
5926
|
* its claims before the OFF's join lands. The joined row then holds the module the operator just
|
|
5542
5927
|
* turned ON, until the next surviving move of that switch releases it (another ON does).
|
|
5543
5928
|
*/
|
|
5544
|
-
async switchInScope(scopeId,
|
|
5929
|
+
async switchInScope(kind, scopeId, key, to, at, tenantHeld = false,
|
|
5930
|
+
/** #2045: the switch call's fence; see `SWITCH_FENCES_DDL`. */
|
|
5931
|
+
fence) {
|
|
5932
|
+
// #2029: a peer's claims live beside the modules', under its own hold key.
|
|
5933
|
+
const holdKey = holdKeyOf(kind, key);
|
|
5545
5934
|
let scopeClaims;
|
|
5546
5935
|
let readError;
|
|
5547
5936
|
try {
|
|
@@ -5550,23 +5939,30 @@ export class CloudflareScopeHost {
|
|
|
5550
5939
|
catch (err) {
|
|
5551
5940
|
readError = err;
|
|
5552
5941
|
}
|
|
5553
|
-
const
|
|
5942
|
+
const stub = this.scopeStub(scopeId);
|
|
5943
|
+
const { instance, ...outcome } = await (kind === 'system'
|
|
5944
|
+
? stub.switchSystemSchedules(key, scopeId, to, at, tenantHeld, fence)
|
|
5945
|
+
: stub.switchPeer(key, scopeId, to, at, tenantHeld, fence));
|
|
5554
5946
|
if (!scopeClaims)
|
|
5555
5947
|
throw readError;
|
|
5948
|
+
// #2045: a move the fence refused wrote nothing, so it is known to survive nothing and to
|
|
5949
|
+
// release nothing; the newer call that superseded it did whatever the hold needed.
|
|
5950
|
+
if (outcome.superseded)
|
|
5951
|
+
return outcome;
|
|
5556
5952
|
const now = Date.now();
|
|
5557
5953
|
const pendingLive = (c) => c.state === 'pending' && now - Date.parse(c.heldAt) <= SWITCH_HOLD_PENDING_MAX_MS;
|
|
5558
|
-
const before = scopeClaims.filter((c) => c.moduleId ===
|
|
5954
|
+
const before = scopeClaims.filter((c) => c.moduleId === holdKey);
|
|
5559
5955
|
if (to === 'on') {
|
|
5560
5956
|
// Every ON, claims or not: its tombstone is what keeps a rewind's stale read from claiming
|
|
5561
5957
|
// the module after this (#1839).
|
|
5562
5958
|
this.holdSnapshot = null;
|
|
5563
|
-
await this.switchHoldsStub().switchHoldOn(scopeId,
|
|
5959
|
+
await this.switchHoldsStub().switchHoldOn(scopeId, holdKey, before.map((c) => c.claimId));
|
|
5564
5960
|
}
|
|
5565
5961
|
else if (outcome.held && before.length > 0) {
|
|
5566
5962
|
const survived = before.filter((c) => !pendingLive(c) && c.doomed !== instance);
|
|
5567
5963
|
if (survived.length > 0) {
|
|
5568
5964
|
this.holdSnapshot = null;
|
|
5569
|
-
await this.switchHoldsStub().switchHoldRelease(scopeId,
|
|
5965
|
+
await this.switchHoldsStub().switchHoldRelease(scopeId, holdKey, survived.map((c) => c.claimId));
|
|
5570
5966
|
}
|
|
5571
5967
|
}
|
|
5572
5968
|
if (to === 'off' && outcome.changed) {
|
|
@@ -5577,7 +5973,7 @@ export class CloudflareScopeHost {
|
|
|
5577
5973
|
const joins = new Set([...scopeClaims, ...after].filter((c) => pendingLive(c) || c.doomed === instance).map((c) => c.claimId));
|
|
5578
5974
|
if (joins.size > 0) {
|
|
5579
5975
|
this.holdSnapshot = null;
|
|
5580
|
-
await this.switchHoldsStub().switchHoldJoin(scopeId,
|
|
5976
|
+
await this.switchHoldsStub().switchHoldJoin(scopeId, holdKey, [...joins]);
|
|
5581
5977
|
}
|
|
5582
5978
|
}
|
|
5583
5979
|
return outcome;
|
|
@@ -5641,11 +6037,16 @@ export class CloudflareScopeHost {
|
|
|
5641
6037
|
await holds.switchHoldArm(scopeId, claimId, result.instance ?? null).catch(() => undefined);
|
|
5642
6038
|
return { rewindingTo: result.rewindingTo };
|
|
5643
6039
|
}
|
|
5644
|
-
/**
|
|
6040
|
+
/**
|
|
6041
|
+
* The subjects this scope's own storage has switched off, as a rewind reads them before it arms:
|
|
6042
|
+
* its modules, and (#2029) its peers, each by its hold key (`holdKeyOf`).
|
|
6043
|
+
*/
|
|
5645
6044
|
async offInStorage(scopeId) {
|
|
5646
|
-
|
|
5647
|
-
|
|
5648
|
-
.map((e) => e.moduleId)
|
|
6045
|
+
const [modules, peers] = await Promise.all([this.systemGrantsStatusLocal(scopeId), this.peerGrantsStatusLocal(scopeId)]);
|
|
6046
|
+
return [
|
|
6047
|
+
...modules.filter((e) => e.schedules === 'off').map((e) => holdKeyOf('system', e.moduleId)),
|
|
6048
|
+
...peers.filter((e) => e.calls === 'off').map((e) => holdKeyOf('peer', e.vertical)),
|
|
6049
|
+
];
|
|
5649
6050
|
}
|
|
5650
6051
|
/**
|
|
5651
6052
|
* #1839: wait until every row of this rewind's claim is `SWITCH_HOLD_SETTLE_MS` old. The settle
|
|
@@ -5714,7 +6115,11 @@ export class CloudflareScopeHost {
|
|
|
5714
6115
|
* arrive as a different class of answer just because the caller asked for a socket.
|
|
5715
6116
|
*/
|
|
5716
6117
|
liveReads = {
|
|
5717
|
-
subscribe: async ({ tenantId, scopeId, principal, request }) => {
|
|
6118
|
+
subscribe: async ({ tenantId, scopeId, principal, request, within }) => {
|
|
6119
|
+
// Decided before anything else, and thrown: a `within` that is neither a plain
|
|
6120
|
+
// EntityRef nor a value `vouchedWithin` built is a caller bug, and the two ways of
|
|
6121
|
+
// guessing at it both open a feed wider than was asked for (#1853).
|
|
6122
|
+
const narrowed = liveWithinOf(within);
|
|
5718
6123
|
if (!isUpgradeRequest(request)) {
|
|
5719
6124
|
return new Response('live reads are a WebSocket surface', {
|
|
5720
6125
|
status: 426,
|
|
@@ -5772,6 +6177,10 @@ export class CloudflareScopeHost {
|
|
|
5772
6177
|
headers.set(LIVE_PRINCIPAL_HEADER, principal);
|
|
5773
6178
|
headers.set(LIVE_TENANT_HEADER, tenantId);
|
|
5774
6179
|
headers.set(LIVE_SCOPE_HEADER, scopeId);
|
|
6180
|
+
if (narrowed)
|
|
6181
|
+
headers.set(LIVE_WITHIN_HEADER, encodeLiveWithin(narrowed));
|
|
6182
|
+
else
|
|
6183
|
+
headers.delete(LIVE_WITHIN_HEADER);
|
|
5775
6184
|
const forwarded = new Request(new URL(LIVE_SUBSCRIBE_PATH, 'https://scope.substrat.internal'),
|
|
5776
6185
|
// `new Request(url, { …, headers })` rather than `new Request(request, …)`:
|
|
5777
6186
|
// the DO is addressed by its own path, not the client's, and the upgrade
|
|
@@ -5853,7 +6262,7 @@ export class CloudflareScopeHost {
|
|
|
5853
6262
|
throw unknownScopeForTenant(tenantId, scopeId);
|
|
5854
6263
|
const { roles, tuples, entitlements, identities } = await this.tenantProjection(tenantId);
|
|
5855
6264
|
const connectionKeys = await this.connectionKeyRows(tenantId, scope.vertical);
|
|
5856
|
-
await this.scopeStub(scopeId).
|
|
6265
|
+
unwrapReply(await this.scopeStub(scopeId).applyProjectionReply(tenantId, roles, tuples, entitlements, undefined, identities, connectionKeys));
|
|
5857
6266
|
}
|
|
5858
6267
|
/**
|
|
5859
6268
|
* Fan the tenant's current state out into ALL its scopes — called after any
|
|
@@ -5902,7 +6311,7 @@ export class CloudflareScopeHost {
|
|
|
5902
6311
|
if (!keysByVertical.has(vertical)) {
|
|
5903
6312
|
keysByVertical.set(vertical, this.connectionKeyRows(tenantId, s.vertical));
|
|
5904
6313
|
}
|
|
5905
|
-
await this.scopeStub(s.scope_id).
|
|
6314
|
+
unwrapReply(await this.scopeStub(s.scope_id).applyProjectionReply(tenantId, roles, tuples, entitlements, undefined, identities, await keysByVertical.get(vertical)));
|
|
5906
6315
|
}));
|
|
5907
6316
|
const failed = settled.flatMap((r, i) => (r.status === 'rejected' ? [{ scope: scopes[i].scope_id, reason: r.reason }] : []));
|
|
5908
6317
|
if (failed.length === 0)
|
|
@@ -5932,10 +6341,11 @@ export class CloudflareScopeHost {
|
|
|
5932
6341
|
* from its own storage. No tenant-level tuples, no control plane.
|
|
5933
6342
|
*/
|
|
5934
6343
|
async provisionScopeLocal(input) {
|
|
6344
|
+
const carry = recordedOffFromWire(input);
|
|
5935
6345
|
const services = await this.servicePrincipals?.(input.tenantId, input.scopeId);
|
|
5936
6346
|
const stub = this.scopeStub(input.scopeId);
|
|
5937
6347
|
await this.migrateAndRecord(input.scopeId); // create the module tables (setMigrationState no-ops on a null CP)
|
|
5938
|
-
const switchedOff = await stub.
|
|
6348
|
+
const switchedOff = unwrapReply(await stub.applyProjectionReply(input.tenantId, input.roles.map((r) => ({ role_key: r.key, permissions: JSON.stringify(r.permissions), source: r.source })), [], // no tenant-level tuples — a CP-less vertical grants at scope level only
|
|
5939
6349
|
// Only PROJECT (and thereby switch on strict enforcement) when the platform actually
|
|
5940
6350
|
// supplies entitlements. Omitting them leaves the scope un-projected and trusting-
|
|
5941
6351
|
// upstream, so a vertical whose provision path predates #304 is never denied — it
|
|
@@ -6025,12 +6435,10 @@ export class CloudflareScopeHost {
|
|
|
6025
6435
|
public_key: k.publicKey,
|
|
6026
6436
|
}))
|
|
6027
6437
|
: undefined,
|
|
6028
|
-
// #1742: the recorded-off modules, switched off after the seat in the
|
|
6029
|
-
// scope this call provisions — the one the seat's tuples name.
|
|
6030
|
-
input.
|
|
6031
|
-
|
|
6032
|
-
: undefined, services?.map((id) => `principal:${id}`));
|
|
6033
|
-
return input.switchedOff ? { switchedOff } : {};
|
|
6438
|
+
// #1742: the recorded-off modules and (#2029) peers, switched off after the seat in the
|
|
6439
|
+
// same unit, on the scope this call provisions — the one the seat's tuples name.
|
|
6440
|
+
carry ? { scopeId: input.scopeId, at: new Date().toISOString(), ...carry } : undefined, services?.map((id) => `principal:${id}`)));
|
|
6441
|
+
return carry ? { switchedOff } : {};
|
|
6034
6442
|
}
|
|
6035
6443
|
/**
|
|
6036
6444
|
* Resolve an external identity from a scope's PROJECTED links (#406) — the CP-less
|
|
@@ -6169,6 +6577,8 @@ export class CloudflareScopeHost {
|
|
|
6169
6577
|
async connectorGrantLocal(connectionId, scopeId, permission, expiresAt) {
|
|
6170
6578
|
await this.writeScopeTuple(scopeId, subjectRef({ kind: 'connection', id: connectionId }), `granted:${permission}`, `scope:${scopeId}`, expiresAt ?? null);
|
|
6171
6579
|
}
|
|
6580
|
+
/** #2045 (Codex r3): every switch mover here honours the fence — what `/internal/switch-fence` reports. */
|
|
6581
|
+
switchFenced = true;
|
|
6172
6582
|
/**
|
|
6173
6583
|
* The far end of the schedule kill switch for a scope served HERE (#1666):
|
|
6174
6584
|
* `/internal/system-switch` lands on this, from the shared control plane's
|
|
@@ -6181,7 +6591,7 @@ export class CloudflareScopeHost {
|
|
|
6181
6591
|
// Parsed on the way out: this is the wire answer the platform reads, and the DO's
|
|
6182
6592
|
// plain strings become the published shape here rather than on trust.
|
|
6183
6593
|
// #1819: `switchInScope` releases a rewind's hold on the module once the switch lands.
|
|
6184
|
-
return systemSwitchOutcome.parse(await this.switchInScope(scopeId, moduleId, to, new Date().toISOString(), opts?.tenantHeld ?? false));
|
|
6594
|
+
return systemSwitchOutcome.parse(await this.switchInScope('system', scopeId, moduleId, to, new Date().toISOString(), opts?.tenantHeld ?? false, opts?.fence));
|
|
6185
6595
|
}
|
|
6186
6596
|
/**
|
|
6187
6597
|
* The far end of the status read for a scope served HERE (#1674): `/internal/system-grants`
|
|
@@ -6216,9 +6626,12 @@ export class CloudflareScopeHost {
|
|
|
6216
6626
|
* Move one peer's kill switch on a scope served HERE (#1706) — the far end of
|
|
6217
6627
|
* `revokeFromPeer` / `restoreToPeer` for a hosted scope. Nothing is audited here; the
|
|
6218
6628
|
* platform writes the admin rows around this call, as for the schedule switch.
|
|
6629
|
+
* #2029: through `switchInScope`, so the move releases a rewind's hold on the peer once it
|
|
6630
|
+
* lands, as a module's does. #2030: `tenantHeld`, the platform's word that the peer holds a
|
|
6631
|
+
* live tenant-level grant, which this deployment has no directory to read.
|
|
6219
6632
|
*/
|
|
6220
|
-
async peerSwitchLocal(scopeId, vertical, to) {
|
|
6221
|
-
return peerSwitchOutcome.parse(await this.
|
|
6633
|
+
async peerSwitchLocal(scopeId, vertical, to, opts) {
|
|
6634
|
+
return peerSwitchOutcome.parse(await this.switchInScope('peer', scopeId, verticalSlug.parse(vertical), to, new Date().toISOString(), opts?.tenantHeld ?? false, opts?.fence));
|
|
6222
6635
|
}
|
|
6223
6636
|
// -- the cross-vertical far ends (#1705 PR 2) --------------------------------
|
|
6224
6637
|
// The shared control plane runs the cross-vertical phase for every hosted edge and reaches the
|
|
@@ -6232,7 +6645,25 @@ export class CloudflareScopeHost {
|
|
|
6232
6645
|
async exportedEventsLocal(tenantId, scopeId, raw) {
|
|
6233
6646
|
const input = exportReadInput.parse(raw);
|
|
6234
6647
|
await this.assertServesLocally(tenantId, scopeId, 'readExportedEvents');
|
|
6235
|
-
return
|
|
6648
|
+
return this.readExportsThroughDoor(input, tenantId, scopeId);
|
|
6649
|
+
}
|
|
6650
|
+
/**
|
|
6651
|
+
* #2029: a producer's export read, through the CONSUMER's peer door: the consumer's grants on
|
|
6652
|
+
* this scope decide what leaves it, so a consumer the rewind hold keeps off must hold nothing
|
|
6653
|
+
* here, exactly as a switched-off one does. Gated and pinned like every door call; held, the
|
|
6654
|
+
* producer answers every key missing, so the edge pauses rather than reading past the hold.
|
|
6655
|
+
*/
|
|
6656
|
+
async readExportsThroughDoor(input, tenantId, scopeId) {
|
|
6657
|
+
const stub = this.scopeStub(scopeId);
|
|
6658
|
+
const door = await this.openDoor('peer', input.consumer, scopeId);
|
|
6659
|
+
try {
|
|
6660
|
+
return exportedBatch.parse(await door.through((instance) => stub.exportedEventsRead(input, tenantId, scopeId, { instance })));
|
|
6661
|
+
}
|
|
6662
|
+
catch (err) {
|
|
6663
|
+
if (!this.isHeldRefusal(err))
|
|
6664
|
+
throw err;
|
|
6665
|
+
return exportedBatch.parse(await stub.exportedEventsRead(input, tenantId, scopeId, { held: true }));
|
|
6666
|
+
}
|
|
6236
6667
|
}
|
|
6237
6668
|
/**
|
|
6238
6669
|
* The consumer's imports and watermarks (#1705), for a scope served HERE. The served-here
|