@substrat-run/adapter-cloudflare 0.135.0 → 0.137.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/checker.d.ts.map +1 -1
- package/dist/checker.js +3 -1
- package/dist/checker.js.map +1 -1
- package/dist/connector-calls.d.ts +11 -0
- package/dist/connector-calls.d.ts.map +1 -0
- package/dist/connector-calls.js +10 -0
- package/dist/connector-calls.js.map +1 -0
- package/dist/control-plane-do.d.ts +77 -3
- package/dist/control-plane-do.d.ts.map +1 -1
- package/dist/control-plane-do.js +250 -35
- package/dist/control-plane-do.js.map +1 -1
- package/dist/host.d.ts +297 -16
- package/dist/host.d.ts.map +1 -1
- package/dist/host.js +895 -132
- package/dist/host.js.map +1 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/live-reads.d.ts +5 -4
- package/dist/live-reads.d.ts.map +1 -1
- package/dist/live-reads.js +5 -4
- package/dist/live-reads.js.map +1 -1
- package/dist/r2.d.ts +2 -1
- package/dist/r2.d.ts.map +1 -1
- package/dist/r2.js.map +1 -1
- package/dist/route-resolver.d.ts +7 -0
- package/dist/route-resolver.d.ts.map +1 -1
- package/dist/route-resolver.js +4 -0
- package/dist/route-resolver.js.map +1 -1
- package/dist/scope-do.d.ts +2 -0
- package/dist/scope-do.d.ts.map +1 -1
- package/dist/scope-do.js +896 -140
- package/dist/scope-do.js.map +1 -1
- package/dist/scope-sweeper-do.d.ts +32 -6
- package/dist/scope-sweeper-do.d.ts.map +1 -1
- package/dist/scope-sweeper-do.js +35 -5
- package/dist/scope-sweeper-do.js.map +1 -1
- package/dist/system-door.d.ts +21 -0
- package/dist/system-door.d.ts.map +1 -0
- package/dist/system-door.js +13 -0
- package/dist/system-door.js.map +1 -0
- package/package.json +10 -9
package/dist/host.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
|
+
import { env as ambientEnv } from 'cloudflare:workers';
|
|
1
2
|
import { isRewindRefusal, REWIND_REFUSED } from './rewind-refusal.js';
|
|
2
|
-
import {
|
|
3
|
+
import { isSystemDoorMoved, SYSTEM_DOOR_MOVED, SYSTEM_DOOR_REGATES } from './system-door.js';
|
|
4
|
+
import { delegatedReadParams, fromWireFailure, exportReadInput, exportedBatch, importBatch, importResult, importState, importsOfManifestJson, exportsOfManifestJson, importCursorMove, importCursorMoveAt, importCursorMoved, accessLogEntry, adminLogEntry, opsFailureEntry, opsFailureFingerprint, issueEntry, sweepRunEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunsPayload, modelUsageEntry, attachmentRecord, createTenantInput, identityLink, identityPool, createOrgInput, promotionAcknowledgement, bindAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, registerVerticalInput, vertical as verticalSchema, verticalServingState, verticalChannel, verticalVersion, connection, capabilityExchange, mintedCapability, capabilityGrant, principalId, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, systemSwitch, systemSwitchOutcome, systemScheduleEntry, systemSwitchRecord, entitlementGrant, entitlementGrantInput, capabilityFilter, instant, meterReading, subjectRef, createConnectionInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, org as orgSchema, orgMembership, resolvedIdentity, identityMembership, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, delegatedReadRecord, ownerTransferAudit, copyMarkAudit, subjectShredReceipt, connectorDispatchKind, callsOfManifestJson, outboundOfManifestJson, substratError, redrainEventsInput, coverage, peerCoverage, peerGrantsEntry, peerSwitch, peerSwitchOutcome, verticalCaller, verticalResolution, verticalSlug, entityObjectRef, ATTACHMENT_BLOB_BINDING, blobStoreBindingName, lifecycleDelivery, scopeLifecycle, } from '@substrat-run/contracts';
|
|
3
5
|
import { normalizeHostname, toRouteTarget } from './route-resolver.js';
|
|
4
|
-
import { attachmentBlobKey, attachmentSha256, entitlementDenial, requiredEntitlementFor, foldMeterReading, parseValidationRecords, resolveScopeRecord, ulid, capabilityTokenHash, checkBecomeInput, plausibleSessionToken, foldModelUsage, backoffAt, resolveRetryPolicy, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, openSealed, ImpersonationRefused, assertSessionUsable, impersonationRowValues, mapImpersonationRow, newImpersonationSession, jobRunOf, runDueJobRuns, startJobRun, attachmentTextJob, assertAttachmentExtractors, assertJobRegistrable, isAttachmentTextRun, searchLimit, searchMatchExpression, globalFetch, assertRedrainWindow, platformRequestOf, undrainedEventsOf, inUnitMovesToAudit, staleCarryRevertRow, staleCarryReverts, withRecorded, systemSwitchedOffMessage, tenantSystemSwitchedOffMessage, CrossVerticalRegistry, importCursorSourceOf, exportBreaksOf, exportBreakRefusal, bindExportBreaksOf, bindExportBreakRefusal, collectPeers, peerSeats, connectorCallRecord, noopConnectorCallRecorder, recordConnectorCall, settleConnectionUse, splitManifestMigrations, unknownRoleError, assertRowLimit, assertRowOffset, } from '@substrat-run/kernel';
|
|
6
|
+
import { attachmentBlobKey, attachmentSha256, entitlementDenial, requiredEntitlementFor, foldMeterReading, parseValidationRecords, resolveScopeRecord, ulid, KEPT_COPY_REFUSAL, capabilityTokenHash, checkBecomeInput, plausibleSessionToken, foldModelUsage, backoffAt, resolveRetryPolicy, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, openSealed, ImpersonationRefused, assertSessionUsable, impersonationRowValues, mapImpersonationRow, newImpersonationSession, jobRunOf, runDueJobRuns, SYSTEM_DOOR_WAIT, startJobRun, attachmentTextJob, assertAttachmentExtractors, assertJobRegistrable, isAttachmentTextRun, searchLimit, searchMatchExpression, globalFetch, assertRedrainWindow, platformRequestOf, undrainedEventsOf, inUnitMovesToAudit, staleCarryRevertRow, staleCarryReverts, withRecorded, systemSwitchedOffMessage, tenantSystemSwitchedOffMessage, CrossVerticalRegistry, importCursorSourceOf, exportBreaksOf, exportBreakRefusal, bindExportBreaksOf, bindExportBreakRefusal, collectPeers, peerSeats, connectorCallRecord, noopConnectorCallRecorder, recordConnectorCall, settleConnectionUse, splitManifestMigrations, unknownRoleError, assertRowLimit, assertRowOffset, INERT_SCOPE_REASON, lifecycleReceipt, lifecycleRefusal, isPrimaryScope, isPrimaryScopeRow, asyncInvocationId, asyncLinePass, } from '@substrat-run/kernel';
|
|
5
7
|
import { attributedHost } from '@substrat-run/kernel';
|
|
6
8
|
import { isOrangeToOrange, isUpgradeRequest, LIVE_MODE_HEADER, LIVE_PRINCIPAL_HEADER, LIVE_SCOPE_HEADER, LIVE_SUBSCRIBE_PATH, LIVE_TENANT_HEADER, } from './live-reads.js';
|
|
7
9
|
import { tenantStoreDatabaseName } from './d1.js';
|
|
@@ -30,12 +32,52 @@ function unwrapCapabilityReply(reply) {
|
|
|
30
32
|
throw fromWireFailure(reply.failure);
|
|
31
33
|
return reply.value;
|
|
32
34
|
}
|
|
35
|
+
/**
|
|
36
|
+
* How far ahead of the clock a held lifecycle epoch may be and still be learned past (#1713). An
|
|
37
|
+
* epoch is the time a directory restore minted it, so a legitimate one is behind its minter's
|
|
38
|
+
* clock; the skew allows for clocks that disagree, and bounds what a forged answer can move.
|
|
39
|
+
*/
|
|
40
|
+
export const LIFECYCLE_EPOCH_SKEW_MS = 24 * 60 * 60 * 1000;
|
|
41
|
+
/**
|
|
42
|
+
* #2016: how many served scopes one heal pass asks for their tenant record, beyond the drift and the
|
|
43
|
+
* holds. Rotated least-recently-asked first, so a deployment that cannot answer yet (a build from
|
|
44
|
+
* before the record) costs at most this many deliveries a pass and starves nothing.
|
|
45
|
+
*/
|
|
46
|
+
export const TENANT_UNRECORDED_PER_PASS = 50;
|
|
47
|
+
/**
|
|
48
|
+
* K-3's refusal of a (tenant, scope) pair the host does not hold — the same answer for a scope that
|
|
49
|
+
* does not exist and for one of another tenant (#1718), in one wording.
|
|
50
|
+
*/
|
|
51
|
+
const unknownScopeForTenant = (tenantId, scopeId) => substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
|
|
52
|
+
/**
|
|
53
|
+
* #2016: the line a CP-less host writes each time it admits a pair to a scope with no tenant record
|
|
54
|
+
* and no role rows — nothing to hold the pair against. Ids only.
|
|
55
|
+
*/
|
|
56
|
+
export const tenantUnrecordedLine = (tenantId, scopeId) => ({
|
|
57
|
+
substrat: 'tenant-unrecorded',
|
|
58
|
+
level: 'warn',
|
|
59
|
+
tenantId,
|
|
60
|
+
scopeId,
|
|
61
|
+
message: 'admitted to a scope whose tenant is not recorded; a reconcile or lifecycle delivery records it',
|
|
62
|
+
});
|
|
63
|
+
/**
|
|
64
|
+
* Refuse to mark a scope a copy unless the request classifies it non-primary (#2005). The
|
|
65
|
+
* classification is the platform directory's; the vertical applies the same `isPrimaryScope` to
|
|
66
|
+
* it, so a primary can be marked only by a request that misstates its lineage, never by default.
|
|
67
|
+
*/
|
|
68
|
+
function assertCopyLineage(lineage) {
|
|
69
|
+
if (isPrimaryScope(lineage)) {
|
|
70
|
+
throw substratError('conflict', 'mark-copy refused: the directory classifies this scope as primary, not as a copy');
|
|
71
|
+
}
|
|
72
|
+
}
|
|
33
73
|
/**
|
|
34
74
|
* A control-plane stand-in for a CP-less vertical (scope-local-permissions.md Phase 3).
|
|
35
75
|
* The hot path a served scope actually touches becomes trust-the-upstream:
|
|
36
|
-
* - `scopeAccessRefusal` / `setMigrationState` → no-op: the router
|
|
37
|
-
*
|
|
38
|
-
*
|
|
76
|
+
* - `scopeAccessRefusal` / `setMigrationState` → no-op: the router gates the scope's
|
|
77
|
+
* lifecycle + tenancy from the shared directory, and the vertical does not re-read a
|
|
78
|
+
* directory it does not have. Its own half of both gates reads the scope's storage
|
|
79
|
+
* instead (`validateScopeAccess`): the tenant the scope was provisioned for (#2016) and the lifecycle
|
|
80
|
+
* the platform delivered (#1713).
|
|
39
81
|
* - `tenantHoldsEntitlement` → true: the SKU was enforced on the shared control plane
|
|
40
82
|
* at provision (before `provisionInstance`), so a scope that EXISTS here was granted
|
|
41
83
|
* it upstream — a single-vertical deployment holds its own entitlements by construction.
|
|
@@ -53,12 +95,14 @@ function nullControlPlane() {
|
|
|
53
95
|
// #1674: the schedule switch's record is a DIRECTORY store, and a CP-less host has no
|
|
54
96
|
// directory — the shared control plane that delegates here keeps it. Same posture as
|
|
55
97
|
// `recordAdmin`: nothing to write, nothing recorded to read back.
|
|
56
|
-
recordSystemSwitchedOff:
|
|
98
|
+
recordSystemSwitchedOff: async () => null,
|
|
57
99
|
recordSystemSwitchedOn: async () => null,
|
|
58
100
|
restoreSystemSwitchRecord: noop,
|
|
59
101
|
listSystemSwitches: async () => [],
|
|
60
102
|
systemSwitchRecordsOf: async () => [],
|
|
61
103
|
switchedOffModulesOf: async () => [],
|
|
104
|
+
// #1823: no tenant tuples here either — the platform reads them and sends `tenantHeld`.
|
|
105
|
+
tenantHeldSystemModules: async () => [],
|
|
62
106
|
};
|
|
63
107
|
return new Proxy({}, {
|
|
64
108
|
get: (_t, prop) => typeof prop === 'string' && prop in passthrough
|
|
@@ -106,6 +150,28 @@ export const SWITCH_HOLD_EXTRA_WAITS = 3;
|
|
|
106
150
|
* clock, against the releasing host's clock: minutes, not seconds.
|
|
107
151
|
*/
|
|
108
152
|
export const SWITCH_HOLD_PENDING_MAX_MS = 5 * 60_000;
|
|
153
|
+
/**
|
|
154
|
+
* An OFF of a tenant-held module whose far end did not attest `deniesTenantGrants` (#1823): it
|
|
155
|
+
* switched, but its evaluator still authorizes the module's tenant-level grant on the scope.
|
|
156
|
+
* Carries the far end's outcome, so the operator's OFF knows whether there is a move to put back.
|
|
157
|
+
*/
|
|
158
|
+
class UnattestedSwitchOff extends Error {
|
|
159
|
+
outcome;
|
|
160
|
+
constructor(scopeId, moduleId, outcome) {
|
|
161
|
+
super(`the deployment serving scope ${scopeId} predates the kill switch's tenant-grant denial (#1823): ` +
|
|
162
|
+
`module '${moduleId}' holds a tenant-level grant that its OFF would leave authorizing there. ` +
|
|
163
|
+
`Redeploy the vertical, then retry.`);
|
|
164
|
+
this.outcome = outcome;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
/**
|
|
168
|
+
* The attachment bucket a host resolves when it was handed no resolver (#1995): the
|
|
169
|
+
* platform's per-tenant binding, read off this script's own env, named by the one shared
|
|
170
|
+
* encoding (`blobStoreBindingName`) for the tenant the caller passed in.
|
|
171
|
+
*/
|
|
172
|
+
function ambientAttachmentBucket(tenantId) {
|
|
173
|
+
return ambientEnv[blobStoreBindingName(ATTACHMENT_BLOB_BINDING, tenantId)] ?? null;
|
|
174
|
+
}
|
|
109
175
|
export class CloudflareScopeHost {
|
|
110
176
|
admin;
|
|
111
177
|
scopeNs;
|
|
@@ -145,7 +211,7 @@ export class CloudflareScopeHost {
|
|
|
145
211
|
tenantStores;
|
|
146
212
|
/** The live R2 client for per-tenant blob stores (#473); undefined ⇒ refuse loudly. */
|
|
147
213
|
blobStores;
|
|
148
|
-
/** Worker-side attachment-bucket resolver (#473);
|
|
214
|
+
/** Worker-side attachment-bucket resolver (#473); the ambient per-tenant binding unless overridden (#1995). */
|
|
149
215
|
attachmentBuckets;
|
|
150
216
|
/** The parsers attachment text is extracted with (K-43); the host's own, never imported here. */
|
|
151
217
|
attachmentExtractors;
|
|
@@ -180,6 +246,8 @@ export class CloudflareScopeHost {
|
|
|
180
246
|
eventDrainDelegation;
|
|
181
247
|
/** #1666: the schedule kill switch's reach into the deployment serving a scope. */
|
|
182
248
|
systemSwitchDelegation;
|
|
249
|
+
/** #1713: the lifecycle's reach into the deployment serving a scope. */
|
|
250
|
+
lifecycleDelegation;
|
|
183
251
|
/** #1706: the peer kill switch's reach into the deployment serving a scope. */
|
|
184
252
|
peerSwitchDelegation;
|
|
185
253
|
/** #1705 PR 3: the replay lever's reach into the deployment serving a consumer scope. */
|
|
@@ -211,7 +279,7 @@ export class CloudflareScopeHost {
|
|
|
211
279
|
this.secretBox = options.secretBox ?? unconfiguredSecretBox;
|
|
212
280
|
this.tenantStores = options.tenantStores;
|
|
213
281
|
this.blobStores = options.blobStores;
|
|
214
|
-
this.attachmentBuckets = options.attachmentBuckets;
|
|
282
|
+
this.attachmentBuckets = options.attachmentBuckets ?? ambientAttachmentBucket;
|
|
215
283
|
assertAttachmentExtractors(options.attachmentExtractors ?? []);
|
|
216
284
|
this.attachmentExtractors = options.attachmentExtractors ?? [];
|
|
217
285
|
this.fetchImpl = options.fetch ?? globalFetch;
|
|
@@ -225,6 +293,7 @@ export class CloudflareScopeHost {
|
|
|
225
293
|
this.connectorDelegation = options.connectorDelegation;
|
|
226
294
|
this.eventDrainDelegation = options.eventDrainDelegation;
|
|
227
295
|
this.systemSwitchDelegation = options.systemSwitchDelegation;
|
|
296
|
+
this.lifecycleDelegation = options.lifecycleDelegation;
|
|
228
297
|
this.peerSwitchDelegation = options.peerSwitchDelegation;
|
|
229
298
|
this.importCursorDelegation = options.importCursorDelegation;
|
|
230
299
|
this.admin = this.buildAdmin();
|
|
@@ -353,21 +422,78 @@ export class CloudflareScopeHost {
|
|
|
353
422
|
};
|
|
354
423
|
if (this.executors.size === 0)
|
|
355
424
|
return report;
|
|
425
|
+
// #1901: one line per attempt, capped per pass. Written on the coordinator, where the
|
|
426
|
+
// attempt runs — the DO only journals it.
|
|
427
|
+
const lines = asyncLinePass();
|
|
428
|
+
try {
|
|
429
|
+
await this.drainExecutorsPass(tenantId, scopeId, invocationId, report, lines);
|
|
430
|
+
}
|
|
431
|
+
finally {
|
|
432
|
+
lines.end();
|
|
433
|
+
}
|
|
434
|
+
return report;
|
|
435
|
+
}
|
|
436
|
+
async drainExecutorsPass(tenantId, scopeId, invocationId, report, lines) {
|
|
356
437
|
const stub = this.scopeStub(scopeId);
|
|
357
|
-
|
|
438
|
+
// #2005: a non-primary scope (a fork, a snapshot, a preview of either kind) causes no
|
|
439
|
+
// outbound effects, so its deliveries are journaled terminal with the reason and no
|
|
440
|
+
// handler runs. Asked of the directory where there is one. A CP-less host has none, so it
|
|
441
|
+
// asks the scope's own storage whether it is classified a copy (`_substrat_copy_origin`'s
|
|
442
|
+
// `is_copy`, #2009, set on the directory's word) — a preview and a snapshot reach a hosted
|
|
443
|
+
// vertical as restores the platform marks.
|
|
444
|
+
// Asked on the first due event, once per pass: most passes have none, and the directory
|
|
445
|
+
// is one global object.
|
|
446
|
+
let inert;
|
|
447
|
+
const isInert = () => (inert ??= this.cpLess
|
|
448
|
+
? stub.isCopy()
|
|
449
|
+
: this.cp.getScopeRecord(tenantId, scopeId).then((row) => !isPrimaryScopeRow(row)));
|
|
450
|
+
// #1713: a CP-less scope its lifecycle holds attempts nothing and journals nothing, so every
|
|
451
|
+
// due delivery stays due for the first pass after it is live again. Asked on the first due
|
|
452
|
+
// event, like `isInert`. Every caller passed `assertLive` already, so this is for the scope
|
|
453
|
+
// suspended between that gate and the drain: a request in flight, whose write commits while
|
|
454
|
+
// its effect waits. A host with a directory answers false without a read.
|
|
455
|
+
let held;
|
|
456
|
+
const isHeld = () => (held ??= this.lifecycleHeld(scopeId));
|
|
457
|
+
drain: for (const [id, executor] of this.executors) {
|
|
358
458
|
const deliveryId = `executor:${id}`;
|
|
359
459
|
const { events, undecodable } = await stub.pendingExecutorDeliveries(deliveryId, executor.eventType);
|
|
360
460
|
// #1636: an event the DO could not decode is dead-lettered for this executor at once,
|
|
361
461
|
// and its handler never sees it. Terminal on the FIRST failure, unlike a handler's:
|
|
362
462
|
// the decode is pure, so a retry cannot succeed. The rows behind it are delivered
|
|
363
463
|
// below — the decode used to throw the whole list, on every pass.
|
|
464
|
+
if ((undecodable.length > 0 || events.length > 0) && (await isHeld())) {
|
|
465
|
+
report.lifecycleHeld = true;
|
|
466
|
+
break drain;
|
|
467
|
+
}
|
|
468
|
+
// #1901: an attempt's line. The attempt number is the one the journal is about to record.
|
|
469
|
+
const unitOf = (eventId, attempt) => ({
|
|
470
|
+
kind: 'consumer',
|
|
471
|
+
tenantId,
|
|
472
|
+
scopeId,
|
|
473
|
+
invocationId: asyncInvocationId(invocationId),
|
|
474
|
+
operation: deliveryId,
|
|
475
|
+
eventType: executor.eventType,
|
|
476
|
+
eventId,
|
|
477
|
+
attempt,
|
|
478
|
+
startedAt: Date.now(),
|
|
479
|
+
});
|
|
364
480
|
for (const bad of undecodable) {
|
|
365
481
|
report.attempted += 1;
|
|
366
|
-
await stub.recordExecutorAttempt(bad.eventId, deliveryId, bad.error, null, invocationId);
|
|
482
|
+
const attempt = await stub.recordExecutorAttempt(bad.eventId, deliveryId, bad.error, null, invocationId);
|
|
367
483
|
report.deadLettered += 1;
|
|
484
|
+
lines.write({ ...unitOf(bad.eventId, attempt), outcome: 'dead-lettered' });
|
|
368
485
|
}
|
|
369
486
|
for (const event of events) {
|
|
370
487
|
report.attempted += 1;
|
|
488
|
+
const startedAt = Date.now();
|
|
489
|
+
if (await isInert()) {
|
|
490
|
+
// No next attempt, like an undecodable row: a scope does not become primary.
|
|
491
|
+
const attempt = await stub.recordExecutorAttempt(event.id, deliveryId, INERT_SCOPE_REASON, null, invocationId);
|
|
492
|
+
report.inert = (report.inert ?? 0) + 1;
|
|
493
|
+
// Its own outcome, never `delivered`: no handler ran (#2005).
|
|
494
|
+
lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'inert' });
|
|
495
|
+
continue;
|
|
496
|
+
}
|
|
371
497
|
this.causedBy = event.id;
|
|
372
498
|
try {
|
|
373
499
|
if (executor.kind === 'connector' && this.cpLess) {
|
|
@@ -381,16 +507,20 @@ export class CloudflareScopeHost {
|
|
|
381
507
|
// the at-least-once residue, as it already must in-process.
|
|
382
508
|
await stub.routeExecutorEventToPlatform(event.id, deliveryId, connectorDispatchKind(executor.provider), JSON.stringify({ executorId: id, event }), JSON.stringify({ system: 'connector-dispatch' }), invocationId);
|
|
383
509
|
report.routedToPlatform += 1;
|
|
510
|
+
// Handed on, not run here: the platform's drain owns its attempts from now on.
|
|
511
|
+
lines.write({ ...unitOf(event.id, 1), startedAt, outcome: 'routed' });
|
|
384
512
|
}
|
|
385
513
|
else if (executor.kind === 'connector') {
|
|
386
514
|
await executor.handler(await this.connectorContext(tenantId, scopeId, executor.timeoutMs, event.id), event);
|
|
387
|
-
await stub.recordExecutorAttempt(event.id, deliveryId, null, null, invocationId);
|
|
515
|
+
const attempt = await stub.recordExecutorAttempt(event.id, deliveryId, null, null, invocationId);
|
|
388
516
|
report.delivered += 1;
|
|
517
|
+
lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'delivered' });
|
|
389
518
|
}
|
|
390
519
|
else {
|
|
391
520
|
await executor.handler(this.admin, event);
|
|
392
|
-
await stub.recordExecutorAttempt(event.id, deliveryId, null, null, invocationId);
|
|
521
|
+
const attempt = await stub.recordExecutorAttempt(event.id, deliveryId, null, null, invocationId);
|
|
393
522
|
report.delivered += 1;
|
|
523
|
+
lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'delivered' });
|
|
394
524
|
}
|
|
395
525
|
}
|
|
396
526
|
catch (err) {
|
|
@@ -405,18 +535,23 @@ export class CloudflareScopeHost {
|
|
|
405
535
|
report.deadLettered += 1;
|
|
406
536
|
else
|
|
407
537
|
report.retrying += 1;
|
|
538
|
+
lines.write({
|
|
539
|
+
...unitOf(event.id, attempts),
|
|
540
|
+
startedAt,
|
|
541
|
+
outcome: exhausted ? 'dead-lettered' : 'retrying',
|
|
542
|
+
error: err,
|
|
543
|
+
});
|
|
408
544
|
}
|
|
409
545
|
finally {
|
|
410
546
|
this.causedBy = null;
|
|
411
547
|
}
|
|
412
548
|
}
|
|
413
549
|
}
|
|
414
|
-
return report;
|
|
415
550
|
}
|
|
416
551
|
async drainDue(tenantId, scopeId) {
|
|
417
552
|
// Same lifecycle gate `getScope` applies (K-3): a suspended or archived scope
|
|
418
553
|
// does not get its effects driven either.
|
|
419
|
-
await this.
|
|
554
|
+
await this.assertLive(tenantId, scopeId);
|
|
420
555
|
await this.migrateAndRecord(scopeId);
|
|
421
556
|
// #1525: null, and honestly so — a sweep is not a call. An attempt this pass makes
|
|
422
557
|
// records no invocation, which is what distinguishes it from the first attempt the
|
|
@@ -448,7 +583,7 @@ export class CloudflareScopeHost {
|
|
|
448
583
|
return {
|
|
449
584
|
startOrJoin: (key, row) => stub.jobRunStartOrJoin(key.moduleId, key.job, key.instance, row),
|
|
450
585
|
get: (id) => stub.jobRunById(id),
|
|
451
|
-
|
|
586
|
+
dueKeys: (now, max) => stub.jobRunsDueKeys(now, max),
|
|
452
587
|
list: (filter) => stub.jobRunList(filter),
|
|
453
588
|
patch: (id, patch) => stub.jobRunPatch(id, patch),
|
|
454
589
|
commitPass: (id, patch) => stub.jobCommitPass(id, patch),
|
|
@@ -474,14 +609,14 @@ export class CloudflareScopeHost {
|
|
|
474
609
|
this.attachmentExtractors);
|
|
475
610
|
}
|
|
476
611
|
async startJobRun(tenantId, scopeId, input) {
|
|
477
|
-
await this.
|
|
612
|
+
await this.assertLive(tenantId, scopeId);
|
|
478
613
|
await this.migrateAndRecord(scopeId);
|
|
479
614
|
return jobRunOf(await startJobRun(this.jobStore(scopeId), input, ulid, () => new Date().toISOString()));
|
|
480
615
|
}
|
|
481
616
|
async runDueJobs(tenantId, scopeId, options) {
|
|
482
617
|
// Same lifecycle gate as `drainDue`: a suspended scope's runs wait rather than
|
|
483
618
|
// advance, and an archived one's never move again.
|
|
484
|
-
await this.
|
|
619
|
+
await this.assertLive(tenantId, scopeId);
|
|
485
620
|
await this.migrateAndRecord(scopeId);
|
|
486
621
|
// #1575: the kernel's extraction job is this host's own, bound to this scope — no
|
|
487
622
|
// deployment registers it, and none can shadow it.
|
|
@@ -490,7 +625,10 @@ export class CloudflareScopeHost {
|
|
|
490
625
|
store: this.jobStore(scopeId),
|
|
491
626
|
handlerFor: (run) => isAttachmentTextRun(run) ? attachmentText : this.jobs.get(`${run.module_id}/${run.job}`),
|
|
492
627
|
now: () => new Date().toISOString(),
|
|
493
|
-
|
|
628
|
+
// #1834: the door is opened FOR this pass, so its "not now" is tied to this pass alone.
|
|
629
|
+
openScope: async (run, pass) => this.buildStub(tenantId, scopeId, undefined, undefined, await this.openSystemDoor(run.module_id, tenantId, scopeId, undefined, pass)),
|
|
630
|
+
// #1834: only this host's own door refusals defer a pass, recognised by identity.
|
|
631
|
+
deferral: (err, pass) => this.takeDoorWait(err, pass),
|
|
494
632
|
maxPasses: options?.maxPasses,
|
|
495
633
|
limit: options?.limit,
|
|
496
634
|
});
|
|
@@ -506,7 +644,7 @@ export class CloudflareScopeHost {
|
|
|
506
644
|
// same context build as the in-process path, so the handler cannot tell which host
|
|
507
645
|
// ran it. On a CP-less host `connectorContext` throws from the null control plane:
|
|
508
646
|
// fail closed, exactly the hole routing exists to avoid.
|
|
509
|
-
await this.
|
|
647
|
+
await this.assertLive(tenantId, scopeId);
|
|
510
648
|
this.causedBy = event.id;
|
|
511
649
|
try {
|
|
512
650
|
await handler(await this.connectorContext(tenantId, scopeId, options?.timeoutMs ?? 30_000, event.id), event);
|
|
@@ -570,7 +708,7 @@ export class CloudflareScopeHost {
|
|
|
570
708
|
const rec = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
571
709
|
// K-3: a scope under another tenant is indistinguishable from one that does not exist.
|
|
572
710
|
if (!rec)
|
|
573
|
-
throw
|
|
711
|
+
throw unknownScopeForTenant(tenantId, scopeId);
|
|
574
712
|
if (rec.status !== 'active' && rec.status !== 'provisioning') {
|
|
575
713
|
throw new Error(`scope not migratable (status: ${rec.status}): ${scopeId}`);
|
|
576
714
|
}
|
|
@@ -642,6 +780,31 @@ export class CloudflareScopeHost {
|
|
|
642
780
|
async listCapabilitiesLocal(scopeId, filter) {
|
|
643
781
|
return this.scopeStub(scopeId).listCapabilities(filter);
|
|
644
782
|
}
|
|
783
|
+
/**
|
|
784
|
+
* The platform's `become` mint on the CP-less path (#1686) — `HostAdmin.mintCapability`'s
|
|
785
|
+
* ScopeDO half, for the deployment that serves the scope. Its one caller is an owner claim
|
|
786
|
+
* link, minted on the platform's instruction behind the vertical's platform-gated
|
|
787
|
+
* `/internal/owner-claim`; the control plane made the K-3 check and keeps the admin log before
|
|
788
|
+
* it called. Not a module verb: a module mints through `ctx.capabilities`, which mints `act`
|
|
789
|
+
* capabilities only. Same lifecycle gate as the exchange, so a held scope mints nothing.
|
|
790
|
+
*/
|
|
791
|
+
async mintCapabilityLocal(tenantId, scopeId, input, actor) {
|
|
792
|
+
// Checked on this side too, as `HostAdmin.mintCapability` does: a typed refusal thrown
|
|
793
|
+
// across the RPC arrives as a bare message.
|
|
794
|
+
checkBecomeInput(input, new Date().toISOString());
|
|
795
|
+
await this.assertLive(tenantId, scopeId);
|
|
796
|
+
await this.migrateAndRecord(scopeId);
|
|
797
|
+
return mintedCapability.parse(await this.scopeStub(scopeId).mintBecomeCapability(input, actor));
|
|
798
|
+
}
|
|
799
|
+
/**
|
|
800
|
+
* The platform's revoke on the CP-less path (#1686) — how a re-minted claim link retires the
|
|
801
|
+
* previous one. True when the scope held the capability (revoked now, or already); false when
|
|
802
|
+
* it holds no such capability. No lifecycle gate: a revoke only narrows, so a held scope still
|
|
803
|
+
* takes one. Idempotent.
|
|
804
|
+
*/
|
|
805
|
+
async revokeCapabilityLocal(scopeId, capabilityId, actor) {
|
|
806
|
+
return (await this.scopeStub(scopeId).revokeCapabilityAsPlatform(capabilityId, actor)) !== null;
|
|
807
|
+
}
|
|
645
808
|
/**
|
|
646
809
|
* Copy one scope's data into a fresh scope DO, entirely within THIS deployment —
|
|
647
810
|
* the data half of an orchestrated snapshot (preview-and-snapshots.md §9). Like the
|
|
@@ -650,10 +813,24 @@ export class CloudflareScopeHost {
|
|
|
650
813
|
* activation, version bind) stays on the control plane's side. Because source and
|
|
651
814
|
* destination sit in the same SCOPE namespace, no scope bytes ever leave the
|
|
652
815
|
* deployment — the §9 property the trust line rests on.
|
|
816
|
+
*
|
|
817
|
+
* #2016: `tenantId`, the tenant the platform snapshots for (absent from an older platform). The
|
|
818
|
+
* source must not be foreign to it — a copy never reads another tenant's scope — and the fork
|
|
819
|
+
* records it as its own receipt, not the source's.
|
|
653
820
|
*/
|
|
654
|
-
async snapshotScopeLocal(sourceScopeId, destScopeId) {
|
|
821
|
+
async snapshotScopeLocal(sourceScopeId, destScopeId, tenantId) {
|
|
822
|
+
if (tenantId !== undefined && (await this.scopeStub(sourceScopeId).admission(tenantId)).verdict === 'foreign') {
|
|
823
|
+
throw unknownScopeForTenant(tenantId, sourceScopeId);
|
|
824
|
+
}
|
|
655
825
|
const tables = await this.scopeStub(sourceScopeId).exportDump();
|
|
656
|
-
|
|
826
|
+
// #2009: a snapshot is a fork by construction (the control plane's row names `forkedFrom`, so
|
|
827
|
+
// the directory never calls it primary), and a load classifies nothing by itself: marked here.
|
|
828
|
+
await this.scopeStub(destScopeId).importDump(tables, destScopeId, {
|
|
829
|
+
sourceScopeId,
|
|
830
|
+
exact: true,
|
|
831
|
+
markCopy: true,
|
|
832
|
+
provisionedFor: tenantId,
|
|
833
|
+
});
|
|
657
834
|
return { tables: tables.length };
|
|
658
835
|
}
|
|
659
836
|
/**
|
|
@@ -666,14 +843,145 @@ export class CloudflareScopeHost {
|
|
|
666
843
|
/** #1742: the directory's recorded-off modules, switched off on THIS scope in the replay's
|
|
667
844
|
* own event — a dump from before a switch was pulled brings its grants back live. #1869:
|
|
668
845
|
* `sourceScopeId`, the scope the dump was captured from, narrows the grant re-point to it;
|
|
669
|
-
* `exact` says the platform exported the dump itself, so the re-point never falls back.
|
|
846
|
+
* `exact` says the platform exported the dump itself, so the re-point never falls back.
|
|
847
|
+
* #1722: `loadStamp`, the stamp a carry leaves on the copy it lands, for a fenced wipe later,
|
|
848
|
+
* and `expect`, the marker the carry read here: the load is refused if the store moved since.
|
|
849
|
+
* #2005: `markCopy` — the directory's classification of the scope, sent when it is not
|
|
850
|
+
* primary; the restore then marks it a copy. Refused if it classifies a primary.
|
|
851
|
+
* #2016: `tenantId`, the tenant the platform restores for: recorded as the scope's receipt in
|
|
852
|
+
* the load's own transaction, and a store provisioned for another tenant refuses the load. */
|
|
670
853
|
opts) {
|
|
671
|
-
|
|
672
|
-
|
|
854
|
+
if (opts?.markCopy)
|
|
855
|
+
assertCopyLineage(opts.markCopy);
|
|
856
|
+
const load = {
|
|
857
|
+
switchOff: opts?.switchedOff
|
|
858
|
+
? { moduleIds: opts.switchedOff, at: new Date().toISOString(), tenantHeld: opts.tenantHeld }
|
|
859
|
+
: undefined,
|
|
673
860
|
sourceScopeId: opts?.sourceScopeId,
|
|
674
861
|
exact: opts?.exact,
|
|
862
|
+
loadStamp: opts?.loadStamp,
|
|
863
|
+
markCopy: opts?.markCopy !== undefined,
|
|
864
|
+
provisionedFor: opts?.tenantId,
|
|
865
|
+
};
|
|
866
|
+
// The refusals come back as values (a throw over the RPC carries only its message) and are
|
|
867
|
+
// thrown here, in the vertical's own isolate, so its route answers 409 or 412, not a fault.
|
|
868
|
+
const out = await this.scopeStub(scopeId).importDumpChecked(tables, scopeId, {
|
|
869
|
+
...load,
|
|
870
|
+
...(opts?.expect ? { expect: opts.expect } : {}),
|
|
675
871
|
});
|
|
676
|
-
|
|
872
|
+
if (out.refused === false) {
|
|
873
|
+
return { tables: tables.length, ...(opts?.switchedOff ? { switchedOff: out.switchedOff } : {}) };
|
|
874
|
+
}
|
|
875
|
+
if (out.refused === 'kept')
|
|
876
|
+
throw substratError('conflict', KEPT_COPY_REFUSAL);
|
|
877
|
+
if (out.refused === 'tenant')
|
|
878
|
+
throw substratError('conflict', out.message);
|
|
879
|
+
// A retry of a load that already committed (its answer was lost on the way back) is
|
|
880
|
+
// refused by the marker that load itself moved. The store holding THIS request's stamp
|
|
881
|
+
// says so: no other load writes it. Answered as applied, without `switchedOff`, so the
|
|
882
|
+
// caller's re-assert after the bind covers the OFF positions.
|
|
883
|
+
const now = await this.scopeStub(scopeId).loadMarker();
|
|
884
|
+
if (!opts?.loadStamp || now.loadStamp !== opts.loadStamp) {
|
|
885
|
+
throw substratError('precondition_failed', 'scope store changed since the carry read it; nothing was loaded (#1722)');
|
|
886
|
+
}
|
|
887
|
+
return { tables: tables.length };
|
|
888
|
+
}
|
|
889
|
+
/** The kept-copy marker of this scope's store in THIS deployment (#1722), or null. */
|
|
890
|
+
async keptCopyLocal(scopeId) {
|
|
891
|
+
return this.scopeStub(scopeId).keptCopy();
|
|
892
|
+
}
|
|
893
|
+
/**
|
|
894
|
+
* Discard the kept copy of a scope in THIS deployment (#1722), behind the vertical's
|
|
895
|
+
* `/internal/kept-copy/discard`: the staff resolution, only at the write revision the operator
|
|
896
|
+
* acted on. Answers whether it was discarded, or why not.
|
|
897
|
+
*/
|
|
898
|
+
async discardKeptCopyLocal(scopeId, revision, carriedAway,
|
|
899
|
+
/** #2005: the directory's classification, sent when the scope is not primary. */
|
|
900
|
+
markCopy,
|
|
901
|
+
/** #1722 (Codex #2008 r13): the load stamp read with `revision`; a replaced store is refused. */
|
|
902
|
+
loadStamp) {
|
|
903
|
+
if (markCopy)
|
|
904
|
+
assertCopyLineage(markCopy);
|
|
905
|
+
return this.scopeStub(scopeId).discardKeptCopy(scopeId, revision, carriedAway, markCopy !== undefined, ...(loadStamp !== undefined ? [loadStamp] : []));
|
|
906
|
+
}
|
|
907
|
+
/**
|
|
908
|
+
* What a carry's restore into this scope's store expects to find unchanged (#1722), behind
|
|
909
|
+
* the vertical's `/internal/load-marker`. Read before the carry checks the binding again; the
|
|
910
|
+
* restore then carries it as `expect`, and is refused if the store was loaded or written since.
|
|
911
|
+
*/
|
|
912
|
+
async loadMarkerLocal(scopeId) {
|
|
913
|
+
return this.scopeStub(scopeId).loadMarker();
|
|
914
|
+
}
|
|
915
|
+
/**
|
|
916
|
+
* Wipe the copy a carry left in THIS deployment (#1722), behind the vertical's
|
|
917
|
+
* `/internal/wipe-carried`: only if nothing was loaded into the scope's DO since the stamp
|
|
918
|
+
* the carry read (`expectLoadStamp`, null for a store no load has stamped). Non-terminal,
|
|
919
|
+
* so a later bind back to this version can restore into it. False when refused.
|
|
920
|
+
*/
|
|
921
|
+
async wipeCarriedLocal(scopeId, expectLoadStamp, carriedAway, opts = {}) {
|
|
922
|
+
if (opts.markCopy)
|
|
923
|
+
assertCopyLineage(opts.markCopy);
|
|
924
|
+
return this.scopeStub(scopeId).wipeCarried(scopeId, expectLoadStamp, carriedAway, {
|
|
925
|
+
...(opts.expectRevision !== undefined ? { expectRevision: opts.expectRevision } : {}),
|
|
926
|
+
...(opts.protectIfChanged !== undefined ? { protectIfChanged: opts.protectIfChanged } : {}),
|
|
927
|
+
markCopy: opts.markCopy !== undefined,
|
|
928
|
+
});
|
|
929
|
+
}
|
|
930
|
+
/** Release a kept copy that is the live store after all (#1722), at the revision read. */
|
|
931
|
+
async releaseKeptCopyLocal(scopeId, revision,
|
|
932
|
+
/** #2005: the directory's classification, sent when the scope is not primary. */
|
|
933
|
+
markCopy,
|
|
934
|
+
/** #1722 (Codex #2008 r13): the load stamp read with `revision`; a replaced store is refused. */
|
|
935
|
+
loadStamp) {
|
|
936
|
+
if (markCopy)
|
|
937
|
+
assertCopyLineage(markCopy);
|
|
938
|
+
return this.scopeStub(scopeId).releaseKeptCopy(revision, markCopy !== undefined, ...(loadStamp !== undefined ? [loadStamp] : []));
|
|
939
|
+
}
|
|
940
|
+
/**
|
|
941
|
+
* Store the lifecycle the platform delivered for one scope in THIS deployment (#1713), behind
|
|
942
|
+
* the vertical's `/internal/lifecycle`. Kept unless the scope already holds a newer one, so a
|
|
943
|
+
* push that arrives late cannot undo a later transition. `applied` is what the platform's heal
|
|
944
|
+
* sweep records as delivered; `changed` says the gate's answer moved.
|
|
945
|
+
*
|
|
946
|
+
* #2016: `tenantId`, the tenant the directory delivered it for (absent from an older platform).
|
|
947
|
+
* A scope provisioned for another tenant refuses the delivery (`conflict`) and stores nothing;
|
|
948
|
+
* a scope provisioned before the tenant receipt existed records it here, its role rows agreeing.
|
|
949
|
+
*/
|
|
950
|
+
async setLifecycleLocal(scopeId, next, tenantId) {
|
|
951
|
+
const out = await this.scopeStub(scopeId).setLifecycle(scopeLifecycle.parse(next), tenantId);
|
|
952
|
+
if ('refused' in out)
|
|
953
|
+
throw substratError('conflict', out.message);
|
|
954
|
+
return lifecycleDelivery.parse(out);
|
|
955
|
+
}
|
|
956
|
+
/**
|
|
957
|
+
* Mark one scope in THIS deployment a copy (#2005), behind the vertical's `/internal/mark-copy`:
|
|
958
|
+
* the repair of a copy that predates the marker, which a CP-less coordinator reads for primacy.
|
|
959
|
+
* The control plane decides which scopes (its directory says they are not primary) and audits.
|
|
960
|
+
*/
|
|
961
|
+
async markCopyLocal(scopeId, lineage) {
|
|
962
|
+
assertCopyLineage(lineage);
|
|
963
|
+
return { marked: await this.scopeStub(scopeId).markCopy() };
|
|
964
|
+
}
|
|
965
|
+
/**
|
|
966
|
+
* Clear a mistaken copy classification from one scope in THIS deployment (#2005), behind the
|
|
967
|
+
* vertical's `/internal/clear-copy-mark`: staff's correction for a scope the directory says IS
|
|
968
|
+
* primary. Refused for a scope classified a copy. A load's copied-events mark is never touched
|
|
969
|
+
* (#2009), so a primary restored from another scope's backup runs its own effects again and
|
|
970
|
+
* still never runs the work it copied.
|
|
971
|
+
*/
|
|
972
|
+
async clearCopyMarkLocal(scopeId, lineage,
|
|
973
|
+
/** #1722 (Codex #2008 r12–r13): the store as the caller means it, its load stamp and write
|
|
974
|
+
* revision, compared in the clear's own transaction; refused (412) if either moved. The
|
|
975
|
+
* platform's reconcile of a carry's destination sends it; staff send none. */
|
|
976
|
+
expect) {
|
|
977
|
+
if (!isPrimaryScope(lineage)) {
|
|
978
|
+
throw substratError('conflict', 'clear-copy-mark refused: the directory classifies this scope as a copy (a preview or a fork)');
|
|
979
|
+
}
|
|
980
|
+
const outcome = await this.scopeStub(scopeId).clearCopyMark(...(expect ? [expect] : []));
|
|
981
|
+
if (outcome === 'changed') {
|
|
982
|
+
throw substratError('precondition_failed', `clear-copy-mark refused: scope ${scopeId}'s store changed since its revision was read`);
|
|
983
|
+
}
|
|
984
|
+
return { cleared: outcome === 'cleared' };
|
|
677
985
|
}
|
|
678
986
|
/**
|
|
679
987
|
* Re-apply the vertical's OWN role definitions to one scope, CP-lessly — the
|
|
@@ -710,6 +1018,13 @@ export class CloudflareScopeHost {
|
|
|
710
1018
|
async exportScopeLocal(scopeId) {
|
|
711
1019
|
return this.scopeStub(scopeId).exportDump();
|
|
712
1020
|
}
|
|
1021
|
+
/**
|
|
1022
|
+
* `exportScopeLocal` with the store's load stamp, read in the same DO call (#1722): what a
|
|
1023
|
+
* carry's fenced wipe of the copy it leaves here expects. Behind the vertical's `/internal/export`.
|
|
1024
|
+
*/
|
|
1025
|
+
async exportScopeStampedLocal(scopeId) {
|
|
1026
|
+
return this.scopeStub(scopeId).exportDumpStamped();
|
|
1027
|
+
}
|
|
713
1028
|
/** Facet this host's own scope's outbox (#1239) — the vertical-host read. */
|
|
714
1029
|
async facetEventsLocal(scopeId, input) {
|
|
715
1030
|
return this.scopeStub(scopeId).facetEvents(input);
|
|
@@ -924,8 +1239,15 @@ export class CloudflareScopeHost {
|
|
|
924
1239
|
// after it then finds them off, and audits what the unit moved.
|
|
925
1240
|
const ownStore = this.cpLess || record.vertical === null;
|
|
926
1241
|
const recordedOff = ownStore ? await this.cp.switchedOffModulesOf(input.tenantId, input.scopeId) : [];
|
|
1242
|
+
const seatAt = new Date().toISOString();
|
|
927
1243
|
const switchedOff = await this.scopeStub(input.scopeId).seatTuples(seats, recordedOff.length
|
|
928
|
-
? {
|
|
1244
|
+
? {
|
|
1245
|
+
scopeId: input.scopeId,
|
|
1246
|
+
moduleIds: recordedOff,
|
|
1247
|
+
at: seatAt,
|
|
1248
|
+
// #1823: a recorded-off module held only by a tenant grant goes back off too.
|
|
1249
|
+
tenantHeld: await this.cp.tenantHeldSystemModules(input.tenantId, recordedOff, seatAt),
|
|
1250
|
+
}
|
|
929
1251
|
: undefined);
|
|
930
1252
|
if (ownStore) {
|
|
931
1253
|
await this.admin.reassertSystemSwitches(actor, { tenantId: input.tenantId, scopeId: input.scopeId }, { appliedInUnit: switchedOff });
|
|
@@ -1065,21 +1387,21 @@ export class CloudflareScopeHost {
|
|
|
1065
1387
|
// gates and metadata facts live in the ScopeDO (per-scope serialization, spine event
|
|
1066
1388
|
// in the same transaction); bytes go straight to the per-tenant R2 bucket through the
|
|
1067
1389
|
// binding the vertical's worker resolved — never through the DO.
|
|
1068
|
-
await this.
|
|
1390
|
+
await this.assertLive(tenantId, scopeId);
|
|
1069
1391
|
await this.migrateAndRecord(scopeId);
|
|
1070
1392
|
const store = await this.resolveAttachmentStore(tenantId);
|
|
1071
1393
|
return this.buildAttachmentSurface({ principal }, tenantId, scopeId, store);
|
|
1072
1394
|
}
|
|
1073
1395
|
async getSystemAttachments(moduleId, tenantId, scopeId) {
|
|
1074
|
-
// Reuse the system invoke door's module, tenant and lifecycle gate
|
|
1075
|
-
// authorizes each open against the attachment target's read permission as
|
|
1396
|
+
// Reuse the system invoke door's module, tenant and lifecycle gate, and its rewind gate
|
|
1397
|
+
// (#1834). The DO authorizes each open against the attachment target's read permission as
|
|
1076
1398
|
// system:<moduleId>; the worker reads bytes only after that gate succeeds.
|
|
1077
|
-
await this.
|
|
1399
|
+
const door = await this.openSystemDoor(moduleId, tenantId, scopeId);
|
|
1078
1400
|
const store = await this.resolveAttachmentStore(tenantId);
|
|
1079
1401
|
const stub = this.scopeStub(scopeId);
|
|
1080
1402
|
return {
|
|
1081
1403
|
open: async (attachmentId) => {
|
|
1082
|
-
const record = await stub.systemAttachmentAuthorize(attachmentId, moduleId, tenantId, scopeId);
|
|
1404
|
+
const record = await door.through((instance) => stub.systemAttachmentAuthorize(attachmentId, moduleId, tenantId, scopeId, instance));
|
|
1083
1405
|
return record ? this.openAttachmentBytes(store, scopeId, record) : null;
|
|
1084
1406
|
},
|
|
1085
1407
|
};
|
|
@@ -1138,17 +1460,14 @@ export class CloudflareScopeHost {
|
|
|
1138
1460
|
const store = await this.resolveAttachmentStore(conn.tenant_id);
|
|
1139
1461
|
return this.buildAttachmentSurface({ connectionId }, conn.tenant_id, scopeId, store, forEvent);
|
|
1140
1462
|
}
|
|
1141
|
-
/** Resolve the per-tenant R2 blob store, or fail closed
|
|
1463
|
+
/** Resolve the per-tenant R2 blob store, or fail closed. */
|
|
1142
1464
|
async resolveAttachmentStore(tenantId) {
|
|
1143
|
-
if (!this.attachmentBuckets) {
|
|
1144
|
-
throw new Error(`attachments are not configured on this host (#473): pass ` +
|
|
1145
|
-
`CloudflareScopeHostOptions.attachmentBuckets — (tenantId) => ` +
|
|
1146
|
-
`env[blobStoreBindingName('<BINDING>', tenantId)] for the vertical's declared blob store`);
|
|
1147
|
-
}
|
|
1148
1465
|
const bucket = await this.attachmentBuckets(tenantId);
|
|
1149
1466
|
if (!bucket) {
|
|
1150
|
-
throw new Error(`no attachment bucket resolved for tenant ${tenantId} (#473) —
|
|
1151
|
-
|
|
1467
|
+
throw new Error(`no attachment bucket resolved for tenant ${tenantId} (#473, #1995) — this script has no ` +
|
|
1468
|
+
`${blobStoreBindingName(ATTACHMENT_BLOB_BINDING, tenantId)} binding. A vertical whose modules declare ` +
|
|
1469
|
+
`attachmentTargets gets one per installed tenant once it is pushed with a CLI that declares the ` +
|
|
1470
|
+
`store (re-push); off the platform, pass CloudflareScopeHostOptions.attachmentBuckets.`);
|
|
1152
1471
|
}
|
|
1153
1472
|
return r2TenantBlobStore(bucket);
|
|
1154
1473
|
}
|
|
@@ -1288,8 +1607,16 @@ export class CloudflareScopeHost {
|
|
|
1288
1607
|
// restored in its deployment, which the platform carries the same list to.
|
|
1289
1608
|
const ownStore = this.cpLess || existing.vertical === null;
|
|
1290
1609
|
const recordedOff = ownStore ? await this.cp.switchedOffModulesOf(tenantId, scopeId) : [];
|
|
1610
|
+
const replayAt = new Date().toISOString();
|
|
1291
1611
|
const switchedOff = await this.scopeStub(scopeId).importDump(dump.tables, scopeId, {
|
|
1292
|
-
switchOff: recordedOff.length
|
|
1612
|
+
switchOff: recordedOff.length
|
|
1613
|
+
? {
|
|
1614
|
+
moduleIds: recordedOff,
|
|
1615
|
+
at: replayAt,
|
|
1616
|
+
// #1823: a recorded-off module held only by a tenant grant goes back off too.
|
|
1617
|
+
tenantHeld: await this.cp.tenantHeldSystemModules(tenantId, recordedOff, replayAt),
|
|
1618
|
+
}
|
|
1619
|
+
: undefined,
|
|
1293
1620
|
sourceScopeId: opts?.sourceScopeId ?? dump.scopeId,
|
|
1294
1621
|
});
|
|
1295
1622
|
await this.recordAdmin(actor, 'restoreScope', { tenantId, scopeId }, null, { sourceScopeId: dump.scopeId, tables: dump.tables.length, capturedAt: dump.capturedAt });
|
|
@@ -1381,7 +1708,7 @@ export class CloudflareScopeHost {
|
|
|
1381
1708
|
async getScope(principal, tenantId, scopeId, options) {
|
|
1382
1709
|
// Lifecycle gates (control-plane.md §4.1/§4.2), the K-3 fail-closed path,
|
|
1383
1710
|
// evaluated durably in the ControlPlaneDO. A throw propagates here.
|
|
1384
|
-
await this.
|
|
1711
|
+
await this.assertLive(tenantId, scopeId);
|
|
1385
1712
|
await this.migrateAndRecord(scopeId);
|
|
1386
1713
|
return this.buildStub(tenantId, scopeId, principal, undefined, undefined, options);
|
|
1387
1714
|
}
|
|
@@ -1456,7 +1783,7 @@ export class CloudflareScopeHost {
|
|
|
1456
1783
|
const record = await this.resolveImpersonation(session, tenantId, scopeId);
|
|
1457
1784
|
// The same lifecycle gate the principal door applies: a suspended tenant
|
|
1458
1785
|
// refuses a support session exactly as it refuses a user.
|
|
1459
|
-
await this.
|
|
1786
|
+
await this.assertLive(tenantId, scopeId);
|
|
1460
1787
|
await this.migrateAndRecord(scopeId);
|
|
1461
1788
|
return this.buildStub(tenantId, scopeId, record.principal, undefined, undefined, options, record.id);
|
|
1462
1789
|
}
|
|
@@ -1467,12 +1794,93 @@ export class CloudflareScopeHost {
|
|
|
1467
1794
|
* against `system:<moduleId>` grants inside the stub.
|
|
1468
1795
|
*/
|
|
1469
1796
|
async getSystemScope(moduleId, tenantId, scopeId) {
|
|
1797
|
+
return this.buildStub(tenantId, scopeId, undefined, undefined, await this.openSystemDoor(moduleId, tenantId, scopeId));
|
|
1798
|
+
}
|
|
1799
|
+
/**
|
|
1800
|
+
* #1834: the system door — the ONE place every entry acting as `system:<moduleId>` passes
|
|
1801
|
+
* through: a schedule's fire, a job run's `pass.scope()`, a module's attachment open. The
|
|
1802
|
+
* module must be registered here and the scope live, as before. Then the door GATES: the
|
|
1803
|
+
* scope's own state read for the module, and after it the rewind hold (#1819), in the order
|
|
1804
|
+
* `SWITCH_HOLD_SETTLE_MS` relies on. A module the hold keeps off is refused here, as a
|
|
1805
|
+
* schedule pass skips it.
|
|
1806
|
+
*
|
|
1807
|
+
* A gate read is about the instance that answered it, and a door can be held for as long as
|
|
1808
|
+
* a job pass runs. So every call through the door is PINNED to that instance: the DO refuses
|
|
1809
|
+
* one that lands on another (`SYSTEM_DOOR_MOVED`), since a PITR restore always restarts the
|
|
1810
|
+
* object, and the door gates again and retries. A restart within one call's round trip is
|
|
1811
|
+
* rare, so past `SYSTEM_DOOR_REGATES` it fails closed (`unavailable`). No clock is involved:
|
|
1812
|
+
* the gate and the call meet the same storage, or the call does not run.
|
|
1813
|
+
*/
|
|
1814
|
+
async openSystemDoor(moduleId, tenantId, scopeId, gated,
|
|
1815
|
+
/** #1834: the job pass this door is opened for; its "not now" refusals are tied to it. */
|
|
1816
|
+
pass) {
|
|
1470
1817
|
if (!this.moduleIds.has(moduleId)) {
|
|
1471
1818
|
throw new Error(`module not registered on this host: ${moduleId}`);
|
|
1472
1819
|
}
|
|
1473
|
-
await this.
|
|
1820
|
+
await this.assertLive(tenantId, scopeId);
|
|
1474
1821
|
await this.migrateAndRecord(scopeId);
|
|
1475
|
-
|
|
1822
|
+
// A gate this door reads itself reports a hold it could not read; one handed in was the
|
|
1823
|
+
// caller's to report (`runDueSchedules` puts it on its pass report).
|
|
1824
|
+
const regate = async () => {
|
|
1825
|
+
const fresh = await this.systemDoorGate(scopeId, moduleId);
|
|
1826
|
+
if (fresh.error)
|
|
1827
|
+
console.error(`substrat: system door ${moduleId} on ${scopeId}: ${fresh.error}`);
|
|
1828
|
+
return fresh;
|
|
1829
|
+
};
|
|
1830
|
+
let gate = gated ?? (await regate());
|
|
1831
|
+
return {
|
|
1832
|
+
moduleId,
|
|
1833
|
+
through: async (call) => {
|
|
1834
|
+
for (let regates = 0;; regates += 1) {
|
|
1835
|
+
if (gate.held) {
|
|
1836
|
+
// The hold ends when the switch is applied again: a job run waits it out without
|
|
1837
|
+
// spending its retries, because the driver defers on THIS refusal (`doorWait`).
|
|
1838
|
+
throw this.doorWait(pass, substratError('forbidden', `module '${moduleId}' is held off on this scope: it was rewound to before its schedule ` +
|
|
1839
|
+
'switch was turned off, and it stays off until the switch is applied again', { reason: SYSTEM_DOOR_WAIT }));
|
|
1840
|
+
}
|
|
1841
|
+
const answer = await call(gate.instance);
|
|
1842
|
+
if (!isSystemDoorMoved(answer))
|
|
1843
|
+
return answer;
|
|
1844
|
+
if (regates >= SYSTEM_DOOR_REGATES) {
|
|
1845
|
+
throw this.doorWait(pass, substratError('unavailable', `the scope kept restarting under the system door of module '${moduleId}' ` +
|
|
1846
|
+
`(${regates + 1} attempts); nothing ran. Retry later`, { reason: SYSTEM_DOOR_WAIT }));
|
|
1847
|
+
}
|
|
1848
|
+
gate = await regate();
|
|
1849
|
+
}
|
|
1850
|
+
},
|
|
1851
|
+
};
|
|
1852
|
+
}
|
|
1853
|
+
/**
|
|
1854
|
+
* #1834: the refusals this host's system door threw that mean "not now" (a hold, a scope that
|
|
1855
|
+
* kept restarting), each tied by identity to the job pass whose door threw it. The job driver
|
|
1856
|
+
* defers a pass only on one of these, never on an error's shape, which any step or operation
|
|
1857
|
+
* could copy. A mark is ORIGIN and FRESHNESS together: it is consumed when the driver takes it,
|
|
1858
|
+
* and only the pass it names can take it, so a handler that keeps a refusal and throws it again
|
|
1859
|
+
* on a later pass is an ordinary failure there. Held weakly: an error nobody holds goes.
|
|
1860
|
+
*/
|
|
1861
|
+
doorWaits = new WeakMap();
|
|
1862
|
+
/** #1834: mark this door's own refusal as "not now" for the pass it was opened for, and hand it back. */
|
|
1863
|
+
doorWait(pass, err) {
|
|
1864
|
+
if (pass)
|
|
1865
|
+
this.doorWaits.set(err, pass);
|
|
1866
|
+
return err;
|
|
1867
|
+
}
|
|
1868
|
+
/** #1834: is `err` a "not now" this host's door threw on `pass`? Consumes the mark when it is. */
|
|
1869
|
+
takeDoorWait(err, pass) {
|
|
1870
|
+
if (typeof err !== 'object' || err === null || this.doorWaits.get(err) !== pass)
|
|
1871
|
+
return false;
|
|
1872
|
+
this.doorWaits.delete(err);
|
|
1873
|
+
return true;
|
|
1874
|
+
}
|
|
1875
|
+
/**
|
|
1876
|
+
* #1834: the system door's gate — the scope's state read for the module, the instance that
|
|
1877
|
+
* answered it, and then (only for a module the scope has on) the rewind hold (#1819). The
|
|
1878
|
+
* hold is read AFTER the state, which is the order `switchHeld` needs.
|
|
1879
|
+
*/
|
|
1880
|
+
async systemDoorGate(scopeId, moduleId) {
|
|
1881
|
+
const { state, instance } = await this.scopeStub(scopeId).systemDoorState(moduleId);
|
|
1882
|
+
const hold = state === 'on' ? await this.switchHeld(scopeId, moduleId) : { held: false };
|
|
1883
|
+
return { state, instance, held: hold.held, ...(hold.error ? { error: hold.error } : {}) };
|
|
1476
1884
|
}
|
|
1477
1885
|
/**
|
|
1478
1886
|
* #1705: the consumer half. The same lifecycle gate as every door (K-3), then the ScopeDO
|
|
@@ -1503,7 +1911,7 @@ export class CloudflareScopeHost {
|
|
|
1503
1911
|
* capability row lives. The answer is re-parsed on this side of the RPC.
|
|
1504
1912
|
*/
|
|
1505
1913
|
async exchangeCapability(tenantId, scopeId, secret, options) {
|
|
1506
|
-
await this.
|
|
1914
|
+
await this.assertLive(tenantId, scopeId);
|
|
1507
1915
|
await this.migrateAndRecord(scopeId);
|
|
1508
1916
|
const outcome = await this.scopeStub(scopeId).exchangeCapability(secret, tenantId, scopeId, options?.mode);
|
|
1509
1917
|
return capabilityExchange.nullable().parse(outcome);
|
|
@@ -1517,7 +1925,7 @@ export class CloudflareScopeHost {
|
|
|
1517
1925
|
if (!plausibleSessionToken(sessionToken)) {
|
|
1518
1926
|
throw substratError('unauthenticated', 'not a capability session token');
|
|
1519
1927
|
}
|
|
1520
|
-
await this.
|
|
1928
|
+
await this.assertLive(tenantId, scopeId);
|
|
1521
1929
|
await this.migrateAndRecord(scopeId);
|
|
1522
1930
|
const hash = await capabilityTokenHash(sessionToken);
|
|
1523
1931
|
return this.buildStub(tenantId, scopeId, undefined, undefined, undefined, options, undefined, hash);
|
|
@@ -1554,7 +1962,7 @@ export class CloudflareScopeHost {
|
|
|
1554
1962
|
// provisioning receipt is K-3 here (#1738), read before anything migrates the DO, and the
|
|
1555
1963
|
// refusal reads as the pure host's does: an unknown scope, not an unknown role.
|
|
1556
1964
|
if (this.cpLess && !(await this.scopeStub(scopeId).servesTenant(tenantId))) {
|
|
1557
|
-
throw
|
|
1965
|
+
throw unknownScopeForTenant(tenantId, scopeId);
|
|
1558
1966
|
}
|
|
1559
1967
|
await this.peerScopeGate(tenantId, scopeId, 'canAssign');
|
|
1560
1968
|
const bound = await this.scopeStub(scopeId).canAssignFor(tenantId, scopeId, principalId.parse(principal), roleKey);
|
|
@@ -1564,7 +1972,7 @@ export class CloudflareScopeHost {
|
|
|
1564
1972
|
}
|
|
1565
1973
|
async assignScopeRoleBounded(tenantId, scopeId, caller, assignee, roleKey) {
|
|
1566
1974
|
if (this.cpLess && !(await this.scopeStub(scopeId).servesTenant(tenantId))) {
|
|
1567
|
-
throw
|
|
1975
|
+
throw unknownScopeForTenant(tenantId, scopeId);
|
|
1568
1976
|
}
|
|
1569
1977
|
await this.peerScopeGate(tenantId, scopeId, 'assignScopeRoleBounded');
|
|
1570
1978
|
const bound = await this.scopeStub(scopeId).assignScopeRoleBoundedFor(tenantId, scopeId, principalId.parse(caller), principalId.parse(assignee), roleKey);
|
|
@@ -1589,11 +1997,11 @@ export class CloudflareScopeHost {
|
|
|
1589
1997
|
if (!this.cpLess) {
|
|
1590
1998
|
const record = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
1591
1999
|
if (!record) {
|
|
1592
|
-
throw
|
|
2000
|
+
throw unknownScopeForTenant(tenantId, scopeId);
|
|
1593
2001
|
}
|
|
1594
2002
|
this.assertServedHere(record, scopeId, verb);
|
|
1595
2003
|
}
|
|
1596
|
-
await this.
|
|
2004
|
+
await this.assertLive(tenantId, scopeId);
|
|
1597
2005
|
await this.migrateAndRecord(scopeId);
|
|
1598
2006
|
}
|
|
1599
2007
|
/**
|
|
@@ -1606,7 +2014,7 @@ export class CloudflareScopeHost {
|
|
|
1606
2014
|
if (!plausibleSessionToken(sessionToken)) {
|
|
1607
2015
|
throw substratError('unauthenticated', 'not a capability session token');
|
|
1608
2016
|
}
|
|
1609
|
-
await this.
|
|
2017
|
+
await this.assertLive(tenantId, scopeId);
|
|
1610
2018
|
await this.migrateAndRecord(scopeId);
|
|
1611
2019
|
const store = await this.resolveAttachmentStore(tenantId);
|
|
1612
2020
|
const capabilitySession = await capabilityTokenHash(sessionToken);
|
|
@@ -1657,7 +2065,7 @@ export class CloudflareScopeHost {
|
|
|
1657
2065
|
async capabilityScopeStub(tenantId, scopeId, verb) {
|
|
1658
2066
|
const rec = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
1659
2067
|
if (!rec)
|
|
1660
|
-
throw
|
|
2068
|
+
throw unknownScopeForTenant(tenantId, scopeId);
|
|
1661
2069
|
if (this.servesScopesElsewhereNow && rec.vertical !== null) {
|
|
1662
2070
|
throw substratError('unavailable', `${verb} cannot reach scope ${scopeId}: it is served by the '${rec.vertical}' ` +
|
|
1663
2071
|
'deployment, and platform capability verbs are not delegated there yet');
|
|
@@ -1703,6 +2111,12 @@ export class CloudflareScopeHost {
|
|
|
1703
2111
|
if (!rec || rec.status !== 'active')
|
|
1704
2112
|
return report;
|
|
1705
2113
|
}
|
|
2114
|
+
else if (lifecycleRefusal(await this.validateScopeAccess(tenantId, scopeId)) !== null) {
|
|
2115
|
+
// #2016: the pair is held to the scope's own record first, so a sweep roster entry whose
|
|
2116
|
+
// tenant disagrees with the scope is refused before the probe reads or writes anything.
|
|
2117
|
+
// #1713: a held scope emits nothing, so judging it stale would only raise a false alarm.
|
|
2118
|
+
return report;
|
|
2119
|
+
}
|
|
1706
2120
|
const stub = this.scopeStub(scopeId);
|
|
1707
2121
|
const windows = new Map();
|
|
1708
2122
|
for (const specs of this.moduleFreshness.values()) {
|
|
@@ -1741,29 +2155,40 @@ export class CloudflareScopeHost {
|
|
|
1741
2155
|
return report;
|
|
1742
2156
|
// Only run on a live scope of this tenant; a scope archived between the sweep's
|
|
1743
2157
|
// enumeration and here simply has nothing due. A CP-less host has no directory
|
|
1744
|
-
// to ask (#461)
|
|
1745
|
-
//
|
|
1746
|
-
//
|
|
2158
|
+
// to ask (#461), so it reads the lifecycle the platform delivered to the scope (#1713), in
|
|
2159
|
+
// the same call that holds the pair to the scope's own record (#2016): a roster entry whose
|
|
2160
|
+
// tenant disagrees is refused before a grant, a cadence row or a run is read or written.
|
|
1747
2161
|
if (!this.cpLess) {
|
|
1748
2162
|
const rec = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
1749
2163
|
if (!rec || rec.status !== 'active')
|
|
1750
2164
|
return report;
|
|
1751
2165
|
}
|
|
2166
|
+
else if (lifecycleRefusal(await this.validateScopeAccess(tenantId, scopeId)) !== null) {
|
|
2167
|
+
// #1713: the lifecycle the platform delivered holds the scope. Every schedule is
|
|
2168
|
+
// `skipped` and no cadence row moves, as under the kill switch, so a schedule that
|
|
2169
|
+
// came due meanwhile fires once on the first pass after the scope is live again.
|
|
2170
|
+
for (const schedule of schedules) {
|
|
2171
|
+
report.skipped += 1;
|
|
2172
|
+
report.runs.push({ operation: schedule.operation, outcome: 'skipped' });
|
|
2173
|
+
}
|
|
2174
|
+
report.lifecycleHeld = true;
|
|
2175
|
+
return report;
|
|
2176
|
+
}
|
|
1752
2177
|
const stub = this.scopeStub(scopeId);
|
|
1753
2178
|
// The grant IS the switch (#383), and the kill switch is its lever (#1666): the
|
|
1754
2179
|
// kernel's `systemScheduleState`, the one predicate both adapters run. A scope that
|
|
1755
2180
|
// never held the module's grant (a foreign vertical's) is a quiet no-op, exactly as
|
|
1756
2181
|
// before. A scope switched OFF reports every schedule `skipped`, never `failed`, and
|
|
1757
2182
|
// does not touch its cadence rows — so a restore fires a due schedule on the next pass.
|
|
1758
|
-
|
|
1759
|
-
|
|
2183
|
+
// #1819: a scope rewound to before its switch was pulled answers `on`, and the rewind held
|
|
2184
|
+
// the module outside the scope. Both are read by the system door's own gate (#1834), which
|
|
2185
|
+
// every fire below then goes through, pinned to the instance this read came from.
|
|
2186
|
+
const gate = await this.systemDoorGate(scopeId, moduleId);
|
|
2187
|
+
if (gate.state === 'ungranted')
|
|
1760
2188
|
return report;
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
if (hold.error)
|
|
1765
|
-
report.errors.push({ operation: 'switch-hold', error: hold.error });
|
|
1766
|
-
if (state === 'off' || hold.held) {
|
|
2189
|
+
if (gate.error)
|
|
2190
|
+
report.errors.push({ operation: 'switch-hold', error: gate.error });
|
|
2191
|
+
if (gate.state === 'off' || gate.held) {
|
|
1767
2192
|
for (const schedule of schedules) {
|
|
1768
2193
|
report.skipped += 1;
|
|
1769
2194
|
report.runs.push({ operation: schedule.operation, outcome: 'skipped' });
|
|
@@ -1772,6 +2197,8 @@ export class CloudflareScopeHost {
|
|
|
1772
2197
|
return report;
|
|
1773
2198
|
}
|
|
1774
2199
|
const now = Date.now();
|
|
2200
|
+
const lines = asyncLinePass();
|
|
2201
|
+
let door;
|
|
1775
2202
|
for (const schedule of schedules) {
|
|
1776
2203
|
const last = await stub.scheduleLastRun(schedule.operation);
|
|
1777
2204
|
const lastRun = last ? Date.parse(last) : null;
|
|
@@ -1786,28 +2213,55 @@ export class CloudflareScopeHost {
|
|
|
1786
2213
|
// produces either way carries it, and so does the row below.
|
|
1787
2214
|
const invocationId = ulid();
|
|
1788
2215
|
let status = 'ok';
|
|
2216
|
+
// #1901: the run's line, under the call's own id — the one its `ctx.log` lines carry.
|
|
2217
|
+
const startedAt = Date.now();
|
|
2218
|
+
let emitted;
|
|
2219
|
+
let failure;
|
|
1789
2220
|
try {
|
|
1790
|
-
|
|
1791
|
-
|
|
2221
|
+
// The gate above already answered for this pass; a fire that meets a restarted scope
|
|
2222
|
+
// is gated again by the door (#1834).
|
|
2223
|
+
door ??= await this.openSystemDoor(moduleId, tenantId, scopeId, gate);
|
|
2224
|
+
const scope = this.buildStub(tenantId, scopeId, undefined, undefined, door);
|
|
2225
|
+
await scope.invoke(schedule.operation, schedule.input, { invocationId, onEmitted: (r) => (emitted = r) });
|
|
1792
2226
|
report.fired += 1;
|
|
1793
2227
|
}
|
|
1794
2228
|
catch (err) {
|
|
1795
2229
|
status = 'failed';
|
|
2230
|
+
failure = { error: err };
|
|
1796
2231
|
report.failed += 1;
|
|
1797
2232
|
report.errors.push({
|
|
1798
2233
|
operation: schedule.operation,
|
|
1799
2234
|
error: err instanceof Error ? err.message : String(err),
|
|
1800
2235
|
});
|
|
1801
2236
|
}
|
|
2237
|
+
lines.write({
|
|
2238
|
+
kind: 'schedule',
|
|
2239
|
+
tenantId,
|
|
2240
|
+
scopeId,
|
|
2241
|
+
invocationId,
|
|
2242
|
+
operation: schedule.operation,
|
|
2243
|
+
startedAt,
|
|
2244
|
+
outcome: status,
|
|
2245
|
+
...failure,
|
|
2246
|
+
dueAt: lastRun === null ? null : new Date(dueAt).toISOString(),
|
|
2247
|
+
latenessMs: lastRun === null ? null : now - dueAt,
|
|
2248
|
+
...(emitted ? { emitted } : {}),
|
|
2249
|
+
});
|
|
1802
2250
|
// #1288: 'schedule', whatever this operation happens to be called — including
|
|
1803
2251
|
// `freshness:<something>`, which is exactly the row the evaluator no longer eats.
|
|
1804
2252
|
await stub.recordScheduleRun(schedule.operation, new Date(now).toISOString(), status, 'schedule', invocationId);
|
|
1805
2253
|
report.runs.push({ operation: schedule.operation, outcome: status === 'ok' ? 'ok' : 'failed' });
|
|
1806
2254
|
}
|
|
2255
|
+
lines.end();
|
|
1807
2256
|
return report;
|
|
1808
2257
|
}
|
|
1809
2258
|
/** The stub body, shared by the principal, connection, system and impersonation doors. */
|
|
1810
|
-
buildStub(tenantId, scopeId, principal, connectionId,
|
|
2259
|
+
buildStub(tenantId, scopeId, principal, connectionId,
|
|
2260
|
+
/**
|
|
2261
|
+
* #1834: the system door this stub acts through. The ONLY way to make a stub acting as
|
|
2262
|
+
* `system:<moduleId>`: the module is the door's, and every invoke is gated and pinned by it.
|
|
2263
|
+
*/
|
|
2264
|
+
systemDoor, options,
|
|
1811
2265
|
/**
|
|
1812
2266
|
* K-42: the session this stub acts under. Re-read from the directory on
|
|
1813
2267
|
* EVERY invoke rather than captured here — a stub is a capability and
|
|
@@ -1828,6 +2282,7 @@ export class CloudflareScopeHost {
|
|
|
1828
2282
|
*/
|
|
1829
2283
|
verticalCaller) {
|
|
1830
2284
|
const stub = this.scopeStub(scopeId);
|
|
2285
|
+
const systemModuleId = systemDoor?.moduleId;
|
|
1831
2286
|
const cp = this.cp;
|
|
1832
2287
|
const operationEntitlement = this.operationEntitlement;
|
|
1833
2288
|
// The DO needs SOME principal-shaped value for `ctx.principal`; for a
|
|
@@ -1882,7 +2337,15 @@ export class CloudflareScopeHost {
|
|
|
1882
2337
|
}
|
|
1883
2338
|
// K-42: resolved per invoke, so expiry and `endImpersonation` both bite.
|
|
1884
2339
|
const session = sessionId === undefined ? undefined : await this.resolveImpersonation(sessionId, tenantId, scopeId);
|
|
1885
|
-
const
|
|
2340
|
+
const send = (systemDoorInstance) => stub.invoke(operation, input, asPrincipalId, tenantId, scopeId, connectionId, requiredKey, systemModuleId, true, invokeOptions, session, capabilitySession, verticalCaller, systemDoorInstance);
|
|
2341
|
+
// #1834: through the system door, pinned to the instance its gate read. The DO's
|
|
2342
|
+
// refusal comes back as a failure envelope, so it is rethrown for the door to re-gate.
|
|
2343
|
+
const envelope = systemDoor
|
|
2344
|
+
? await systemDoor.through(async (instance) => {
|
|
2345
|
+
const sent = await send(instance);
|
|
2346
|
+
return sent.systemDoorMoved === true ? SYSTEM_DOOR_MOVED : sent;
|
|
2347
|
+
})
|
|
2348
|
+
: await send();
|
|
1886
2349
|
// The operation failed and the DO handed the error back as DATA — so it still
|
|
1887
2350
|
// has its code and extensions, which a throw across this boundary would have
|
|
1888
2351
|
// stripped down to a message (#113 §3). Rethrown here, where the caller expects
|
|
@@ -1931,6 +2394,12 @@ export class CloudflareScopeHost {
|
|
|
1931
2394
|
throw substratError('unavailable', `${operation} ran on a scope host that did not understand peer callers. ` +
|
|
1932
2395
|
'Retry once the scope has been migrated');
|
|
1933
2396
|
}
|
|
2397
|
+
// #1834, the same shape: a DO too old to know the system door's pin ran the call
|
|
2398
|
+
// without checking it met the storage the door's gate read.
|
|
2399
|
+
if (systemDoor !== undefined && envelope.systemDoor?.honoured !== true) {
|
|
2400
|
+
throw substratError('unavailable', `${operation} ran on a scope host that did not understand the system door's pin. ` +
|
|
2401
|
+
'Retry once the scope has been migrated');
|
|
2402
|
+
}
|
|
1934
2403
|
if (session !== undefined && envelope.impersonation?.honoured !== true) {
|
|
1935
2404
|
throw substratError('unavailable', `${operation} ran on a scope host that did not understand impersonation — the ` +
|
|
1936
2405
|
'operation may have run unrecorded and unbounded. Retry once the scope has ' +
|
|
@@ -2125,6 +2594,8 @@ export class CloudflareScopeHost {
|
|
|
2125
2594
|
// returns it with the previous status, so the trail cannot disagree with
|
|
2126
2595
|
// the directory about which deployment the action touched.
|
|
2127
2596
|
await this.recordAdmin(actor, action, { tenantId, scopeId, vertical: before.vertical }, { status: before.status }, { status: to, ...afterExtra });
|
|
2597
|
+
// #1713: after the directory moved and the move was audited, never before or instead.
|
|
2598
|
+
await this.deliverLifecycles(actor, { scopeId });
|
|
2128
2599
|
};
|
|
2129
2600
|
const writeGrant = async (subject, permission, node, entity, expiresAt) => {
|
|
2130
2601
|
if (entity) {
|
|
@@ -2161,7 +2632,7 @@ export class CloudflareScopeHost {
|
|
|
2161
2632
|
if (!this.cpLess) {
|
|
2162
2633
|
const rec = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
2163
2634
|
if (!rec)
|
|
2164
|
-
throw
|
|
2635
|
+
throw unknownScopeForTenant(tenantId, scopeId);
|
|
2165
2636
|
vertical = rec.vertical;
|
|
2166
2637
|
}
|
|
2167
2638
|
const action = to === 'off' ? 'revokeFromPeer' : 'restoreToPeer';
|
|
@@ -2231,21 +2702,50 @@ export class CloudflareScopeHost {
|
|
|
2231
2702
|
if (!this.cpLess) {
|
|
2232
2703
|
const rec = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
2233
2704
|
if (!rec)
|
|
2234
|
-
throw
|
|
2705
|
+
throw unknownScopeForTenant(tenantId, scopeId);
|
|
2235
2706
|
vertical = rec.vertical;
|
|
2236
2707
|
}
|
|
2237
2708
|
const delegation = this.cpLess || vertical !== null ? this.systemSwitchDelegation : undefined;
|
|
2238
2709
|
// #1819: a scope whose store is here was rewound here, so `switchInScope` releases its
|
|
2239
2710
|
// hold here too. A delegated move releases in the deployment serving the scope.
|
|
2240
|
-
|
|
2241
|
-
|
|
2242
|
-
|
|
2243
|
-
|
|
2711
|
+
// #1823: a module whose only authority on the scope is a TENANT-level grant has nothing in
|
|
2712
|
+
// the scope's storage for the switch to find, so the directory says whether it is held.
|
|
2713
|
+
// A caller moving several modules reads them in one call and passes each answer in.
|
|
2714
|
+
// An OFF of a tenant-held module is refused, after the far end answers, unless that answer
|
|
2715
|
+
// attests the evaluator that denies a tenant-level grant (`deniesTenantGrants`). A
|
|
2716
|
+
// deployment built before #1823 drops `tenantHeld`, still answers `held: true` (the module's
|
|
2717
|
+
// scope-level grants were there to switch), and keeps authorizing the tenant-level grant —
|
|
2718
|
+
// so its `held` alone would have the platform record a scope off that is not. Every OFF the
|
|
2719
|
+
// platform drives moves through here: an operator's (`switchSystem`, which then puts the
|
|
2720
|
+
// switch back) and a re-assert's, which every carry ends with.
|
|
2721
|
+
const move = async (moduleId, to, at, tenantHeld) => {
|
|
2722
|
+
const held = tenantHeld ?? (await this.cp.tenantHeldSystemModules(tenantId, [moduleId], at)).length > 0;
|
|
2723
|
+
const outcome = await (delegation
|
|
2724
|
+
? delegation.switch({ tenantId, scopeId, moduleId, to, tenantHeld: held })
|
|
2725
|
+
: this.switchInScope(scopeId, moduleId, to, at, held));
|
|
2726
|
+
if (to === 'off' && held && outcome.held && outcome.deniesTenantGrants !== true) {
|
|
2727
|
+
throw new UnattestedSwitchOff(scopeId, moduleId, outcome);
|
|
2728
|
+
}
|
|
2729
|
+
return outcome;
|
|
2730
|
+
};
|
|
2731
|
+
/**
|
|
2732
|
+
* Is the module switched off on the scope right now? Read from wherever `move` writes — the
|
|
2733
|
+
* deployment's status read or this scope's own — for a move that threw (#1823): the switch
|
|
2734
|
+
* may already have moved before the throw (`switchInScope` moves it, then works the rewind
|
|
2735
|
+
* hold), and then the directory's record must stay.
|
|
2736
|
+
*/
|
|
2737
|
+
const isOff = async (moduleId) => {
|
|
2738
|
+
const rows = delegation
|
|
2739
|
+
? await delegation.status({ tenantId, scopeId })
|
|
2740
|
+
: systemScheduleEntry.array().parse(await this.scopeStub(scopeId).systemGrantsStatus());
|
|
2741
|
+
return rows.some((r) => r.moduleId === moduleId && r.schedules === 'off');
|
|
2742
|
+
};
|
|
2743
|
+
return { vertical, move, isOff };
|
|
2244
2744
|
};
|
|
2245
2745
|
const switchSystem = async (actor, raw, to) => {
|
|
2246
2746
|
const input = systemSwitch.parse(raw);
|
|
2247
2747
|
const { tenantId, scopeId } = input.node;
|
|
2248
|
-
const { vertical, move } = await systemSwitchTarget(tenantId, scopeId);
|
|
2748
|
+
const { vertical, move, isOff } = await systemSwitchTarget(tenantId, scopeId);
|
|
2249
2749
|
// AUDIT FIRST (#1666 review): the intent row lands before anything moves, and the
|
|
2250
2750
|
// outcome row after — every attempt, a repeat included. The scope's store and the
|
|
2251
2751
|
// admin log are separate, so no order makes the pair atomic; this one fails toward
|
|
@@ -2256,37 +2756,37 @@ export class CloudflareScopeHost {
|
|
|
2256
2756
|
const target = { tenantId, scopeId, vertical };
|
|
2257
2757
|
const base = { operationId, moduleId: input.moduleId, schedules: to };
|
|
2258
2758
|
await this.recordAdmin(actor, action, target, null, { ...base, phase: 'intent', reason: input.reason });
|
|
2259
|
-
// The directory's record (#1674)
|
|
2260
|
-
//
|
|
2759
|
+
// The directory's record (#1674), written BEFORE the scope moves, both ways, and undone
|
|
2760
|
+
// if the move throws or holds nothing — see `recordSystemSwitchedOn` and (#1823)
|
|
2761
|
+
// `recordSystemSwitchedOff` for why that order is the safe one.
|
|
2261
2762
|
const at = new Date().toISOString();
|
|
2262
2763
|
const record = { tenantId, scopeId, moduleId: input.moduleId, actor, reason: input.reason, operationId, at };
|
|
2263
2764
|
const errorOf = (err) => (err instanceof Error ? err.message : String(err));
|
|
2264
|
-
let prior
|
|
2265
|
-
|
|
2266
|
-
|
|
2267
|
-
|
|
2268
|
-
|
|
2269
|
-
|
|
2270
|
-
|
|
2271
|
-
|
|
2272
|
-
|
|
2273
|
-
|
|
2274
|
-
}
|
|
2765
|
+
let prior;
|
|
2766
|
+
try {
|
|
2767
|
+
prior = to === 'on' ? await this.cp.recordSystemSwitchedOn(record) : await this.cp.recordSystemSwitchedOff(record);
|
|
2768
|
+
}
|
|
2769
|
+
catch (err) {
|
|
2770
|
+
// Nothing has moved: fail the call here, audited. An ON must not switch a scope on
|
|
2771
|
+
// whose record still says off (the next reconcile would switch it back off), and an
|
|
2772
|
+
// OFF must not leave a scope off that the record does not know of (#1823).
|
|
2773
|
+
await this.recordAdmin(actor, action, target, null, { ...base, phase: 'failed', error: errorOf(err) }).catch(() => undefined);
|
|
2774
|
+
throw err;
|
|
2275
2775
|
}
|
|
2276
2776
|
/**
|
|
2277
|
-
*
|
|
2278
|
-
* is answered rather than swallowed — it lands on the outcome row as
|
|
2279
|
-
* the call
|
|
2280
|
-
* turns into a module running again.
|
|
2777
|
+
* The record's undo, after the scope's move threw or held nothing: retried once, and a
|
|
2778
|
+
* failure of both is answered rather than swallowed — it lands on the outcome row as
|
|
2779
|
+
* `recordError`, and the call says so.
|
|
2281
2780
|
*/
|
|
2282
|
-
const
|
|
2781
|
+
const undoRecord = async () => {
|
|
2782
|
+
const undo = () => this.cp.restoreSystemSwitchRecord(record, prior);
|
|
2283
2783
|
try {
|
|
2284
|
-
await
|
|
2784
|
+
await undo();
|
|
2285
2785
|
return null;
|
|
2286
2786
|
}
|
|
2287
2787
|
catch {
|
|
2288
2788
|
try {
|
|
2289
|
-
await
|
|
2789
|
+
await undo();
|
|
2290
2790
|
return null;
|
|
2291
2791
|
}
|
|
2292
2792
|
catch (err) {
|
|
@@ -2299,25 +2799,54 @@ export class CloudflareScopeHost {
|
|
|
2299
2799
|
outcome = await move(input.moduleId, to, at);
|
|
2300
2800
|
}
|
|
2301
2801
|
catch (err) {
|
|
2302
|
-
|
|
2802
|
+
if (err instanceof UnattestedSwitchOff) {
|
|
2803
|
+
// The far end switched, and cannot hold it against the tenant-level grant. Refused:
|
|
2804
|
+
// ON gives back exactly what this OFF took (nothing, when it changed nothing), and
|
|
2805
|
+
// then the record goes back too — neither the scope nor the directory says off. An ON
|
|
2806
|
+
// that fails here leaves the scope off, and the readback rule below keeps its record.
|
|
2807
|
+
const putBack = err.outcome.changed
|
|
2808
|
+
? await move(input.moduleId, 'on', new Date().toISOString(), true).then(() => true, () => false)
|
|
2809
|
+
: true;
|
|
2810
|
+
if (putBack) {
|
|
2811
|
+
const recordError = await undoRecord();
|
|
2812
|
+
await this.recordAdmin(actor, action, target, null, {
|
|
2813
|
+
...base,
|
|
2814
|
+
phase: 'refused',
|
|
2815
|
+
error: err.message,
|
|
2816
|
+
...(recordError ? { recordError } : {}),
|
|
2817
|
+
}).catch(() => undefined);
|
|
2818
|
+
throw substratError('precondition_failed', `${err.message} Nothing was switched` +
|
|
2819
|
+
(recordError ? `, but its directory record could not be put back (${recordError}).` : '.'));
|
|
2820
|
+
}
|
|
2821
|
+
}
|
|
2822
|
+
// A throw does not say the scope did not move: the move crosses the scope's DO and then
|
|
2823
|
+
// the rewind hold, and a throw from the second leaves the scope's switch moved. So the
|
|
2824
|
+
// record is undone only when the scope reads back in the OTHER position — an OFF that
|
|
2825
|
+
// reads not off, an ON that still reads off — and kept when it reads back as asked.
|
|
2826
|
+
// Undoing an ON that landed would put the record back to `off` beside a scope that is
|
|
2827
|
+
// on, and the next re-assert would switch the operator's ON back off. A read that fails
|
|
2828
|
+
// too counts as off, both ways, so what is left fails toward off: an OFF keeps its
|
|
2829
|
+
// record (never a scope switched off with no record — a wipe or restore would have
|
|
2830
|
+
// nothing to re-assert, and #1743's refusal nothing to read), and an ON is undone back
|
|
2831
|
+
// to `off`, which the next re-assert completes if the scope did move.
|
|
2832
|
+
const off = await isOff(input.moduleId).catch(() => true);
|
|
2833
|
+
const recordKept = off === (to === 'off');
|
|
2834
|
+
const recordError = recordKept ? null : await undoRecord();
|
|
2303
2835
|
// Best effort: the original error is what the caller must see, and the intent row
|
|
2304
2836
|
// already says an attempt was made.
|
|
2305
2837
|
await this.recordAdmin(actor, action, target, null, {
|
|
2306
2838
|
...base,
|
|
2307
2839
|
phase: 'failed',
|
|
2308
2840
|
error: errorOf(err),
|
|
2841
|
+
...(recordKept ? { recordKept: true } : {}),
|
|
2309
2842
|
...(recordError ? { recordError } : {}),
|
|
2310
2843
|
}).catch(() => undefined);
|
|
2311
2844
|
throw err;
|
|
2312
2845
|
}
|
|
2313
|
-
//
|
|
2314
|
-
//
|
|
2315
|
-
// module running.
|
|
2316
|
-
const recordError =
|
|
2317
|
-
? await recordWrite(() => this.cp.recordSystemSwitchedOff(record))
|
|
2318
|
-
: to === 'on' && !outcome.held && prior
|
|
2319
|
-
? await recordWrite(() => this.cp.restoreSystemSwitchRecord(record, prior))
|
|
2320
|
-
: null;
|
|
2846
|
+
// A call that held nothing moved nothing, so its record write is undone: left `off`, it
|
|
2847
|
+
// would switch the module off the day it is installed; left `on`, the next reconcile of
|
|
2848
|
+
// a wiped scope would leave the module running.
|
|
2849
|
+
const recordError = outcome.held ? null : await undoRecord();
|
|
2321
2850
|
await this.recordAdmin(actor, action, target, null, {
|
|
2322
2851
|
...base,
|
|
2323
2852
|
phase: outcome.held ? 'applied' : 'refused',
|
|
@@ -2330,11 +2859,6 @@ export class CloudflareScopeHost {
|
|
|
2330
2859
|
`(check the module id: it is the module's manifest id, e.g. '@substrat-run/engine-absence')` +
|
|
2331
2860
|
(recordError ? `; and its directory record could not be put back (${recordError})` : ''));
|
|
2332
2861
|
}
|
|
2333
|
-
if (recordError) {
|
|
2334
|
-
throw substratError('unavailable', `module '${input.moduleId}' is switched ${to} on scope ${scopeId}, but the directory's record of it ` +
|
|
2335
|
-
`could not be written (${recordError}) — repeat the call to record it; a wipe of this scope would ` +
|
|
2336
|
-
`otherwise lose the switch`);
|
|
2337
|
-
}
|
|
2338
2862
|
return {
|
|
2339
2863
|
operationId,
|
|
2340
2864
|
moduleId: input.moduleId,
|
|
@@ -2398,7 +2922,7 @@ export class CloudflareScopeHost {
|
|
|
2398
2922
|
if (!this.cpLess) {
|
|
2399
2923
|
const rec = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
2400
2924
|
if (!rec)
|
|
2401
|
-
throw
|
|
2925
|
+
throw unknownScopeForTenant(tenantId, scopeId);
|
|
2402
2926
|
vertical = rec.vertical;
|
|
2403
2927
|
}
|
|
2404
2928
|
// A hosted scope (a vertical bound, on a non-cpLess host) has no local storage worth
|
|
@@ -2485,8 +3009,9 @@ export class CloudflareScopeHost {
|
|
|
2485
3009
|
});
|
|
2486
3010
|
}
|
|
2487
3011
|
const results = [];
|
|
3012
|
+
const tenantHeld = new Set(modules.length ? await this.cp.tenantHeldSystemModules(tenantId, modules, at) : []);
|
|
2488
3013
|
for (const moduleId of modules) {
|
|
2489
|
-
const outcome = await move(moduleId, 'off', at);
|
|
3014
|
+
const outcome = await move(moduleId, 'off', at, tenantHeld.has(moduleId));
|
|
2490
3015
|
if (outcome.changed) {
|
|
2491
3016
|
await this.recordAdmin(actor, 'reassertSystemSwitch', { tenantId, scopeId, vertical }, null, {
|
|
2492
3017
|
operationId: ulid(),
|
|
@@ -2516,7 +3041,7 @@ export class CloudflareScopeHost {
|
|
|
2516
3041
|
if (!this.cpLess) {
|
|
2517
3042
|
const rec = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
2518
3043
|
if (!rec)
|
|
2519
|
-
throw
|
|
3044
|
+
throw unknownScopeForTenant(tenantId, scopeId);
|
|
2520
3045
|
vertical = rec.vertical;
|
|
2521
3046
|
}
|
|
2522
3047
|
if (!this.cpLess && vertical !== null && !this.peerSwitchDelegation) {
|
|
@@ -2715,6 +3240,13 @@ export class CloudflareScopeHost {
|
|
|
2715
3240
|
await this.recordAccess(actor, 'listSystemSwitches', { tenantId: filter?.tenantId ?? null }, filter ?? null, rows.length);
|
|
2716
3241
|
return rows.map((r) => systemSwitchRecord.parse(r));
|
|
2717
3242
|
},
|
|
3243
|
+
tenantHeldSystemModules: async (actor, tenantId, moduleIds) => {
|
|
3244
|
+
const held = moduleIds.length
|
|
3245
|
+
? await this.cp.tenantHeldSystemModules(tenantId, moduleIds, new Date().toISOString())
|
|
3246
|
+
: [];
|
|
3247
|
+
await this.recordAccess(actor, 'tenantHeldSystemModules', { tenantId }, { moduleIds: [...moduleIds] }, held.length);
|
|
3248
|
+
return held;
|
|
3249
|
+
},
|
|
2718
3250
|
reassertSystemSwitches: reassertSystemSwitchesOf,
|
|
2719
3251
|
// #1672 — the platform's two capability verbs. Audited AFTER the write, on both, and
|
|
2720
3252
|
// the failure each leaves is the safe one: a mint whose audit row did not land never
|
|
@@ -3126,7 +3658,7 @@ export class CloudflareScopeHost {
|
|
|
3126
3658
|
if (channel === 'prod' && !incoming.deployment_ref) {
|
|
3127
3659
|
const owning = await this.cp.readVertical(verticalSlug);
|
|
3128
3660
|
if (owning && owning.owner_tenant !== null && !owning.listed) {
|
|
3129
|
-
const bound = (await this.cp.listScopes({ tenantId: owning.owner_tenant, vertical: verticalSlug, status: ['active'] })).filter(
|
|
3661
|
+
const bound = (await this.cp.listScopes({ tenantId: owning.owner_tenant, vertical: verticalSlug, status: ['active'] })).filter(isPrimaryScopeRow);
|
|
3130
3662
|
for (const s of bound) {
|
|
3131
3663
|
if (s.vertical_version_id === versionId)
|
|
3132
3664
|
continue;
|
|
@@ -3284,6 +3816,12 @@ export class CloudflareScopeHost {
|
|
|
3284
3816
|
await this.cp.setScopeServingRef(scopeId, servingRef);
|
|
3285
3817
|
await this.recordAdmin(actor, 'setScopeServingRef', { tenantId, scopeId }, { servingRef: scope.serving_ref ?? null }, { servingRef, ...(ack.exportBreak ? { acknowledged: ack } : {}) });
|
|
3286
3818
|
},
|
|
3819
|
+
recordKeptCopyResolution: async (actor, tenantId, scopeId, r) => {
|
|
3820
|
+
const scope = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
3821
|
+
if (!scope)
|
|
3822
|
+
throw substratError('not_found', `unknown scope ${scopeId} in tenant ${tenantId}`);
|
|
3823
|
+
await this.recordAdmin(actor, 'resolveKeptCopy', { tenantId, scopeId }, { script: r.script, keptAt: r.keptAt, revision: r.revisionBefore }, { action: r.action, liveScript: r.liveScript, revision: r.revisionAfter });
|
|
3824
|
+
},
|
|
3287
3825
|
setScopeExpiresAt: async (actor, tenantId, scopeId, expiresAt) => {
|
|
3288
3826
|
const scope = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
3289
3827
|
if (!scope)
|
|
@@ -3388,6 +3926,8 @@ export class CloudflareScopeHost {
|
|
|
3388
3926
|
setTenantStatus: async (actor, tenantId, status) => {
|
|
3389
3927
|
const before = await this.cp.setTenantStatus(tenantId, status);
|
|
3390
3928
|
await this.recordAdmin(actor, 'setTenantStatus', { tenantId }, { status: before }, { status });
|
|
3929
|
+
// #1713: every hosted scope under the tenant holds or resumes its work by this.
|
|
3930
|
+
await this.deliverLifecycles(actor, { tenantId });
|
|
3391
3931
|
},
|
|
3392
3932
|
setTenantName: async (actor, tenantId, name) => {
|
|
3393
3933
|
const before = await this.cp.setTenantName(tenantId, name);
|
|
@@ -4260,6 +4800,11 @@ export class CloudflareScopeHost {
|
|
|
4260
4800
|
const { tenantId, scopeId, ...after } = ownerTransferAudit.parse(entry);
|
|
4261
4801
|
await this.recordAdmin(actor, 'transferOwner', { tenantId, scopeId }, null, after);
|
|
4262
4802
|
},
|
|
4803
|
+
/** #2005: one change to a scope's copy marker, written around the vertical's own change. */
|
|
4804
|
+
recordCopyMark: async (actor, entry) => {
|
|
4805
|
+
const { tenantId, scopeId, action, ...after } = copyMarkAudit.parse(entry);
|
|
4806
|
+
await this.recordAdmin(actor, action === 'mark' ? 'markScopeCopy' : 'clearScopeCopyMark', { tenantId, scopeId }, null, after);
|
|
4807
|
+
},
|
|
4263
4808
|
accessLog: async (actor, filter) => {
|
|
4264
4809
|
const rows = await this.cp.accessLog({
|
|
4265
4810
|
actor: filter?.actor,
|
|
@@ -4472,14 +5017,225 @@ export class CloudflareScopeHost {
|
|
|
4472
5017
|
* The getScope gate (control-plane.md §4.1/§4.2) — the pair check, then the tenant's and
|
|
4473
5018
|
* the scope's lifecycle — thrown on THIS side of the RPC (#1718). The ControlPlaneDO
|
|
4474
5019
|
* answers its refusal as data, because an error thrown there arrives here flattened, with
|
|
4475
|
-
* no code; a record crosses intact.
|
|
4476
|
-
*
|
|
5020
|
+
* no code; a record crosses intact.
|
|
5021
|
+
*
|
|
5022
|
+
* A CP-less host's null control plane answers nothing: the router made the pair check from
|
|
5023
|
+
* the shared directory. The scope's own storage then makes it again (#2016), against the
|
|
5024
|
+
* tenant it was provisioned for (`ScopeDO.admission`), so a vertical that passes a scope id
|
|
5025
|
+
* under the wrong tenant meets K-3's refusal at the door instead of relying on the permission
|
|
5026
|
+
* gate's grant data to deny it.
|
|
5027
|
+
*
|
|
5028
|
+
* Answers with the lifecycle the CP-less scope holds (#1713), read in the same DO call as the
|
|
5029
|
+
* pair check, so an `assertLive` door still costs one round-trip; a door that reads the scope
|
|
5030
|
+
* without the lifecycle half pays the one DO call the pair check needs. Null with a directory,
|
|
5031
|
+
* whose refusal already covered the lifecycle.
|
|
4477
5032
|
*/
|
|
4478
5033
|
async validateScopeAccess(tenantId, scopeId) {
|
|
4479
5034
|
const refusal = await this.cp.scopeAccessRefusal(tenantId, scopeId);
|
|
4480
|
-
if (
|
|
4481
|
-
|
|
4482
|
-
|
|
5035
|
+
if (refusal)
|
|
5036
|
+
throw refusal.code ? substratError(refusal.code, refusal.message) : new Error(refusal.message);
|
|
5037
|
+
if (!this.cpLess)
|
|
5038
|
+
return null;
|
|
5039
|
+
const { verdict, lifecycle } = await this.scopeStub(scopeId).admission(tenantId);
|
|
5040
|
+
if (verdict === 'foreign')
|
|
5041
|
+
throw unknownScopeForTenant(tenantId, scopeId);
|
|
5042
|
+
// A scope with no tenant record and no role rows has nothing to hold the pair against, so it is
|
|
5043
|
+
// let through — a compatibility state, not a steady one: the next reconcile (every push runs
|
|
5044
|
+
// one against the scopes behind it, #1172) or lifecycle delivery records the directory's tenant.
|
|
5045
|
+
// Logged on every admission, so an operator sees a scope that has not converged.
|
|
5046
|
+
if (verdict === 'unknown')
|
|
5047
|
+
console.warn(JSON.stringify(tenantUnrecordedLine(tenantId, scopeId)));
|
|
5048
|
+
return lifecycle;
|
|
5049
|
+
}
|
|
5050
|
+
/**
|
|
5051
|
+
* THE lifecycle gate (#1713) at every door that runs a scope's work: a request's stubs and
|
|
5052
|
+
* attachments, a capability or impersonation session, a peer call or delivery, a subscription,
|
|
5053
|
+
* and the entry points no request asked for — the retry driver, the job runner, a schedule's
|
|
5054
|
+
* system stub, a routed connector delivery.
|
|
5055
|
+
*
|
|
5056
|
+
* `validateScopeAccess` first, which is the directory's gate where there is a directory. A
|
|
5057
|
+
* CP-less host has none, so it then reads the lifecycle the platform delivered to the scope's
|
|
5058
|
+
* own storage (`/internal/lifecycle`), judged by the kernel's `lifecycleRefusal` in the
|
|
5059
|
+
* directory's own words. The router refuses a suspended scope's requests first (#1730); this
|
|
5060
|
+
* is the deployment's own half, and the only one a timer or a retry ever meets.
|
|
5061
|
+
*
|
|
5062
|
+
* The admin and diagnostic reads (dead letters, job runs, platform requests, delivered
|
|
5063
|
+
* grants) keep `validateScopeAccess` alone: an operator still reads a suspended scope.
|
|
5064
|
+
*/
|
|
5065
|
+
async assertLive(tenantId, scopeId) {
|
|
5066
|
+
const refusal = lifecycleRefusal(await this.validateScopeAccess(tenantId, scopeId), { tenantId, scopeId });
|
|
5067
|
+
if (refusal)
|
|
5068
|
+
throw new Error(refusal);
|
|
5069
|
+
}
|
|
5070
|
+
/**
|
|
5071
|
+
* Deliver the directory's lifecycle to the deployments serving the matching scopes (#1713):
|
|
5072
|
+
* one scope after its transition, a tenant's scopes after the tenant's, or the heal sweep's
|
|
5073
|
+
* drift. Each delivery carries the directory's revisions, read in the same query as the
|
|
5074
|
+
* statuses, so a late one cannot undo a later transition: the deployment keeps only a delivery
|
|
5075
|
+
* strictly newer than the one it holds.
|
|
5076
|
+
*
|
|
5077
|
+
* Never throws. A transition is the operator's lever in an incident, and the router refuses
|
|
5078
|
+
* the scope's requests as soon as the directory moves (#1730), so a deployment that cannot be
|
|
5079
|
+
* reached must not refuse the lever. Each failure is an ops-failure row, no receipt is written,
|
|
5080
|
+
* and the heal sweep delivers again.
|
|
5081
|
+
*/
|
|
5082
|
+
async deliverLifecycles(actor, filter) {
|
|
5083
|
+
const report = { attempted: 0, delivered: 0, failed: 0 };
|
|
5084
|
+
const delegation = this.lifecycleDelegation;
|
|
5085
|
+
if (!delegation || this.cpLess)
|
|
5086
|
+
return report;
|
|
5087
|
+
let targets;
|
|
5088
|
+
// #2016: the heal also asks the served scopes whose deployment has not answered that it holds a
|
|
5089
|
+
// record of the scope's tenant, a copy as much as a primary (no push reconciles a copy), until
|
|
5090
|
+
// each answers that it does. A bounded slice per pass, after the drift and the holds.
|
|
5091
|
+
const askTenant = new Set();
|
|
5092
|
+
try {
|
|
5093
|
+
targets = await this.cp.lifecycleTargets(filter);
|
|
5094
|
+
if (filter.drift) {
|
|
5095
|
+
const unrecorded = await this.cp.lifecycleTargets({ unrecorded: true, limit: TENANT_UNRECORDED_PER_PASS });
|
|
5096
|
+
// Noted before asking, so an ask that fails (or a deployment that never answers) rotates
|
|
5097
|
+
// behind the scopes not asked yet, and the next pass reaches them.
|
|
5098
|
+
if (unrecorded.length > 0)
|
|
5099
|
+
await this.cp.recordTenantAsks(unrecorded.map((u) => u.scope_id), new Date().toISOString());
|
|
5100
|
+
const seen = new Set(targets.map((t) => t.scope_id));
|
|
5101
|
+
for (const u of unrecorded) {
|
|
5102
|
+
askTenant.add(u.scope_id);
|
|
5103
|
+
if (!seen.has(u.scope_id))
|
|
5104
|
+
targets.push(u);
|
|
5105
|
+
}
|
|
5106
|
+
}
|
|
5107
|
+
}
|
|
5108
|
+
catch (err) {
|
|
5109
|
+
console.error('substrat: could not read the scopes to deliver a lifecycle to (#1713)', err);
|
|
5110
|
+
return report;
|
|
5111
|
+
}
|
|
5112
|
+
for (const first of targets) {
|
|
5113
|
+
const tenantId = first.tenant_id;
|
|
5114
|
+
const scopeId = first.scope_id;
|
|
5115
|
+
let t = first;
|
|
5116
|
+
let lifecycle = this.lifecycleOfTarget(t);
|
|
5117
|
+
if (!lifecycle) {
|
|
5118
|
+
// A status this code does not know (a newer directory): delivering a guess could lift a hold.
|
|
5119
|
+
report.failed += 1;
|
|
5120
|
+
console.error(`substrat: scope ${scopeId} has a lifecycle this code cannot read (#1713)`);
|
|
5121
|
+
continue;
|
|
5122
|
+
}
|
|
5123
|
+
// A live scope whose deployment already acknowledged exactly this has nothing to receive,
|
|
5124
|
+
// and neither does one with no receipt in a directory never restored: a deployment holding
|
|
5125
|
+
// no lifecycle runs live. That keeps an activation, and every transition before a
|
|
5126
|
+
// deployment carries the route, from posting a delivery that changes nothing. After a
|
|
5127
|
+
// restore (epoch > 0) no receipt says nothing about what the deployment holds, so it is
|
|
5128
|
+
// delivered. A HOLD is always delivered, and so is a scope the heal asks for its tenant record.
|
|
5129
|
+
if (!askTenant.has(scopeId) &&
|
|
5130
|
+
lifecycleRefusal(lifecycle) === null &&
|
|
5131
|
+
(t.delivered === null ? t.epoch === 0 : t.delivered === lifecycleReceipt(lifecycle))) {
|
|
5132
|
+
continue;
|
|
5133
|
+
}
|
|
5134
|
+
report.attempted += 1;
|
|
5135
|
+
try {
|
|
5136
|
+
let answer = await delegation.deliver({ tenantId, scopeId, lifecycle });
|
|
5137
|
+
// SINGLE AUTHORITY (the invariant this rests on): an environment has exactly one directory,
|
|
5138
|
+
// the singleton `CONTROL_PLANE.idFromName('control-plane')`. A "fresh directory" restore is
|
|
5139
|
+
// that same object restored after its storage was lost, never a second live writer, and
|
|
5140
|
+
// the raise below always re-reads the CURRENT store and delivers what it says. Two control
|
|
5141
|
+
// planes healing one dispatch namespace at once is a split brain that corrupts far more
|
|
5142
|
+
// than lifecycle, and is out of scope here; fencing it in lifecycle alone would guarantee
|
|
5143
|
+
// nothing.
|
|
5144
|
+
//
|
|
5145
|
+
// The scope refused us as OLDER while holding a revision this directory's history did not
|
|
5146
|
+
// write: an epoch ahead of ours, or ours with other counters. Only another history of the
|
|
5147
|
+
// directory delivers those — the one a fresh-directory restore replaced, whose epoch a
|
|
5148
|
+
// clock running behind (or a restore in the same millisecond) cannot outrank. A scope only
|
|
5149
|
+
// ever holds epochs a directory minted, so this directory learns past it: its epoch is
|
|
5150
|
+
// raised above the one held (monotonic, one statement), and the scope is delivered again.
|
|
5151
|
+
const held = answer.lifecycle.revision;
|
|
5152
|
+
const foreign = !answer.applied &&
|
|
5153
|
+
held !== undefined &&
|
|
5154
|
+
(held.epoch > lifecycle.revision.epoch ||
|
|
5155
|
+
(held.epoch === lifecycle.revision.epoch && lifecycleReceipt(answer.lifecycle) !== lifecycleReceipt(lifecycle)));
|
|
5156
|
+
// Bounded, because the deployment answering is the vertical's own code: only a directory
|
|
5157
|
+
// that has been restored can meet another history (epoch > 0), and a legitimate epoch is
|
|
5158
|
+
// a mint time, so one further ahead than the skew is forged or broken. Either way nothing
|
|
5159
|
+
// is raised; the refusal is an ops failure for an operator to look at.
|
|
5160
|
+
if (foreign && (lifecycle.revision.epoch === 0 || held.epoch > Date.now() + LIFECYCLE_EPOCH_SKEW_MS)) {
|
|
5161
|
+
report.failed += 1;
|
|
5162
|
+
await this.admin
|
|
5163
|
+
.recordOpsFailure({
|
|
5164
|
+
actor,
|
|
5165
|
+
operation: 'scope.lifecycle',
|
|
5166
|
+
stage: 'foreign-epoch',
|
|
5167
|
+
tenantId,
|
|
5168
|
+
scopeId,
|
|
5169
|
+
message: `the deployment refused this directory's lifecycle while holding epoch ${held.epoch} ` +
|
|
5170
|
+
`(this directory is at ${lifecycle.revision.epoch}); not raised past it — ` +
|
|
5171
|
+
(lifecycle.revision.epoch === 0
|
|
5172
|
+
? 'this directory has never been restored, so no other history should have written it'
|
|
5173
|
+
: 'it is further ahead of the clock than any directory mints'),
|
|
5174
|
+
})
|
|
5175
|
+
.catch((e) => console.error('substrat: could not record a foreign lifecycle epoch (#1713)', e));
|
|
5176
|
+
continue;
|
|
5177
|
+
}
|
|
5178
|
+
if (foreign) {
|
|
5179
|
+
await this.cp.raiseLifecycleEpoch(held.epoch + 1);
|
|
5180
|
+
const [again] = await this.cp.lifecycleTargets({ scopeId });
|
|
5181
|
+
const next = again ? this.lifecycleOfTarget(again) : null;
|
|
5182
|
+
if (again && next) {
|
|
5183
|
+
t = again;
|
|
5184
|
+
lifecycle = next;
|
|
5185
|
+
answer = await delegation.deliver({ tenantId, scopeId, lifecycle });
|
|
5186
|
+
}
|
|
5187
|
+
}
|
|
5188
|
+
// The receipt is what the scope HOLDS: a newer delivery it kept instead is the truth.
|
|
5189
|
+
await this.cp.recordLifecycleReceipt(scopeId, lifecycleReceipt(answer.lifecycle), new Date().toISOString(), answer.tenantRecorded === true);
|
|
5190
|
+
report.delivered += 1;
|
|
5191
|
+
}
|
|
5192
|
+
catch (err) {
|
|
5193
|
+
report.failed += 1;
|
|
5194
|
+
try {
|
|
5195
|
+
await this.admin.recordOpsFailure({
|
|
5196
|
+
actor,
|
|
5197
|
+
operation: 'scope.lifecycle',
|
|
5198
|
+
stage: 'deliver',
|
|
5199
|
+
tenantId,
|
|
5200
|
+
scopeId,
|
|
5201
|
+
message: `the lifecycle (scope ${t.scope_status}, tenant ${t.tenant_status}) did not reach the ` +
|
|
5202
|
+
`deployment serving this scope; the heal sweep will deliver it again: ${err instanceof Error ? err.message : String(err)}`,
|
|
5203
|
+
});
|
|
5204
|
+
}
|
|
5205
|
+
catch (recordErr) {
|
|
5206
|
+
console.error('substrat: could not record a lifecycle delivery failure (#1713)', recordErr);
|
|
5207
|
+
}
|
|
5208
|
+
}
|
|
5209
|
+
}
|
|
5210
|
+
return report;
|
|
5211
|
+
}
|
|
5212
|
+
/** One directory row as the lifecycle a delivery carries (#1713), or null for a status this code does not know. */
|
|
5213
|
+
lifecycleOfTarget(t) {
|
|
5214
|
+
const parsed = scopeLifecycle.safeParse({
|
|
5215
|
+
scope: t.scope_status,
|
|
5216
|
+
tenant: t.tenant_status,
|
|
5217
|
+
at: new Date().toISOString(),
|
|
5218
|
+
revision: { epoch: t.epoch, scope: t.scope_rev, tenant: t.tenant_rev },
|
|
5219
|
+
});
|
|
5220
|
+
return parsed.success ? parsed.data : null;
|
|
5221
|
+
}
|
|
5222
|
+
/**
|
|
5223
|
+
* The heal sweep (#1713): deliver the lifecycle to every hosted scope whose deployment's
|
|
5224
|
+
* acknowledged state differs from the directory, and again to every scope held now — which is
|
|
5225
|
+
* what puts a hold back on a store a carry or a restore landed without it. Bounded per pass
|
|
5226
|
+
* (`limit`), drifted scopes first. Run from the shared control plane's cron.
|
|
5227
|
+
*/
|
|
5228
|
+
async healLifecycles(actor, opts) {
|
|
5229
|
+
return this.deliverLifecycles(actor, { drift: true, limit: opts?.limit ?? 200 });
|
|
5230
|
+
}
|
|
5231
|
+
/**
|
|
5232
|
+
* Whether a CP-less host holds this scope's work (#1713): `assertLive`'s predicate, answered
|
|
5233
|
+
* rather than thrown, for the entry points that defer instead of refusing — a sweep pass, a
|
|
5234
|
+
* schedule run, an executor drain. Always false on a host with a directory, whose entry
|
|
5235
|
+
* points ask the directory themselves.
|
|
5236
|
+
*/
|
|
5237
|
+
async lifecycleHeld(scopeId) {
|
|
5238
|
+
return this.cpLess && lifecycleRefusal(await this.scopeStub(scopeId).lifecycle()) !== null;
|
|
4483
5239
|
}
|
|
4484
5240
|
/**
|
|
4485
5241
|
* K-3's cross-check on its own: the (tenant, scope) pair must exist and agree before a
|
|
@@ -4489,7 +5245,7 @@ export class CloudflareScopeHost {
|
|
|
4489
5245
|
async assertScope(tenantId, scopeId) {
|
|
4490
5246
|
const rec = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
4491
5247
|
if (!rec)
|
|
4492
|
-
throw
|
|
5248
|
+
throw unknownScopeForTenant(tenantId, scopeId);
|
|
4493
5249
|
}
|
|
4494
5250
|
/**
|
|
4495
5251
|
* The same K-3 cross-check, for a read that then opens the scope's STORAGE — every
|
|
@@ -4504,7 +5260,7 @@ export class CloudflareScopeHost {
|
|
|
4504
5260
|
async scopeRecordForRead(tenantId, scopeId) {
|
|
4505
5261
|
const row = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
4506
5262
|
if (!row)
|
|
4507
|
-
throw
|
|
5263
|
+
throw unknownScopeForTenant(tenantId, scopeId);
|
|
4508
5264
|
if (row.status === 'reaped') {
|
|
4509
5265
|
throw new Error(`scope ${scopeId} is reaped — its storage is gone and cannot be read`);
|
|
4510
5266
|
}
|
|
@@ -4785,7 +5541,7 @@ export class CloudflareScopeHost {
|
|
|
4785
5541
|
* its claims before the OFF's join lands. The joined row then holds the module the operator just
|
|
4786
5542
|
* turned ON, until the next surviving move of that switch releases it (another ON does).
|
|
4787
5543
|
*/
|
|
4788
|
-
async switchInScope(scopeId, moduleId, to, at) {
|
|
5544
|
+
async switchInScope(scopeId, moduleId, to, at, tenantHeld = false) {
|
|
4789
5545
|
let scopeClaims;
|
|
4790
5546
|
let readError;
|
|
4791
5547
|
try {
|
|
@@ -4794,7 +5550,7 @@ export class CloudflareScopeHost {
|
|
|
4794
5550
|
catch (err) {
|
|
4795
5551
|
readError = err;
|
|
4796
5552
|
}
|
|
4797
|
-
const { instance, ...outcome } = await this.scopeStub(scopeId).switchSystemSchedules(moduleId, scopeId, to, at);
|
|
5553
|
+
const { instance, ...outcome } = await this.scopeStub(scopeId).switchSystemSchedules(moduleId, scopeId, to, at, tenantHeld);
|
|
4798
5554
|
if (!scopeClaims)
|
|
4799
5555
|
throw readError;
|
|
4800
5556
|
const now = Date.now();
|
|
@@ -4997,16 +5753,16 @@ export class CloudflareScopeHost {
|
|
|
4997
5753
|
* suspended or archiving scope could still be addressed and handed a 101 — a
|
|
4998
5754
|
* scope that refuses every ordinary read while quietly holding an open socket.
|
|
4999
5755
|
*
|
|
5000
|
-
*
|
|
5001
|
-
*
|
|
5002
|
-
*
|
|
5756
|
+
* On a CP-less vertical, `validateScopeAccess` is a no-op on the null control
|
|
5757
|
+
* plane, and `assertLive` reads the lifecycle the platform delivered to the
|
|
5758
|
+
* scope's own storage instead (#1713).
|
|
5003
5759
|
*
|
|
5004
5760
|
* `migrateAndRecord`, not the DO's own `ensureMigrations`: the DO migrates
|
|
5005
5761
|
* itself when the socket opens either way, but only this reports the applied
|
|
5006
5762
|
* count to the directory — so a scope whose first contact after a deploy is a
|
|
5007
5763
|
* subscription does not go dark in the migration fleet view.
|
|
5008
5764
|
*/
|
|
5009
|
-
await this.
|
|
5765
|
+
await this.assertLive(tenantId, scopeId);
|
|
5010
5766
|
await this.migrateAndRecord(scopeId);
|
|
5011
5767
|
// Asserted, not carried through from the client: the principal is the
|
|
5012
5768
|
// vertical's own resolution of its session, and the tenant and scope are the
|
|
@@ -5094,7 +5850,7 @@ export class CloudflareScopeHost {
|
|
|
5094
5850
|
// good. Read first, so a pair the directory does not hold writes nothing.
|
|
5095
5851
|
const scope = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
5096
5852
|
if (!scope)
|
|
5097
|
-
throw
|
|
5853
|
+
throw unknownScopeForTenant(tenantId, scopeId);
|
|
5098
5854
|
const { roles, tuples, entitlements, identities } = await this.tenantProjection(tenantId);
|
|
5099
5855
|
const connectionKeys = await this.connectionKeyRows(tenantId, scope.vertical);
|
|
5100
5856
|
await this.scopeStub(scopeId).applyProjection(tenantId, roles, tuples, entitlements, undefined, identities, connectionKeys);
|
|
@@ -5272,7 +6028,7 @@ export class CloudflareScopeHost {
|
|
|
5272
6028
|
// #1742: the recorded-off modules, switched off after the seat in the same unit, on the
|
|
5273
6029
|
// scope this call provisions — the one the seat's tuples name.
|
|
5274
6030
|
input.switchedOff
|
|
5275
|
-
? { scopeId: input.scopeId, moduleIds: input.switchedOff, at: new Date().toISOString() }
|
|
6031
|
+
? { scopeId: input.scopeId, moduleIds: input.switchedOff, at: new Date().toISOString(), tenantHeld: input.tenantHeld }
|
|
5276
6032
|
: undefined, services?.map((id) => `principal:${id}`));
|
|
5277
6033
|
return input.switchedOff ? { switchedOff } : {};
|
|
5278
6034
|
}
|
|
@@ -5369,14 +6125,19 @@ export class CloudflareScopeHost {
|
|
|
5369
6125
|
// side before the call; what runs HERE is the half only this deployment can
|
|
5370
6126
|
// enforce: the scope's own permission check against its delivered
|
|
5371
6127
|
// `connection:<id>` tuple, in the scope's own DO. Fail closed — no grant, no
|
|
5372
|
-
// effect — exactly as for any other caller.
|
|
6128
|
+
// effect — exactly as for any other caller. And the lifecycle delivered to the
|
|
6129
|
+
// scope (#1713): a held scope's invoke and bytes doors refuse, as every other door does.
|
|
5373
6130
|
/** Invoke ONE operation in this deployment as a CONNECTION (#574). */
|
|
5374
6131
|
async connectorInvokeLocal(connectionId, tenantId, scopeId, operation, input) {
|
|
6132
|
+
await this.assertLive(tenantId, scopeId); // #1713: a held scope runs no connector work either
|
|
5375
6133
|
await this.migrateAndRecord(scopeId);
|
|
5376
6134
|
return this.buildStub(tenantId, scopeId, undefined, connectionId).invoke(operation, input);
|
|
5377
6135
|
}
|
|
5378
6136
|
/** Land provider bytes in this deployment as a CONNECTION — the bytes leg (#574). */
|
|
5379
6137
|
async connectorAttachmentUploadLocal(connectionId, tenantId, scopeId, upload) {
|
|
6138
|
+
// The lifecycle gate every attachments door takes (#1713), ahead of the bucket: a held
|
|
6139
|
+
// scope's provider bytes wait for the delivery's retry rather than land (#1995).
|
|
6140
|
+
await this.assertLive(tenantId, scopeId);
|
|
5380
6141
|
await this.migrateAndRecord(scopeId);
|
|
5381
6142
|
const store = await this.resolveAttachmentStore(tenantId);
|
|
5382
6143
|
return this.buildAttachmentSurface({ connectionId }, tenantId, scopeId, store).upload(upload);
|
|
@@ -5389,6 +6150,8 @@ export class CloudflareScopeHost {
|
|
|
5389
6150
|
* so a caller falls back rather than failing a dispatch over a missing file.
|
|
5390
6151
|
*/
|
|
5391
6152
|
async connectorAttachmentOpenLocal(connectionId, tenantId, scopeId, attachmentId, eventId) {
|
|
6153
|
+
// Same gate as the upload leg: a held scope's bytes are not read out of its bucket.
|
|
6154
|
+
await this.assertLive(tenantId, scopeId);
|
|
5392
6155
|
await this.migrateAndRecord(scopeId);
|
|
5393
6156
|
const store = await this.resolveAttachmentStore(tenantId);
|
|
5394
6157
|
return this.buildAttachmentSurface({ connectionId }, tenantId, scopeId, store,
|
|
@@ -5414,11 +6177,11 @@ export class CloudflareScopeHost {
|
|
|
5414
6177
|
* admin log and writes the row once this returns. `held: false` is an answer, not a
|
|
5415
6178
|
* throw — see `systemSwitchOutcome`.
|
|
5416
6179
|
*/
|
|
5417
|
-
async systemSwitchLocal(scopeId, moduleId, to) {
|
|
6180
|
+
async systemSwitchLocal(scopeId, moduleId, to, opts) {
|
|
5418
6181
|
// Parsed on the way out: this is the wire answer the platform reads, and the DO's
|
|
5419
6182
|
// plain strings become the published shape here rather than on trust.
|
|
5420
6183
|
// #1819: `switchInScope` releases a rewind's hold on the module once the switch lands.
|
|
5421
|
-
return systemSwitchOutcome.parse(await this.switchInScope(scopeId, moduleId, to, new Date().toISOString()));
|
|
6184
|
+
return systemSwitchOutcome.parse(await this.switchInScope(scopeId, moduleId, to, new Date().toISOString(), opts?.tenantHeld ?? false));
|
|
5422
6185
|
}
|
|
5423
6186
|
/**
|
|
5424
6187
|
* The far end of the status read for a scope served HERE (#1674): `/internal/system-grants`
|
|
@@ -5519,7 +6282,7 @@ export class CloudflareScopeHost {
|
|
|
5519
6282
|
}
|
|
5520
6283
|
const rec = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
5521
6284
|
if (!rec)
|
|
5522
|
-
throw
|
|
6285
|
+
throw unknownScopeForTenant(tenantId, scopeId);
|
|
5523
6286
|
const source = await importCursorSourceOf((t, v) => this.admin.resolveVerticalInstance(t, v), { tenantId, scopeId, vertical: rec.vertical }, move.from);
|
|
5524
6287
|
// Where the write lands is the peer switch's rule: a scope bound to a vertical is served by
|
|
5525
6288
|
// that vertical's deployment, and its watermark lives there. Without a delegation that
|