@substrat-run/adapter-cloudflare 0.134.0 → 0.136.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/connector-calls.d.ts +11 -0
- package/dist/connector-calls.d.ts.map +1 -0
- package/dist/connector-calls.js +10 -0
- package/dist/connector-calls.js.map +1 -0
- package/dist/control-plane-do.d.ts +76 -0
- package/dist/control-plane-do.d.ts.map +1 -1
- package/dist/control-plane-do.js +172 -49
- package/dist/control-plane-do.js.map +1 -1
- package/dist/host.d.ts +202 -12
- package/dist/host.d.ts.map +1 -1
- package/dist/host.js +543 -50
- package/dist/host.js.map +1 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/live-reads.d.ts +5 -4
- package/dist/live-reads.d.ts.map +1 -1
- package/dist/live-reads.js +5 -4
- package/dist/live-reads.js.map +1 -1
- package/dist/r2.d.ts +2 -1
- package/dist/r2.d.ts.map +1 -1
- package/dist/r2.js.map +1 -1
- package/dist/route-resolver.d.ts +7 -0
- package/dist/route-resolver.d.ts.map +1 -1
- package/dist/route-resolver.js +4 -0
- package/dist/route-resolver.js.map +1 -1
- package/dist/scope-do.d.ts.map +1 -1
- package/dist/scope-do.js +653 -101
- package/dist/scope-do.js.map +1 -1
- package/dist/scope-sweeper-do.d.ts +32 -6
- package/dist/scope-sweeper-do.d.ts.map +1 -1
- package/dist/scope-sweeper-do.js +35 -5
- package/dist/scope-sweeper-do.js.map +1 -1
- package/package.json +10 -9
package/dist/host.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
|
+
import { env as ambientEnv } from 'cloudflare:workers';
|
|
1
2
|
import { isRewindRefusal, REWIND_REFUSED } from './rewind-refusal.js';
|
|
2
|
-
import { delegatedReadParams, fromWireFailure, exportReadInput, exportedBatch, importBatch, importResult, importState, importsOfManifestJson, exportsOfManifestJson, importCursorMove, importCursorMoveAt, importCursorMoved, accessLogEntry, adminLogEntry, opsFailureEntry, opsFailureFingerprint, issueEntry, sweepRunEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunsPayload, modelUsageEntry, attachmentRecord, createTenantInput, identityLink, identityPool, createOrgInput, promotionAcknowledgement, bindAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, registerVerticalInput, vertical as verticalSchema, verticalServingState, verticalChannel, verticalVersion, connection, capabilityExchange, capabilityGrant, principalId, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, systemSwitch, systemSwitchOutcome, systemScheduleEntry, systemSwitchRecord, entitlementGrant, entitlementGrantInput, instant, meterReading, subjectRef, createConnectionInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, org as orgSchema, orgMembership, resolvedIdentity, identityMembership, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, delegatedReadRecord, ownerTransferAudit, subjectShredReceipt, connectorDispatchKind, callsOfManifestJson, outboundOfManifestJson, substratError, redrainEventsInput, coverage, peerCoverage, peerGrantsEntry, peerSwitch, peerSwitchOutcome, verticalCaller, verticalResolution, verticalSlug, entityObjectRef, } from '@substrat-run/contracts';
|
|
3
|
+
import { delegatedReadParams, fromWireFailure, exportReadInput, exportedBatch, importBatch, importResult, importState, importsOfManifestJson, exportsOfManifestJson, importCursorMove, importCursorMoveAt, importCursorMoved, accessLogEntry, adminLogEntry, opsFailureEntry, opsFailureFingerprint, issueEntry, sweepRunEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunsPayload, modelUsageEntry, attachmentRecord, createTenantInput, identityLink, identityPool, createOrgInput, promotionAcknowledgement, bindAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, registerVerticalInput, vertical as verticalSchema, verticalServingState, verticalChannel, verticalVersion, connection, capabilityExchange, capabilityGrant, principalId, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, systemSwitch, systemSwitchOutcome, systemScheduleEntry, systemSwitchRecord, entitlementGrant, entitlementGrantInput, capabilityFilter, instant, meterReading, subjectRef, createConnectionInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, org as orgSchema, orgMembership, resolvedIdentity, identityMembership, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, delegatedReadRecord, ownerTransferAudit, copyMarkAudit, subjectShredReceipt, connectorDispatchKind, callsOfManifestJson, outboundOfManifestJson, substratError, redrainEventsInput, coverage, peerCoverage, peerGrantsEntry, peerSwitch, peerSwitchOutcome, verticalCaller, verticalResolution, verticalSlug, entityObjectRef, ATTACHMENT_BLOB_BINDING, blobStoreBindingName, lifecycleDelivery, scopeLifecycle, } from '@substrat-run/contracts';
|
|
3
4
|
import { normalizeHostname, toRouteTarget } from './route-resolver.js';
|
|
4
|
-
import { attachmentBlobKey, attachmentSha256, entitlementDenial, foldMeterReading, parseValidationRecords, resolveScopeRecord, ulid, capabilityTokenHash, checkBecomeInput, plausibleSessionToken, foldModelUsage, backoffAt, resolveRetryPolicy, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, openSealed, ImpersonationRefused, assertSessionUsable, impersonationRowValues, mapImpersonationRow, newImpersonationSession, jobRunOf, runDueJobRuns, startJobRun, attachmentTextJob, assertAttachmentExtractors, assertJobRegistrable, isAttachmentTextRun, searchLimit, searchMatchExpression, globalFetch, assertRedrainWindow, platformRequestOf, undrainedEventsOf, inUnitMovesToAudit, staleCarryRevertRow, staleCarryReverts, withRecorded, systemSwitchedOffMessage, tenantSystemSwitchedOffMessage, CrossVerticalRegistry, importCursorSourceOf, exportBreaksOf, exportBreakRefusal, bindExportBreaksOf, bindExportBreakRefusal, collectPeers, peerSeats, connectorCallRecord, noopConnectorCallRecorder, recordConnectorCall, settleConnectionUse, splitManifestMigrations, unknownRoleError, assertRowLimit, assertRowOffset, } from '@substrat-run/kernel';
|
|
5
|
+
import { attachmentBlobKey, attachmentSha256, entitlementDenial, requiredEntitlementFor, foldMeterReading, parseValidationRecords, resolveScopeRecord, ulid, KEPT_COPY_REFUSAL, capabilityTokenHash, checkBecomeInput, plausibleSessionToken, foldModelUsage, backoffAt, resolveRetryPolicy, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, openSealed, ImpersonationRefused, assertSessionUsable, impersonationRowValues, mapImpersonationRow, newImpersonationSession, jobRunOf, runDueJobRuns, startJobRun, attachmentTextJob, assertAttachmentExtractors, assertJobRegistrable, isAttachmentTextRun, searchLimit, searchMatchExpression, globalFetch, assertRedrainWindow, platformRequestOf, undrainedEventsOf, inUnitMovesToAudit, staleCarryRevertRow, staleCarryReverts, withRecorded, systemSwitchedOffMessage, tenantSystemSwitchedOffMessage, CrossVerticalRegistry, importCursorSourceOf, exportBreaksOf, exportBreakRefusal, bindExportBreaksOf, bindExportBreakRefusal, collectPeers, peerSeats, connectorCallRecord, noopConnectorCallRecorder, recordConnectorCall, settleConnectionUse, splitManifestMigrations, unknownRoleError, assertRowLimit, assertRowOffset, INERT_SCOPE_REASON, lifecycleReceipt, lifecycleRefusal, isPrimaryScope, isPrimaryScopeRow, asyncInvocationId, asyncLinePass, } from '@substrat-run/kernel';
|
|
5
6
|
import { attributedHost } from '@substrat-run/kernel';
|
|
6
7
|
import { isOrangeToOrange, isUpgradeRequest, LIVE_MODE_HEADER, LIVE_PRINCIPAL_HEADER, LIVE_SCOPE_HEADER, LIVE_SUBSCRIBE_PATH, LIVE_TENANT_HEADER, } from './live-reads.js';
|
|
7
8
|
import { tenantStoreDatabaseName } from './d1.js';
|
|
@@ -30,12 +31,29 @@ function unwrapCapabilityReply(reply) {
|
|
|
30
31
|
throw fromWireFailure(reply.failure);
|
|
31
32
|
return reply.value;
|
|
32
33
|
}
|
|
34
|
+
/**
|
|
35
|
+
* How far ahead of the clock a held lifecycle epoch may be and still be learned past (#1713). An
|
|
36
|
+
* epoch is the time a directory restore minted it, so a legitimate one is behind its minter's
|
|
37
|
+
* clock; the skew allows for clocks that disagree, and bounds what a forged answer can move.
|
|
38
|
+
*/
|
|
39
|
+
export const LIFECYCLE_EPOCH_SKEW_MS = 24 * 60 * 60 * 1000;
|
|
40
|
+
/**
|
|
41
|
+
* Refuse to mark a scope a copy unless the request classifies it non-primary (#2005). The
|
|
42
|
+
* classification is the platform directory's; the vertical applies the same `isPrimaryScope` to
|
|
43
|
+
* it, so a primary can be marked only by a request that misstates its lineage, never by default.
|
|
44
|
+
*/
|
|
45
|
+
function assertCopyLineage(lineage) {
|
|
46
|
+
if (isPrimaryScope(lineage)) {
|
|
47
|
+
throw substratError('conflict', 'mark-copy refused: the directory classifies this scope as primary, not as a copy');
|
|
48
|
+
}
|
|
49
|
+
}
|
|
33
50
|
/**
|
|
34
51
|
* A control-plane stand-in for a CP-less vertical (scope-local-permissions.md Phase 3).
|
|
35
52
|
* The hot path a served scope actually touches becomes trust-the-upstream:
|
|
36
|
-
* - `scopeAccessRefusal` / `setMigrationState` → no-op: the router
|
|
37
|
-
*
|
|
38
|
-
*
|
|
53
|
+
* - `scopeAccessRefusal` / `setMigrationState` → no-op: the router gates the scope's
|
|
54
|
+
* lifecycle + tenancy from the shared directory, and the vertical trusts the asserted
|
|
55
|
+
* node rather than re-reading a directory it does not have. Its own half of the
|
|
56
|
+
* lifecycle gate reads the scope's storage instead (`assertLive`, #1713).
|
|
39
57
|
* - `tenantHoldsEntitlement` → true: the SKU was enforced on the shared control plane
|
|
40
58
|
* at provision (before `provisionInstance`), so a scope that EXISTS here was granted
|
|
41
59
|
* it upstream — a single-vertical deployment holds its own entitlements by construction.
|
|
@@ -106,6 +124,14 @@ export const SWITCH_HOLD_EXTRA_WAITS = 3;
|
|
|
106
124
|
* clock, against the releasing host's clock: minutes, not seconds.
|
|
107
125
|
*/
|
|
108
126
|
export const SWITCH_HOLD_PENDING_MAX_MS = 5 * 60_000;
|
|
127
|
+
/**
|
|
128
|
+
* The attachment bucket a host resolves when it was handed no resolver (#1995): the
|
|
129
|
+
* platform's per-tenant binding, read off this script's own env, named by the one shared
|
|
130
|
+
* encoding (`blobStoreBindingName`) for the tenant the caller passed in.
|
|
131
|
+
*/
|
|
132
|
+
function ambientAttachmentBucket(tenantId) {
|
|
133
|
+
return ambientEnv[blobStoreBindingName(ATTACHMENT_BLOB_BINDING, tenantId)] ?? null;
|
|
134
|
+
}
|
|
109
135
|
export class CloudflareScopeHost {
|
|
110
136
|
admin;
|
|
111
137
|
scopeNs;
|
|
@@ -145,7 +171,7 @@ export class CloudflareScopeHost {
|
|
|
145
171
|
tenantStores;
|
|
146
172
|
/** The live R2 client for per-tenant blob stores (#473); undefined ⇒ refuse loudly. */
|
|
147
173
|
blobStores;
|
|
148
|
-
/** Worker-side attachment-bucket resolver (#473);
|
|
174
|
+
/** Worker-side attachment-bucket resolver (#473); the ambient per-tenant binding unless overridden (#1995). */
|
|
149
175
|
attachmentBuckets;
|
|
150
176
|
/** The parsers attachment text is extracted with (K-43); the host's own, never imported here. */
|
|
151
177
|
attachmentExtractors;
|
|
@@ -171,6 +197,8 @@ export class CloudflareScopeHost {
|
|
|
171
197
|
*/
|
|
172
198
|
onBehalfOf = null;
|
|
173
199
|
withdrawn = new Map(); // operation → module
|
|
200
|
+
/** Operation → who binds it (module, SKU flag, its declared schedules) — the §4.3 gate's
|
|
201
|
+
* input, resolved per invoke by the kernel's `requiredEntitlementFor` (#1654). */
|
|
174
202
|
operationEntitlement = new Map();
|
|
175
203
|
/** #574: remote connector write-back for scopes served by another deployment. */
|
|
176
204
|
connectorDelegation;
|
|
@@ -178,6 +206,8 @@ export class CloudflareScopeHost {
|
|
|
178
206
|
eventDrainDelegation;
|
|
179
207
|
/** #1666: the schedule kill switch's reach into the deployment serving a scope. */
|
|
180
208
|
systemSwitchDelegation;
|
|
209
|
+
/** #1713: the lifecycle's reach into the deployment serving a scope. */
|
|
210
|
+
lifecycleDelegation;
|
|
181
211
|
/** #1706: the peer kill switch's reach into the deployment serving a scope. */
|
|
182
212
|
peerSwitchDelegation;
|
|
183
213
|
/** #1705 PR 3: the replay lever's reach into the deployment serving a consumer scope. */
|
|
@@ -209,7 +239,7 @@ export class CloudflareScopeHost {
|
|
|
209
239
|
this.secretBox = options.secretBox ?? unconfiguredSecretBox;
|
|
210
240
|
this.tenantStores = options.tenantStores;
|
|
211
241
|
this.blobStores = options.blobStores;
|
|
212
|
-
this.attachmentBuckets = options.attachmentBuckets;
|
|
242
|
+
this.attachmentBuckets = options.attachmentBuckets ?? ambientAttachmentBucket;
|
|
213
243
|
assertAttachmentExtractors(options.attachmentExtractors ?? []);
|
|
214
244
|
this.attachmentExtractors = options.attachmentExtractors ?? [];
|
|
215
245
|
this.fetchImpl = options.fetch ?? globalFetch;
|
|
@@ -223,6 +253,7 @@ export class CloudflareScopeHost {
|
|
|
223
253
|
this.connectorDelegation = options.connectorDelegation;
|
|
224
254
|
this.eventDrainDelegation = options.eventDrainDelegation;
|
|
225
255
|
this.systemSwitchDelegation = options.systemSwitchDelegation;
|
|
256
|
+
this.lifecycleDelegation = options.lifecycleDelegation;
|
|
226
257
|
this.peerSwitchDelegation = options.peerSwitchDelegation;
|
|
227
258
|
this.importCursorDelegation = options.importCursorDelegation;
|
|
228
259
|
this.admin = this.buildAdmin();
|
|
@@ -351,21 +382,78 @@ export class CloudflareScopeHost {
|
|
|
351
382
|
};
|
|
352
383
|
if (this.executors.size === 0)
|
|
353
384
|
return report;
|
|
385
|
+
// #1901: one line per attempt, capped per pass. Written on the coordinator, where the
|
|
386
|
+
// attempt runs — the DO only journals it.
|
|
387
|
+
const lines = asyncLinePass();
|
|
388
|
+
try {
|
|
389
|
+
await this.drainExecutorsPass(tenantId, scopeId, invocationId, report, lines);
|
|
390
|
+
}
|
|
391
|
+
finally {
|
|
392
|
+
lines.end();
|
|
393
|
+
}
|
|
394
|
+
return report;
|
|
395
|
+
}
|
|
396
|
+
async drainExecutorsPass(tenantId, scopeId, invocationId, report, lines) {
|
|
354
397
|
const stub = this.scopeStub(scopeId);
|
|
355
|
-
|
|
398
|
+
// #2005: a non-primary scope (a fork, a snapshot, a preview of either kind) causes no
|
|
399
|
+
// outbound effects, so its deliveries are journaled terminal with the reason and no
|
|
400
|
+
// handler runs. Asked of the directory where there is one. A CP-less host has none, so it
|
|
401
|
+
// asks the scope's own storage whether it is classified a copy (`_substrat_copy_origin`'s
|
|
402
|
+
// `is_copy`, #2009, set on the directory's word) — a preview and a snapshot reach a hosted
|
|
403
|
+
// vertical as restores the platform marks.
|
|
404
|
+
// Asked on the first due event, once per pass: most passes have none, and the directory
|
|
405
|
+
// is one global object.
|
|
406
|
+
let inert;
|
|
407
|
+
const isInert = () => (inert ??= this.cpLess
|
|
408
|
+
? stub.isCopy()
|
|
409
|
+
: this.cp.getScopeRecord(tenantId, scopeId).then((row) => !isPrimaryScopeRow(row)));
|
|
410
|
+
// #1713: a CP-less scope its lifecycle holds attempts nothing and journals nothing, so every
|
|
411
|
+
// due delivery stays due for the first pass after it is live again. Asked on the first due
|
|
412
|
+
// event, like `isInert`. Every caller passed `assertLive` already, so this is for the scope
|
|
413
|
+
// suspended between that gate and the drain: a request in flight, whose write commits while
|
|
414
|
+
// its effect waits. A host with a directory answers false without a read.
|
|
415
|
+
let held;
|
|
416
|
+
const isHeld = () => (held ??= this.lifecycleHeld(scopeId));
|
|
417
|
+
drain: for (const [id, executor] of this.executors) {
|
|
356
418
|
const deliveryId = `executor:${id}`;
|
|
357
419
|
const { events, undecodable } = await stub.pendingExecutorDeliveries(deliveryId, executor.eventType);
|
|
358
420
|
// #1636: an event the DO could not decode is dead-lettered for this executor at once,
|
|
359
421
|
// and its handler never sees it. Terminal on the FIRST failure, unlike a handler's:
|
|
360
422
|
// the decode is pure, so a retry cannot succeed. The rows behind it are delivered
|
|
361
423
|
// below — the decode used to throw the whole list, on every pass.
|
|
424
|
+
if ((undecodable.length > 0 || events.length > 0) && (await isHeld())) {
|
|
425
|
+
report.lifecycleHeld = true;
|
|
426
|
+
break drain;
|
|
427
|
+
}
|
|
428
|
+
// #1901: an attempt's line. The attempt number is the one the journal is about to record.
|
|
429
|
+
const unitOf = (eventId, attempt) => ({
|
|
430
|
+
kind: 'consumer',
|
|
431
|
+
tenantId,
|
|
432
|
+
scopeId,
|
|
433
|
+
invocationId: asyncInvocationId(invocationId),
|
|
434
|
+
operation: deliveryId,
|
|
435
|
+
eventType: executor.eventType,
|
|
436
|
+
eventId,
|
|
437
|
+
attempt,
|
|
438
|
+
startedAt: Date.now(),
|
|
439
|
+
});
|
|
362
440
|
for (const bad of undecodable) {
|
|
363
441
|
report.attempted += 1;
|
|
364
|
-
await stub.recordExecutorAttempt(bad.eventId, deliveryId, bad.error, null, invocationId);
|
|
442
|
+
const attempt = await stub.recordExecutorAttempt(bad.eventId, deliveryId, bad.error, null, invocationId);
|
|
365
443
|
report.deadLettered += 1;
|
|
444
|
+
lines.write({ ...unitOf(bad.eventId, attempt), outcome: 'dead-lettered' });
|
|
366
445
|
}
|
|
367
446
|
for (const event of events) {
|
|
368
447
|
report.attempted += 1;
|
|
448
|
+
const startedAt = Date.now();
|
|
449
|
+
if (await isInert()) {
|
|
450
|
+
// No next attempt, like an undecodable row: a scope does not become primary.
|
|
451
|
+
const attempt = await stub.recordExecutorAttempt(event.id, deliveryId, INERT_SCOPE_REASON, null, invocationId);
|
|
452
|
+
report.inert = (report.inert ?? 0) + 1;
|
|
453
|
+
// Its own outcome, never `delivered`: no handler ran (#2005).
|
|
454
|
+
lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'inert' });
|
|
455
|
+
continue;
|
|
456
|
+
}
|
|
369
457
|
this.causedBy = event.id;
|
|
370
458
|
try {
|
|
371
459
|
if (executor.kind === 'connector' && this.cpLess) {
|
|
@@ -379,16 +467,20 @@ export class CloudflareScopeHost {
|
|
|
379
467
|
// the at-least-once residue, as it already must in-process.
|
|
380
468
|
await stub.routeExecutorEventToPlatform(event.id, deliveryId, connectorDispatchKind(executor.provider), JSON.stringify({ executorId: id, event }), JSON.stringify({ system: 'connector-dispatch' }), invocationId);
|
|
381
469
|
report.routedToPlatform += 1;
|
|
470
|
+
// Handed on, not run here: the platform's drain owns its attempts from now on.
|
|
471
|
+
lines.write({ ...unitOf(event.id, 1), startedAt, outcome: 'routed' });
|
|
382
472
|
}
|
|
383
473
|
else if (executor.kind === 'connector') {
|
|
384
474
|
await executor.handler(await this.connectorContext(tenantId, scopeId, executor.timeoutMs, event.id), event);
|
|
385
|
-
await stub.recordExecutorAttempt(event.id, deliveryId, null, null, invocationId);
|
|
475
|
+
const attempt = await stub.recordExecutorAttempt(event.id, deliveryId, null, null, invocationId);
|
|
386
476
|
report.delivered += 1;
|
|
477
|
+
lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'delivered' });
|
|
387
478
|
}
|
|
388
479
|
else {
|
|
389
480
|
await executor.handler(this.admin, event);
|
|
390
|
-
await stub.recordExecutorAttempt(event.id, deliveryId, null, null, invocationId);
|
|
481
|
+
const attempt = await stub.recordExecutorAttempt(event.id, deliveryId, null, null, invocationId);
|
|
391
482
|
report.delivered += 1;
|
|
483
|
+
lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'delivered' });
|
|
392
484
|
}
|
|
393
485
|
}
|
|
394
486
|
catch (err) {
|
|
@@ -403,18 +495,23 @@ export class CloudflareScopeHost {
|
|
|
403
495
|
report.deadLettered += 1;
|
|
404
496
|
else
|
|
405
497
|
report.retrying += 1;
|
|
498
|
+
lines.write({
|
|
499
|
+
...unitOf(event.id, attempts),
|
|
500
|
+
startedAt,
|
|
501
|
+
outcome: exhausted ? 'dead-lettered' : 'retrying',
|
|
502
|
+
error: err,
|
|
503
|
+
});
|
|
406
504
|
}
|
|
407
505
|
finally {
|
|
408
506
|
this.causedBy = null;
|
|
409
507
|
}
|
|
410
508
|
}
|
|
411
509
|
}
|
|
412
|
-
return report;
|
|
413
510
|
}
|
|
414
511
|
async drainDue(tenantId, scopeId) {
|
|
415
512
|
// Same lifecycle gate `getScope` applies (K-3): a suspended or archived scope
|
|
416
513
|
// does not get its effects driven either.
|
|
417
|
-
await this.
|
|
514
|
+
await this.assertLive(tenantId, scopeId);
|
|
418
515
|
await this.migrateAndRecord(scopeId);
|
|
419
516
|
// #1525: null, and honestly so — a sweep is not a call. An attempt this pass makes
|
|
420
517
|
// records no invocation, which is what distinguishes it from the first attempt the
|
|
@@ -472,14 +569,14 @@ export class CloudflareScopeHost {
|
|
|
472
569
|
this.attachmentExtractors);
|
|
473
570
|
}
|
|
474
571
|
async startJobRun(tenantId, scopeId, input) {
|
|
475
|
-
await this.
|
|
572
|
+
await this.assertLive(tenantId, scopeId);
|
|
476
573
|
await this.migrateAndRecord(scopeId);
|
|
477
574
|
return jobRunOf(await startJobRun(this.jobStore(scopeId), input, ulid, () => new Date().toISOString()));
|
|
478
575
|
}
|
|
479
576
|
async runDueJobs(tenantId, scopeId, options) {
|
|
480
577
|
// Same lifecycle gate as `drainDue`: a suspended scope's runs wait rather than
|
|
481
578
|
// advance, and an archived one's never move again.
|
|
482
|
-
await this.
|
|
579
|
+
await this.assertLive(tenantId, scopeId);
|
|
483
580
|
await this.migrateAndRecord(scopeId);
|
|
484
581
|
// #1575: the kernel's extraction job is this host's own, bound to this scope — no
|
|
485
582
|
// deployment registers it, and none can shadow it.
|
|
@@ -504,7 +601,7 @@ export class CloudflareScopeHost {
|
|
|
504
601
|
// same context build as the in-process path, so the handler cannot tell which host
|
|
505
602
|
// ran it. On a CP-less host `connectorContext` throws from the null control plane:
|
|
506
603
|
// fail closed, exactly the hole routing exists to avoid.
|
|
507
|
-
await this.
|
|
604
|
+
await this.assertLive(tenantId, scopeId);
|
|
508
605
|
this.causedBy = event.id;
|
|
509
606
|
try {
|
|
510
607
|
await handler(await this.connectorContext(tenantId, scopeId, options?.timeoutMs ?? 30_000, event.id), event);
|
|
@@ -631,6 +728,15 @@ export class CloudflareScopeHost {
|
|
|
631
728
|
async summarizeDenialsLocal(scopeId, filter) {
|
|
632
729
|
return this.scopeStub(scopeId).summarizeDenials(filter);
|
|
633
730
|
}
|
|
731
|
+
/**
|
|
732
|
+
* The operator's capability read's CP-less path (#1686) — the denial log's trust line: the
|
|
733
|
+
* directory rows live in the scope's own DO, in the vertical's deployment, reachable only
|
|
734
|
+
* through its platform-gated `/internal/capabilities`; the K-3 check and the K-24 entry are
|
|
735
|
+
* the control plane's, made before it calls. Records only — the DO's query selects no hash.
|
|
736
|
+
*/
|
|
737
|
+
async listCapabilitiesLocal(scopeId, filter) {
|
|
738
|
+
return this.scopeStub(scopeId).listCapabilities(filter);
|
|
739
|
+
}
|
|
634
740
|
/**
|
|
635
741
|
* Copy one scope's data into a fresh scope DO, entirely within THIS deployment —
|
|
636
742
|
* the data half of an orchestrated snapshot (preview-and-snapshots.md §9). Like the
|
|
@@ -642,7 +748,9 @@ export class CloudflareScopeHost {
|
|
|
642
748
|
*/
|
|
643
749
|
async snapshotScopeLocal(sourceScopeId, destScopeId) {
|
|
644
750
|
const tables = await this.scopeStub(sourceScopeId).exportDump();
|
|
645
|
-
|
|
751
|
+
// #2009: a snapshot is a fork by construction (the control plane's row names `forkedFrom`, so
|
|
752
|
+
// the directory never calls it primary), and a load classifies nothing by itself: marked here.
|
|
753
|
+
await this.scopeStub(destScopeId).importDump(tables, destScopeId, { sourceScopeId, exact: true, markCopy: true });
|
|
646
754
|
return { tables: tables.length };
|
|
647
755
|
}
|
|
648
756
|
/**
|
|
@@ -655,14 +763,131 @@ export class CloudflareScopeHost {
|
|
|
655
763
|
/** #1742: the directory's recorded-off modules, switched off on THIS scope in the replay's
|
|
656
764
|
* own event — a dump from before a switch was pulled brings its grants back live. #1869:
|
|
657
765
|
* `sourceScopeId`, the scope the dump was captured from, narrows the grant re-point to it;
|
|
658
|
-
* `exact` says the platform exported the dump itself, so the re-point never falls back.
|
|
766
|
+
* `exact` says the platform exported the dump itself, so the re-point never falls back.
|
|
767
|
+
* #1722: `loadStamp`, the stamp a carry leaves on the copy it lands, for a fenced wipe later,
|
|
768
|
+
* and `expect`, the marker the carry read here: the load is refused if the store moved since.
|
|
769
|
+
* #2005: `markCopy` — the directory's classification of the scope, sent when it is not
|
|
770
|
+
* primary; the restore then marks it a copy. Refused if it classifies a primary. */
|
|
659
771
|
opts) {
|
|
660
|
-
|
|
772
|
+
if (opts?.markCopy)
|
|
773
|
+
assertCopyLineage(opts.markCopy);
|
|
774
|
+
const load = {
|
|
661
775
|
switchOff: opts?.switchedOff ? { moduleIds: opts.switchedOff, at: new Date().toISOString() } : undefined,
|
|
662
776
|
sourceScopeId: opts?.sourceScopeId,
|
|
663
777
|
exact: opts?.exact,
|
|
778
|
+
loadStamp: opts?.loadStamp,
|
|
779
|
+
markCopy: opts?.markCopy !== undefined,
|
|
780
|
+
};
|
|
781
|
+
// Both refusals come back as values (a throw over the RPC carries only its message) and are
|
|
782
|
+
// thrown here, in the vertical's own isolate, so its route answers 409 or 412, not a fault.
|
|
783
|
+
const out = await this.scopeStub(scopeId).importDumpChecked(tables, scopeId, {
|
|
784
|
+
...load,
|
|
785
|
+
...(opts?.expect ? { expect: opts.expect } : {}),
|
|
664
786
|
});
|
|
665
|
-
|
|
787
|
+
if (out.refused === false) {
|
|
788
|
+
return { tables: tables.length, ...(opts?.switchedOff ? { switchedOff: out.switchedOff } : {}) };
|
|
789
|
+
}
|
|
790
|
+
if (out.refused === 'kept')
|
|
791
|
+
throw substratError('conflict', KEPT_COPY_REFUSAL);
|
|
792
|
+
// A retry of a load that already committed (its answer was lost on the way back) is
|
|
793
|
+
// refused by the marker that load itself moved. The store holding THIS request's stamp
|
|
794
|
+
// says so: no other load writes it. Answered as applied, without `switchedOff`, so the
|
|
795
|
+
// caller's re-assert after the bind covers the OFF positions.
|
|
796
|
+
const now = await this.scopeStub(scopeId).loadMarker();
|
|
797
|
+
if (!opts?.loadStamp || now.loadStamp !== opts.loadStamp) {
|
|
798
|
+
throw substratError('precondition_failed', 'scope store changed since the carry read it; nothing was loaded (#1722)');
|
|
799
|
+
}
|
|
800
|
+
return { tables: tables.length };
|
|
801
|
+
}
|
|
802
|
+
/** The kept-copy marker of this scope's store in THIS deployment (#1722), or null. */
|
|
803
|
+
async keptCopyLocal(scopeId) {
|
|
804
|
+
return this.scopeStub(scopeId).keptCopy();
|
|
805
|
+
}
|
|
806
|
+
/**
|
|
807
|
+
* Discard the kept copy of a scope in THIS deployment (#1722), behind the vertical's
|
|
808
|
+
* `/internal/kept-copy/discard`: the staff resolution, only at the write revision the operator
|
|
809
|
+
* acted on. Answers whether it was discarded, or why not.
|
|
810
|
+
*/
|
|
811
|
+
async discardKeptCopyLocal(scopeId, revision, carriedAway,
|
|
812
|
+
/** #2005: the directory's classification, sent when the scope is not primary. */
|
|
813
|
+
markCopy,
|
|
814
|
+
/** #1722 (Codex #2008 r13): the load stamp read with `revision`; a replaced store is refused. */
|
|
815
|
+
loadStamp) {
|
|
816
|
+
if (markCopy)
|
|
817
|
+
assertCopyLineage(markCopy);
|
|
818
|
+
return this.scopeStub(scopeId).discardKeptCopy(scopeId, revision, carriedAway, markCopy !== undefined, ...(loadStamp !== undefined ? [loadStamp] : []));
|
|
819
|
+
}
|
|
820
|
+
/**
|
|
821
|
+
* What a carry's restore into this scope's store expects to find unchanged (#1722), behind
|
|
822
|
+
* the vertical's `/internal/load-marker`. Read before the carry checks the binding again; the
|
|
823
|
+
* restore then carries it as `expect`, and is refused if the store was loaded or written since.
|
|
824
|
+
*/
|
|
825
|
+
async loadMarkerLocal(scopeId) {
|
|
826
|
+
return this.scopeStub(scopeId).loadMarker();
|
|
827
|
+
}
|
|
828
|
+
/**
|
|
829
|
+
* Wipe the copy a carry left in THIS deployment (#1722), behind the vertical's
|
|
830
|
+
* `/internal/wipe-carried`: only if nothing was loaded into the scope's DO since the stamp
|
|
831
|
+
* the carry read (`expectLoadStamp`, null for a store no load has stamped). Non-terminal,
|
|
832
|
+
* so a later bind back to this version can restore into it. False when refused.
|
|
833
|
+
*/
|
|
834
|
+
async wipeCarriedLocal(scopeId, expectLoadStamp, carriedAway, opts = {}) {
|
|
835
|
+
if (opts.markCopy)
|
|
836
|
+
assertCopyLineage(opts.markCopy);
|
|
837
|
+
return this.scopeStub(scopeId).wipeCarried(scopeId, expectLoadStamp, carriedAway, {
|
|
838
|
+
...(opts.expectRevision !== undefined ? { expectRevision: opts.expectRevision } : {}),
|
|
839
|
+
...(opts.protectIfChanged !== undefined ? { protectIfChanged: opts.protectIfChanged } : {}),
|
|
840
|
+
markCopy: opts.markCopy !== undefined,
|
|
841
|
+
});
|
|
842
|
+
}
|
|
843
|
+
/** Release a kept copy that is the live store after all (#1722), at the revision read. */
|
|
844
|
+
async releaseKeptCopyLocal(scopeId, revision,
|
|
845
|
+
/** #2005: the directory's classification, sent when the scope is not primary. */
|
|
846
|
+
markCopy,
|
|
847
|
+
/** #1722 (Codex #2008 r13): the load stamp read with `revision`; a replaced store is refused. */
|
|
848
|
+
loadStamp) {
|
|
849
|
+
if (markCopy)
|
|
850
|
+
assertCopyLineage(markCopy);
|
|
851
|
+
return this.scopeStub(scopeId).releaseKeptCopy(revision, markCopy !== undefined, ...(loadStamp !== undefined ? [loadStamp] : []));
|
|
852
|
+
}
|
|
853
|
+
/**
|
|
854
|
+
* Store the lifecycle the platform delivered for one scope in THIS deployment (#1713), behind
|
|
855
|
+
* the vertical's `/internal/lifecycle`. Kept unless the scope already holds a newer one, so a
|
|
856
|
+
* push that arrives late cannot undo a later transition. `applied` is what the platform's heal
|
|
857
|
+
* sweep records as delivered; `changed` says the gate's answer moved.
|
|
858
|
+
*/
|
|
859
|
+
async setLifecycleLocal(scopeId, next) {
|
|
860
|
+
return lifecycleDelivery.parse(await this.scopeStub(scopeId).setLifecycle(scopeLifecycle.parse(next)));
|
|
861
|
+
}
|
|
862
|
+
/**
|
|
863
|
+
* Mark one scope in THIS deployment a copy (#2005), behind the vertical's `/internal/mark-copy`:
|
|
864
|
+
* the repair of a copy that predates the marker, which a CP-less coordinator reads for primacy.
|
|
865
|
+
* The control plane decides which scopes (its directory says they are not primary) and audits.
|
|
866
|
+
*/
|
|
867
|
+
async markCopyLocal(scopeId, lineage) {
|
|
868
|
+
assertCopyLineage(lineage);
|
|
869
|
+
return { marked: await this.scopeStub(scopeId).markCopy() };
|
|
870
|
+
}
|
|
871
|
+
/**
|
|
872
|
+
* Clear a mistaken copy classification from one scope in THIS deployment (#2005), behind the
|
|
873
|
+
* vertical's `/internal/clear-copy-mark`: staff's correction for a scope the directory says IS
|
|
874
|
+
* primary. Refused for a scope classified a copy. A load's copied-events mark is never touched
|
|
875
|
+
* (#2009), so a primary restored from another scope's backup runs its own effects again and
|
|
876
|
+
* still never runs the work it copied.
|
|
877
|
+
*/
|
|
878
|
+
async clearCopyMarkLocal(scopeId, lineage,
|
|
879
|
+
/** #1722 (Codex #2008 r12–r13): the store as the caller means it, its load stamp and write
|
|
880
|
+
* revision, compared in the clear's own transaction; refused (412) if either moved. The
|
|
881
|
+
* platform's reconcile of a carry's destination sends it; staff send none. */
|
|
882
|
+
expect) {
|
|
883
|
+
if (!isPrimaryScope(lineage)) {
|
|
884
|
+
throw substratError('conflict', 'clear-copy-mark refused: the directory classifies this scope as a copy (a preview or a fork)');
|
|
885
|
+
}
|
|
886
|
+
const outcome = await this.scopeStub(scopeId).clearCopyMark(...(expect ? [expect] : []));
|
|
887
|
+
if (outcome === 'changed') {
|
|
888
|
+
throw substratError('precondition_failed', `clear-copy-mark refused: scope ${scopeId}'s store changed since its revision was read`);
|
|
889
|
+
}
|
|
890
|
+
return { cleared: outcome === 'cleared' };
|
|
666
891
|
}
|
|
667
892
|
/**
|
|
668
893
|
* Re-apply the vertical's OWN role definitions to one scope, CP-lessly — the
|
|
@@ -699,6 +924,13 @@ export class CloudflareScopeHost {
|
|
|
699
924
|
async exportScopeLocal(scopeId) {
|
|
700
925
|
return this.scopeStub(scopeId).exportDump();
|
|
701
926
|
}
|
|
927
|
+
/**
|
|
928
|
+
* `exportScopeLocal` with the store's load stamp, read in the same DO call (#1722): what a
|
|
929
|
+
* carry's fenced wipe of the copy it leaves here expects. Behind the vertical's `/internal/export`.
|
|
930
|
+
*/
|
|
931
|
+
async exportScopeStampedLocal(scopeId) {
|
|
932
|
+
return this.scopeStub(scopeId).exportDumpStamped();
|
|
933
|
+
}
|
|
702
934
|
/** Facet this host's own scope's outbox (#1239) — the vertical-host read. */
|
|
703
935
|
async facetEventsLocal(scopeId, input) {
|
|
704
936
|
return this.scopeStub(scopeId).facetEvents(input);
|
|
@@ -845,9 +1077,14 @@ export class CloudflareScopeHost {
|
|
|
845
1077
|
throw new Error(`${manifest.id} declares operationInputs for unbound operation(s): ` +
|
|
846
1078
|
`${unboundInputs.sort().join(', ')} — a schema on nothing reads as a parse that is not there`);
|
|
847
1079
|
}
|
|
1080
|
+
const binding = {
|
|
1081
|
+
moduleId: manifest.id,
|
|
1082
|
+
entitlementKey: manifest.entitlementKey,
|
|
1083
|
+
scheduledOperations: new Set((manifest.schedules ?? []).map((sch) => sch.operation)),
|
|
1084
|
+
};
|
|
848
1085
|
for (const name of Object.keys(registration.operations ?? {})) {
|
|
849
1086
|
this.bindOperation(name);
|
|
850
|
-
this.operationEntitlement.set(name,
|
|
1087
|
+
this.operationEntitlement.set(name, binding);
|
|
851
1088
|
}
|
|
852
1089
|
}
|
|
853
1090
|
defineOperation(name, _handler) {
|
|
@@ -1049,7 +1286,7 @@ export class CloudflareScopeHost {
|
|
|
1049
1286
|
// gates and metadata facts live in the ScopeDO (per-scope serialization, spine event
|
|
1050
1287
|
// in the same transaction); bytes go straight to the per-tenant R2 bucket through the
|
|
1051
1288
|
// binding the vertical's worker resolved — never through the DO.
|
|
1052
|
-
await this.
|
|
1289
|
+
await this.assertLive(tenantId, scopeId);
|
|
1053
1290
|
await this.migrateAndRecord(scopeId);
|
|
1054
1291
|
const store = await this.resolveAttachmentStore(tenantId);
|
|
1055
1292
|
return this.buildAttachmentSurface({ principal }, tenantId, scopeId, store);
|
|
@@ -1122,17 +1359,14 @@ export class CloudflareScopeHost {
|
|
|
1122
1359
|
const store = await this.resolveAttachmentStore(conn.tenant_id);
|
|
1123
1360
|
return this.buildAttachmentSurface({ connectionId }, conn.tenant_id, scopeId, store, forEvent);
|
|
1124
1361
|
}
|
|
1125
|
-
/** Resolve the per-tenant R2 blob store, or fail closed
|
|
1362
|
+
/** Resolve the per-tenant R2 blob store, or fail closed. */
|
|
1126
1363
|
async resolveAttachmentStore(tenantId) {
|
|
1127
|
-
if (!this.attachmentBuckets) {
|
|
1128
|
-
throw new Error(`attachments are not configured on this host (#473): pass ` +
|
|
1129
|
-
`CloudflareScopeHostOptions.attachmentBuckets — (tenantId) => ` +
|
|
1130
|
-
`env[blobStoreBindingName('<BINDING>', tenantId)] for the vertical's declared blob store`);
|
|
1131
|
-
}
|
|
1132
1364
|
const bucket = await this.attachmentBuckets(tenantId);
|
|
1133
1365
|
if (!bucket) {
|
|
1134
|
-
throw new Error(`no attachment bucket resolved for tenant ${tenantId} (#473) —
|
|
1135
|
-
|
|
1366
|
+
throw new Error(`no attachment bucket resolved for tenant ${tenantId} (#473, #1995) — this script has no ` +
|
|
1367
|
+
`${blobStoreBindingName(ATTACHMENT_BLOB_BINDING, tenantId)} binding. A vertical whose modules declare ` +
|
|
1368
|
+
`attachmentTargets gets one per installed tenant once it is pushed with a CLI that declares the ` +
|
|
1369
|
+
`store (re-push); off the platform, pass CloudflareScopeHostOptions.attachmentBuckets.`);
|
|
1136
1370
|
}
|
|
1137
1371
|
return r2TenantBlobStore(bucket);
|
|
1138
1372
|
}
|
|
@@ -1365,7 +1599,7 @@ export class CloudflareScopeHost {
|
|
|
1365
1599
|
async getScope(principal, tenantId, scopeId, options) {
|
|
1366
1600
|
// Lifecycle gates (control-plane.md §4.1/§4.2), the K-3 fail-closed path,
|
|
1367
1601
|
// evaluated durably in the ControlPlaneDO. A throw propagates here.
|
|
1368
|
-
await this.
|
|
1602
|
+
await this.assertLive(tenantId, scopeId);
|
|
1369
1603
|
await this.migrateAndRecord(scopeId);
|
|
1370
1604
|
return this.buildStub(tenantId, scopeId, principal, undefined, undefined, options);
|
|
1371
1605
|
}
|
|
@@ -1440,7 +1674,7 @@ export class CloudflareScopeHost {
|
|
|
1440
1674
|
const record = await this.resolveImpersonation(session, tenantId, scopeId);
|
|
1441
1675
|
// The same lifecycle gate the principal door applies: a suspended tenant
|
|
1442
1676
|
// refuses a support session exactly as it refuses a user.
|
|
1443
|
-
await this.
|
|
1677
|
+
await this.assertLive(tenantId, scopeId);
|
|
1444
1678
|
await this.migrateAndRecord(scopeId);
|
|
1445
1679
|
return this.buildStub(tenantId, scopeId, record.principal, undefined, undefined, options, record.id);
|
|
1446
1680
|
}
|
|
@@ -1454,7 +1688,7 @@ export class CloudflareScopeHost {
|
|
|
1454
1688
|
if (!this.moduleIds.has(moduleId)) {
|
|
1455
1689
|
throw new Error(`module not registered on this host: ${moduleId}`);
|
|
1456
1690
|
}
|
|
1457
|
-
await this.
|
|
1691
|
+
await this.assertLive(tenantId, scopeId);
|
|
1458
1692
|
await this.migrateAndRecord(scopeId);
|
|
1459
1693
|
return this.buildStub(tenantId, scopeId, undefined, undefined, moduleId);
|
|
1460
1694
|
}
|
|
@@ -1487,7 +1721,7 @@ export class CloudflareScopeHost {
|
|
|
1487
1721
|
* capability row lives. The answer is re-parsed on this side of the RPC.
|
|
1488
1722
|
*/
|
|
1489
1723
|
async exchangeCapability(tenantId, scopeId, secret, options) {
|
|
1490
|
-
await this.
|
|
1724
|
+
await this.assertLive(tenantId, scopeId);
|
|
1491
1725
|
await this.migrateAndRecord(scopeId);
|
|
1492
1726
|
const outcome = await this.scopeStub(scopeId).exchangeCapability(secret, tenantId, scopeId, options?.mode);
|
|
1493
1727
|
return capabilityExchange.nullable().parse(outcome);
|
|
@@ -1501,7 +1735,7 @@ export class CloudflareScopeHost {
|
|
|
1501
1735
|
if (!plausibleSessionToken(sessionToken)) {
|
|
1502
1736
|
throw substratError('unauthenticated', 'not a capability session token');
|
|
1503
1737
|
}
|
|
1504
|
-
await this.
|
|
1738
|
+
await this.assertLive(tenantId, scopeId);
|
|
1505
1739
|
await this.migrateAndRecord(scopeId);
|
|
1506
1740
|
const hash = await capabilityTokenHash(sessionToken);
|
|
1507
1741
|
return this.buildStub(tenantId, scopeId, undefined, undefined, undefined, options, undefined, hash);
|
|
@@ -1577,7 +1811,7 @@ export class CloudflareScopeHost {
|
|
|
1577
1811
|
}
|
|
1578
1812
|
this.assertServedHere(record, scopeId, verb);
|
|
1579
1813
|
}
|
|
1580
|
-
await this.
|
|
1814
|
+
await this.assertLive(tenantId, scopeId);
|
|
1581
1815
|
await this.migrateAndRecord(scopeId);
|
|
1582
1816
|
}
|
|
1583
1817
|
/**
|
|
@@ -1590,7 +1824,7 @@ export class CloudflareScopeHost {
|
|
|
1590
1824
|
if (!plausibleSessionToken(sessionToken)) {
|
|
1591
1825
|
throw substratError('unauthenticated', 'not a capability session token');
|
|
1592
1826
|
}
|
|
1593
|
-
await this.
|
|
1827
|
+
await this.assertLive(tenantId, scopeId);
|
|
1594
1828
|
await this.migrateAndRecord(scopeId);
|
|
1595
1829
|
const store = await this.resolveAttachmentStore(tenantId);
|
|
1596
1830
|
const capabilitySession = await capabilityTokenHash(sessionToken);
|
|
@@ -1687,6 +1921,10 @@ export class CloudflareScopeHost {
|
|
|
1687
1921
|
if (!rec || rec.status !== 'active')
|
|
1688
1922
|
return report;
|
|
1689
1923
|
}
|
|
1924
|
+
else if (await this.lifecycleHeld(scopeId)) {
|
|
1925
|
+
// #1713: a held scope emits nothing, so judging it stale would only raise a false alarm.
|
|
1926
|
+
return report;
|
|
1927
|
+
}
|
|
1690
1928
|
const stub = this.scopeStub(scopeId);
|
|
1691
1929
|
const windows = new Map();
|
|
1692
1930
|
for (const specs of this.moduleFreshness.values()) {
|
|
@@ -1725,14 +1963,23 @@ export class CloudflareScopeHost {
|
|
|
1725
1963
|
return report;
|
|
1726
1964
|
// Only run on a live scope of this tenant; a scope archived between the sweep's
|
|
1727
1965
|
// enumeration and here simply has nothing due. A CP-less host has no directory
|
|
1728
|
-
// to ask (#461)
|
|
1729
|
-
// whole request path, and lifecycle for its scopes lives wherever provisioning
|
|
1730
|
-
// does, so the grant check below is the only gate it can and does enforce.
|
|
1966
|
+
// to ask (#461), so it reads the lifecycle the platform delivered to the scope (#1713).
|
|
1731
1967
|
if (!this.cpLess) {
|
|
1732
1968
|
const rec = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
1733
1969
|
if (!rec || rec.status !== 'active')
|
|
1734
1970
|
return report;
|
|
1735
1971
|
}
|
|
1972
|
+
else if (await this.lifecycleHeld(scopeId)) {
|
|
1973
|
+
// #1713: the lifecycle the platform delivered holds the scope. Every schedule is
|
|
1974
|
+
// `skipped` and no cadence row moves, as under the kill switch, so a schedule that
|
|
1975
|
+
// came due meanwhile fires once on the first pass after the scope is live again.
|
|
1976
|
+
for (const schedule of schedules) {
|
|
1977
|
+
report.skipped += 1;
|
|
1978
|
+
report.runs.push({ operation: schedule.operation, outcome: 'skipped' });
|
|
1979
|
+
}
|
|
1980
|
+
report.lifecycleHeld = true;
|
|
1981
|
+
return report;
|
|
1982
|
+
}
|
|
1736
1983
|
const stub = this.scopeStub(scopeId);
|
|
1737
1984
|
// The grant IS the switch (#383), and the kill switch is its lever (#1666): the
|
|
1738
1985
|
// kernel's `systemScheduleState`, the one predicate both adapters run. A scope that
|
|
@@ -1756,6 +2003,7 @@ export class CloudflareScopeHost {
|
|
|
1756
2003
|
return report;
|
|
1757
2004
|
}
|
|
1758
2005
|
const now = Date.now();
|
|
2006
|
+
const lines = asyncLinePass();
|
|
1759
2007
|
for (const schedule of schedules) {
|
|
1760
2008
|
const last = await stub.scheduleLastRun(schedule.operation);
|
|
1761
2009
|
const lastRun = last ? Date.parse(last) : null;
|
|
@@ -1770,24 +2018,43 @@ export class CloudflareScopeHost {
|
|
|
1770
2018
|
// produces either way carries it, and so does the row below.
|
|
1771
2019
|
const invocationId = ulid();
|
|
1772
2020
|
let status = 'ok';
|
|
2021
|
+
// #1901: the run's line, under the call's own id — the one its `ctx.log` lines carry.
|
|
2022
|
+
const startedAt = Date.now();
|
|
2023
|
+
let emitted;
|
|
2024
|
+
let failure;
|
|
1773
2025
|
try {
|
|
1774
2026
|
const scope = await this.getSystemScope(moduleId, tenantId, scopeId);
|
|
1775
|
-
await scope.invoke(schedule.operation, schedule.input, { invocationId });
|
|
2027
|
+
await scope.invoke(schedule.operation, schedule.input, { invocationId, onEmitted: (r) => (emitted = r) });
|
|
1776
2028
|
report.fired += 1;
|
|
1777
2029
|
}
|
|
1778
2030
|
catch (err) {
|
|
1779
2031
|
status = 'failed';
|
|
2032
|
+
failure = { error: err };
|
|
1780
2033
|
report.failed += 1;
|
|
1781
2034
|
report.errors.push({
|
|
1782
2035
|
operation: schedule.operation,
|
|
1783
2036
|
error: err instanceof Error ? err.message : String(err),
|
|
1784
2037
|
});
|
|
1785
2038
|
}
|
|
2039
|
+
lines.write({
|
|
2040
|
+
kind: 'schedule',
|
|
2041
|
+
tenantId,
|
|
2042
|
+
scopeId,
|
|
2043
|
+
invocationId,
|
|
2044
|
+
operation: schedule.operation,
|
|
2045
|
+
startedAt,
|
|
2046
|
+
outcome: status,
|
|
2047
|
+
...failure,
|
|
2048
|
+
dueAt: lastRun === null ? null : new Date(dueAt).toISOString(),
|
|
2049
|
+
latenessMs: lastRun === null ? null : now - dueAt,
|
|
2050
|
+
...(emitted ? { emitted } : {}),
|
|
2051
|
+
});
|
|
1786
2052
|
// #1288: 'schedule', whatever this operation happens to be called — including
|
|
1787
2053
|
// `freshness:<something>`, which is exactly the row the evaluator no longer eats.
|
|
1788
2054
|
await stub.recordScheduleRun(schedule.operation, new Date(now).toISOString(), status, 'schedule', invocationId);
|
|
1789
2055
|
report.runs.push({ operation: schedule.operation, outcome: status === 'ok' ? 'ok' : 'failed' });
|
|
1790
2056
|
}
|
|
2057
|
+
lines.end();
|
|
1791
2058
|
return report;
|
|
1792
2059
|
}
|
|
1793
2060
|
/** The stub body, shared by the principal, connection, system and impersonation doors. */
|
|
@@ -1848,7 +2115,13 @@ export class CloudflareScopeHost {
|
|
|
1848
2115
|
// (`cp.tenantHoldsEntitlement`); for a hosted/CP-less scope that call is a trusting
|
|
1849
2116
|
// no-op, so the SAME `requiredKey` is passed to the DO, which fails closed against
|
|
1850
2117
|
// its PROJECTED entitlements (#304). One or the other enforces, never neither.
|
|
1851
|
-
|
|
2118
|
+
//
|
|
2119
|
+
// EXCEPTION (#1654): a module's own declared schedule, through this system door,
|
|
2120
|
+
// demands no SKU — its `system:<moduleId>` grant is the switch, as it already is for
|
|
2121
|
+
// permissions (#383). `requiredEntitlementFor` holds the whole rule and why; it
|
|
2122
|
+
// answers `undefined` there, so neither the coordinator nor the DO asks. Every other
|
|
2123
|
+
// door passes no `systemModuleId` and is gated exactly as before.
|
|
2124
|
+
const requiredKey = requiredEntitlementFor(operation, operationEntitlement.get(operation), systemModuleId);
|
|
1852
2125
|
if (requiredKey && !(await cp.tenantHoldsEntitlement(tenantId, requiredKey))) {
|
|
1853
2126
|
// Required AND held (#691) — a second CP read, but only on the denial path.
|
|
1854
2127
|
const now = new Date().toISOString();
|
|
@@ -2103,6 +2376,8 @@ export class CloudflareScopeHost {
|
|
|
2103
2376
|
// returns it with the previous status, so the trail cannot disagree with
|
|
2104
2377
|
// the directory about which deployment the action touched.
|
|
2105
2378
|
await this.recordAdmin(actor, action, { tenantId, scopeId, vertical: before.vertical }, { status: before.status }, { status: to, ...afterExtra });
|
|
2379
|
+
// #1713: after the directory moved and the move was audited, never before or instead.
|
|
2380
|
+
await this.deliverLifecycles(actor, { scopeId });
|
|
2106
2381
|
};
|
|
2107
2382
|
const writeGrant = async (subject, permission, node, entity, expiresAt) => {
|
|
2108
2383
|
if (entity) {
|
|
@@ -3032,7 +3307,7 @@ export class CloudflareScopeHost {
|
|
|
3032
3307
|
if (!v)
|
|
3033
3308
|
throw substratError('not_found', `unknown version ${versionId}`);
|
|
3034
3309
|
if (v.admission === 'admitted') {
|
|
3035
|
-
throw
|
|
3310
|
+
throw substratError('conflict', `version ${versionId} is already admitted — it may be bound`);
|
|
3036
3311
|
}
|
|
3037
3312
|
if (v.admission === 'rejected')
|
|
3038
3313
|
return;
|
|
@@ -3044,7 +3319,7 @@ export class CloudflareScopeHost {
|
|
|
3044
3319
|
if (!incoming)
|
|
3045
3320
|
throw substratError('not_found', `unknown version ${versionId}`);
|
|
3046
3321
|
if (incoming.vertical_slug !== verticalSlug) {
|
|
3047
|
-
throw
|
|
3322
|
+
throw substratError('conflict', `version ${versionId} belongs to '${incoming.vertical_slug}'`);
|
|
3048
3323
|
}
|
|
3049
3324
|
if (incoming.admission !== 'admitted') {
|
|
3050
3325
|
throw substratError('conflict', `version ${versionId} is ${incoming.admission}, not admitted — it cannot be promoted`);
|
|
@@ -3104,7 +3379,7 @@ export class CloudflareScopeHost {
|
|
|
3104
3379
|
if (channel === 'prod' && !incoming.deployment_ref) {
|
|
3105
3380
|
const owning = await this.cp.readVertical(verticalSlug);
|
|
3106
3381
|
if (owning && owning.owner_tenant !== null && !owning.listed) {
|
|
3107
|
-
const bound = (await this.cp.listScopes({ tenantId: owning.owner_tenant, vertical: verticalSlug, status: ['active'] })).filter(
|
|
3382
|
+
const bound = (await this.cp.listScopes({ tenantId: owning.owner_tenant, vertical: verticalSlug, status: ['active'] })).filter(isPrimaryScopeRow);
|
|
3108
3383
|
for (const s of bound) {
|
|
3109
3384
|
if (s.vertical_version_id === versionId)
|
|
3110
3385
|
continue;
|
|
@@ -3262,6 +3537,12 @@ export class CloudflareScopeHost {
|
|
|
3262
3537
|
await this.cp.setScopeServingRef(scopeId, servingRef);
|
|
3263
3538
|
await this.recordAdmin(actor, 'setScopeServingRef', { tenantId, scopeId }, { servingRef: scope.serving_ref ?? null }, { servingRef, ...(ack.exportBreak ? { acknowledged: ack } : {}) });
|
|
3264
3539
|
},
|
|
3540
|
+
recordKeptCopyResolution: async (actor, tenantId, scopeId, r) => {
|
|
3541
|
+
const scope = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
3542
|
+
if (!scope)
|
|
3543
|
+
throw substratError('not_found', `unknown scope ${scopeId} in tenant ${tenantId}`);
|
|
3544
|
+
await this.recordAdmin(actor, 'resolveKeptCopy', { tenantId, scopeId }, { script: r.script, keptAt: r.keptAt, revision: r.revisionBefore }, { action: r.action, liveScript: r.liveScript, revision: r.revisionAfter });
|
|
3545
|
+
},
|
|
3265
3546
|
setScopeExpiresAt: async (actor, tenantId, scopeId, expiresAt) => {
|
|
3266
3547
|
const scope = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
3267
3548
|
if (!scope)
|
|
@@ -3366,6 +3647,8 @@ export class CloudflareScopeHost {
|
|
|
3366
3647
|
setTenantStatus: async (actor, tenantId, status) => {
|
|
3367
3648
|
const before = await this.cp.setTenantStatus(tenantId, status);
|
|
3368
3649
|
await this.recordAdmin(actor, 'setTenantStatus', { tenantId }, { status: before }, { status });
|
|
3650
|
+
// #1713: every hosted scope under the tenant holds or resumes its work by this.
|
|
3651
|
+
await this.deliverLifecycles(actor, { tenantId });
|
|
3369
3652
|
},
|
|
3370
3653
|
setTenantName: async (actor, tenantId, name) => {
|
|
3371
3654
|
const before = await this.cp.setTenantName(tenantId, name);
|
|
@@ -3660,6 +3943,26 @@ export class CloudflareScopeHost {
|
|
|
3660
3943
|
await this.recordAccess(actor, 'listDenials', { tenantId, scopeId }, filter ?? null, rows.length);
|
|
3661
3944
|
return rows;
|
|
3662
3945
|
},
|
|
3946
|
+
// #1686: the operator's capability read. K-3 on the directory before the DO, like the
|
|
3947
|
+
// denial log. On the shared control plane a scope a vertical's own deployment serves
|
|
3948
|
+
// holds its storage THERE, and this namespace holds a placeholder: the control-plane
|
|
3949
|
+
// API asks the vertical (`listCapabilitiesLocal`), so this branch is the co-located one.
|
|
3950
|
+
listCapabilities: async (actor, tenantId, scopeId, filter) => {
|
|
3951
|
+
// Parsed here too: a typed refusal thrown inside the DO arrives as a bare message.
|
|
3952
|
+
const parsed = capabilityFilter.parse(filter ?? {});
|
|
3953
|
+
const rec = await this.scopeRecordForRead(tenantId, scopeId);
|
|
3954
|
+
this.assertServedHere(rec, scopeId, 'listCapabilities');
|
|
3955
|
+
const page = await this.scopeStub(scopeId).listCapabilities(parsed);
|
|
3956
|
+
await this.recordAccess(actor, 'listCapabilities', { tenantId, scopeId }, filter ?? null, page.entries.length);
|
|
3957
|
+
return page;
|
|
3958
|
+
},
|
|
3959
|
+
listRefusals: async (actor, tenantId, scopeId, filter) => {
|
|
3960
|
+
// #1745: the denial log's discipline — K-3 on the directory before the DO is reached.
|
|
3961
|
+
await this.scopeRecordForRead(tenantId, scopeId);
|
|
3962
|
+
const rows = await this.scopeStub(scopeId).listRefusals(filter);
|
|
3963
|
+
await this.recordAccess(actor, 'listRefusals', { tenantId, scopeId }, filter ?? null, rows.length);
|
|
3964
|
+
return rows;
|
|
3965
|
+
},
|
|
3663
3966
|
summarizeDenials: async (actor, tenantId, scopeId, filter) => {
|
|
3664
3967
|
await this.scopeRecordForRead(tenantId, scopeId);
|
|
3665
3968
|
const summary = await this.scopeStub(scopeId).summarizeDenials(filter);
|
|
@@ -4218,6 +4521,11 @@ export class CloudflareScopeHost {
|
|
|
4218
4521
|
const { tenantId, scopeId, ...after } = ownerTransferAudit.parse(entry);
|
|
4219
4522
|
await this.recordAdmin(actor, 'transferOwner', { tenantId, scopeId }, null, after);
|
|
4220
4523
|
},
|
|
4524
|
+
/** #2005: one change to a scope's copy marker, written around the vertical's own change. */
|
|
4525
|
+
recordCopyMark: async (actor, entry) => {
|
|
4526
|
+
const { tenantId, scopeId, action, ...after } = copyMarkAudit.parse(entry);
|
|
4527
|
+
await this.recordAdmin(actor, action === 'mark' ? 'markScopeCopy' : 'clearScopeCopyMark', { tenantId, scopeId }, null, after);
|
|
4528
|
+
},
|
|
4221
4529
|
accessLog: async (actor, filter) => {
|
|
4222
4530
|
const rows = await this.cp.accessLog({
|
|
4223
4531
|
actor: filter?.actor,
|
|
@@ -4439,6 +4747,184 @@ export class CloudflareScopeHost {
|
|
|
4439
4747
|
return;
|
|
4440
4748
|
throw refusal.code ? substratError(refusal.code, refusal.message) : new Error(refusal.message);
|
|
4441
4749
|
}
|
|
4750
|
+
/**
|
|
4751
|
+
* THE lifecycle gate (#1713) at every door that runs a scope's work: a request's stubs and
|
|
4752
|
+
* attachments, a capability or impersonation session, a peer call or delivery, a subscription,
|
|
4753
|
+
* and the entry points no request asked for — the retry driver, the job runner, a schedule's
|
|
4754
|
+
* system stub, a routed connector delivery.
|
|
4755
|
+
*
|
|
4756
|
+
* `validateScopeAccess` first, which is the directory's gate where there is a directory. A
|
|
4757
|
+
* CP-less host has none, so it then reads the lifecycle the platform delivered to the scope's
|
|
4758
|
+
* own storage (`/internal/lifecycle`), judged by the kernel's `lifecycleRefusal` in the
|
|
4759
|
+
* directory's own words. The router refuses a suspended scope's requests first (#1730); this
|
|
4760
|
+
* is the deployment's own half, and the only one a timer or a retry ever meets.
|
|
4761
|
+
*
|
|
4762
|
+
* The admin and diagnostic reads (dead letters, job runs, platform requests, delivered
|
|
4763
|
+
* grants) keep `validateScopeAccess` alone: an operator still reads a suspended scope.
|
|
4764
|
+
*/
|
|
4765
|
+
async assertLive(tenantId, scopeId) {
|
|
4766
|
+
await this.validateScopeAccess(tenantId, scopeId);
|
|
4767
|
+
const refusal = await this.cpLessRefusal(scopeId, { tenantId, scopeId });
|
|
4768
|
+
if (refusal)
|
|
4769
|
+
throw new Error(refusal);
|
|
4770
|
+
}
|
|
4771
|
+
/** The delivered lifecycle's refusal on a CP-less host (#1713); always null with a directory. */
|
|
4772
|
+
async cpLessRefusal(scopeId, ids) {
|
|
4773
|
+
if (!this.cpLess)
|
|
4774
|
+
return null;
|
|
4775
|
+
return lifecycleRefusal(await this.scopeStub(scopeId).lifecycle(), ids);
|
|
4776
|
+
}
|
|
4777
|
+
/**
|
|
4778
|
+
* Deliver the directory's lifecycle to the deployments serving the matching scopes (#1713):
|
|
4779
|
+
* one scope after its transition, a tenant's scopes after the tenant's, or the heal sweep's
|
|
4780
|
+
* drift. Each delivery carries the directory's revisions, read in the same query as the
|
|
4781
|
+
* statuses, so a late one cannot undo a later transition: the deployment keeps only a delivery
|
|
4782
|
+
* strictly newer than the one it holds.
|
|
4783
|
+
*
|
|
4784
|
+
* Never throws. A transition is the operator's lever in an incident, and the router refuses
|
|
4785
|
+
* the scope's requests as soon as the directory moves (#1730), so a deployment that cannot be
|
|
4786
|
+
* reached must not refuse the lever. Each failure is an ops-failure row, no receipt is written,
|
|
4787
|
+
* and the heal sweep delivers again.
|
|
4788
|
+
*/
|
|
4789
|
+
async deliverLifecycles(actor, filter) {
|
|
4790
|
+
const report = { attempted: 0, delivered: 0, failed: 0 };
|
|
4791
|
+
const delegation = this.lifecycleDelegation;
|
|
4792
|
+
if (!delegation || this.cpLess)
|
|
4793
|
+
return report;
|
|
4794
|
+
let targets;
|
|
4795
|
+
try {
|
|
4796
|
+
targets = await this.cp.lifecycleTargets(filter);
|
|
4797
|
+
}
|
|
4798
|
+
catch (err) {
|
|
4799
|
+
console.error('substrat: could not read the scopes to deliver a lifecycle to (#1713)', err);
|
|
4800
|
+
return report;
|
|
4801
|
+
}
|
|
4802
|
+
for (const first of targets) {
|
|
4803
|
+
const tenantId = first.tenant_id;
|
|
4804
|
+
const scopeId = first.scope_id;
|
|
4805
|
+
let t = first;
|
|
4806
|
+
let lifecycle = this.lifecycleOfTarget(t);
|
|
4807
|
+
if (!lifecycle) {
|
|
4808
|
+
// A status this code does not know (a newer directory): delivering a guess could lift a hold.
|
|
4809
|
+
report.failed += 1;
|
|
4810
|
+
console.error(`substrat: scope ${scopeId} has a lifecycle this code cannot read (#1713)`);
|
|
4811
|
+
continue;
|
|
4812
|
+
}
|
|
4813
|
+
// A live scope whose deployment already acknowledged exactly this has nothing to receive,
|
|
4814
|
+
// and neither does one with no receipt in a directory never restored: a deployment holding
|
|
4815
|
+
// no lifecycle runs live. That keeps an activation, and every transition before a
|
|
4816
|
+
// deployment carries the route, from posting a delivery that changes nothing. After a
|
|
4817
|
+
// restore (epoch > 0) no receipt says nothing about what the deployment holds, so it is
|
|
4818
|
+
// delivered. A HOLD is always delivered.
|
|
4819
|
+
if (lifecycleRefusal(lifecycle) === null && (t.delivered === null ? t.epoch === 0 : t.delivered === lifecycleReceipt(lifecycle))) {
|
|
4820
|
+
continue;
|
|
4821
|
+
}
|
|
4822
|
+
report.attempted += 1;
|
|
4823
|
+
try {
|
|
4824
|
+
let answer = await delegation.deliver({ tenantId, scopeId, lifecycle });
|
|
4825
|
+
// SINGLE AUTHORITY (the invariant this rests on): an environment has exactly one directory,
|
|
4826
|
+
// the singleton `CONTROL_PLANE.idFromName('control-plane')`. A "fresh directory" restore is
|
|
4827
|
+
// that same object restored after its storage was lost, never a second live writer, and
|
|
4828
|
+
// the raise below always re-reads the CURRENT store and delivers what it says. Two control
|
|
4829
|
+
// planes healing one dispatch namespace at once is a split brain that corrupts far more
|
|
4830
|
+
// than lifecycle, and is out of scope here; fencing it in lifecycle alone would guarantee
|
|
4831
|
+
// nothing.
|
|
4832
|
+
//
|
|
4833
|
+
// The scope refused us as OLDER while holding a revision this directory's history did not
|
|
4834
|
+
// write: an epoch ahead of ours, or ours with other counters. Only another history of the
|
|
4835
|
+
// directory delivers those — the one a fresh-directory restore replaced, whose epoch a
|
|
4836
|
+
// clock running behind (or a restore in the same millisecond) cannot outrank. A scope only
|
|
4837
|
+
// ever holds epochs a directory minted, so this directory learns past it: its epoch is
|
|
4838
|
+
// raised above the one held (monotonic, one statement), and the scope is delivered again.
|
|
4839
|
+
const held = answer.lifecycle.revision;
|
|
4840
|
+
const foreign = !answer.applied &&
|
|
4841
|
+
held !== undefined &&
|
|
4842
|
+
(held.epoch > lifecycle.revision.epoch ||
|
|
4843
|
+
(held.epoch === lifecycle.revision.epoch && lifecycleReceipt(answer.lifecycle) !== lifecycleReceipt(lifecycle)));
|
|
4844
|
+
// Bounded, because the deployment answering is the vertical's own code: only a directory
|
|
4845
|
+
// that has been restored can meet another history (epoch > 0), and a legitimate epoch is
|
|
4846
|
+
// a mint time, so one further ahead than the skew is forged or broken. Either way nothing
|
|
4847
|
+
// is raised; the refusal is an ops failure for an operator to look at.
|
|
4848
|
+
if (foreign && (lifecycle.revision.epoch === 0 || held.epoch > Date.now() + LIFECYCLE_EPOCH_SKEW_MS)) {
|
|
4849
|
+
report.failed += 1;
|
|
4850
|
+
await this.admin
|
|
4851
|
+
.recordOpsFailure({
|
|
4852
|
+
actor,
|
|
4853
|
+
operation: 'scope.lifecycle',
|
|
4854
|
+
stage: 'foreign-epoch',
|
|
4855
|
+
tenantId,
|
|
4856
|
+
scopeId,
|
|
4857
|
+
message: `the deployment refused this directory's lifecycle while holding epoch ${held.epoch} ` +
|
|
4858
|
+
`(this directory is at ${lifecycle.revision.epoch}); not raised past it — ` +
|
|
4859
|
+
(lifecycle.revision.epoch === 0
|
|
4860
|
+
? 'this directory has never been restored, so no other history should have written it'
|
|
4861
|
+
: 'it is further ahead of the clock than any directory mints'),
|
|
4862
|
+
})
|
|
4863
|
+
.catch((e) => console.error('substrat: could not record a foreign lifecycle epoch (#1713)', e));
|
|
4864
|
+
continue;
|
|
4865
|
+
}
|
|
4866
|
+
if (foreign) {
|
|
4867
|
+
await this.cp.raiseLifecycleEpoch(held.epoch + 1);
|
|
4868
|
+
const [again] = await this.cp.lifecycleTargets({ scopeId });
|
|
4869
|
+
const next = again ? this.lifecycleOfTarget(again) : null;
|
|
4870
|
+
if (again && next) {
|
|
4871
|
+
t = again;
|
|
4872
|
+
lifecycle = next;
|
|
4873
|
+
answer = await delegation.deliver({ tenantId, scopeId, lifecycle });
|
|
4874
|
+
}
|
|
4875
|
+
}
|
|
4876
|
+
// The receipt is what the scope HOLDS: a newer delivery it kept instead is the truth.
|
|
4877
|
+
await this.cp.recordLifecycleReceipt(scopeId, lifecycleReceipt(answer.lifecycle), new Date().toISOString());
|
|
4878
|
+
report.delivered += 1;
|
|
4879
|
+
}
|
|
4880
|
+
catch (err) {
|
|
4881
|
+
report.failed += 1;
|
|
4882
|
+
try {
|
|
4883
|
+
await this.admin.recordOpsFailure({
|
|
4884
|
+
actor,
|
|
4885
|
+
operation: 'scope.lifecycle',
|
|
4886
|
+
stage: 'deliver',
|
|
4887
|
+
tenantId,
|
|
4888
|
+
scopeId,
|
|
4889
|
+
message: `the lifecycle (scope ${t.scope_status}, tenant ${t.tenant_status}) did not reach the ` +
|
|
4890
|
+
`deployment serving this scope; the heal sweep will deliver it again: ${err instanceof Error ? err.message : String(err)}`,
|
|
4891
|
+
});
|
|
4892
|
+
}
|
|
4893
|
+
catch (recordErr) {
|
|
4894
|
+
console.error('substrat: could not record a lifecycle delivery failure (#1713)', recordErr);
|
|
4895
|
+
}
|
|
4896
|
+
}
|
|
4897
|
+
}
|
|
4898
|
+
return report;
|
|
4899
|
+
}
|
|
4900
|
+
/** One directory row as the lifecycle a delivery carries (#1713), or null for a status this code does not know. */
|
|
4901
|
+
lifecycleOfTarget(t) {
|
|
4902
|
+
const parsed = scopeLifecycle.safeParse({
|
|
4903
|
+
scope: t.scope_status,
|
|
4904
|
+
tenant: t.tenant_status,
|
|
4905
|
+
at: new Date().toISOString(),
|
|
4906
|
+
revision: { epoch: t.epoch, scope: t.scope_rev, tenant: t.tenant_rev },
|
|
4907
|
+
});
|
|
4908
|
+
return parsed.success ? parsed.data : null;
|
|
4909
|
+
}
|
|
4910
|
+
/**
|
|
4911
|
+
* The heal sweep (#1713): deliver the lifecycle to every hosted scope whose deployment's
|
|
4912
|
+
* acknowledged state differs from the directory, and again to every scope held now — which is
|
|
4913
|
+
* what puts a hold back on a store a carry or a restore landed without it. Bounded per pass
|
|
4914
|
+
* (`limit`), drifted scopes first. Run from the shared control plane's cron.
|
|
4915
|
+
*/
|
|
4916
|
+
async healLifecycles(actor, opts) {
|
|
4917
|
+
return this.deliverLifecycles(actor, { drift: true, limit: opts?.limit ?? 200 });
|
|
4918
|
+
}
|
|
4919
|
+
/**
|
|
4920
|
+
* Whether a CP-less host holds this scope's work (#1713): `assertLive`'s predicate, answered
|
|
4921
|
+
* rather than thrown, for the entry points that defer instead of refusing — a sweep pass, a
|
|
4922
|
+
* schedule run, an executor drain. Always false on a host with a directory, whose entry
|
|
4923
|
+
* points ask the directory themselves.
|
|
4924
|
+
*/
|
|
4925
|
+
async lifecycleHeld(scopeId) {
|
|
4926
|
+
return (await this.cpLessRefusal(scopeId)) !== null;
|
|
4927
|
+
}
|
|
4442
4928
|
/**
|
|
4443
4929
|
* K-3's cross-check on its own: the (tenant, scope) pair must exist and agree before a
|
|
4444
4930
|
* subject-key operation touches anything. Without it a caller could reach another
|
|
@@ -4955,16 +5441,16 @@ export class CloudflareScopeHost {
|
|
|
4955
5441
|
* suspended or archiving scope could still be addressed and handed a 101 — a
|
|
4956
5442
|
* scope that refuses every ordinary read while quietly holding an open socket.
|
|
4957
5443
|
*
|
|
4958
|
-
*
|
|
4959
|
-
*
|
|
4960
|
-
*
|
|
5444
|
+
* On a CP-less vertical, `validateScopeAccess` is a no-op on the null control
|
|
5445
|
+
* plane, and `assertLive` reads the lifecycle the platform delivered to the
|
|
5446
|
+
* scope's own storage instead (#1713).
|
|
4961
5447
|
*
|
|
4962
5448
|
* `migrateAndRecord`, not the DO's own `ensureMigrations`: the DO migrates
|
|
4963
5449
|
* itself when the socket opens either way, but only this reports the applied
|
|
4964
5450
|
* count to the directory — so a scope whose first contact after a deploy is a
|
|
4965
5451
|
* subscription does not go dark in the migration fleet view.
|
|
4966
5452
|
*/
|
|
4967
|
-
await this.
|
|
5453
|
+
await this.assertLive(tenantId, scopeId);
|
|
4968
5454
|
await this.migrateAndRecord(scopeId);
|
|
4969
5455
|
// Asserted, not carried through from the client: the principal is the
|
|
4970
5456
|
// vertical's own resolution of its session, and the tenant and scope are the
|
|
@@ -5327,14 +5813,19 @@ export class CloudflareScopeHost {
|
|
|
5327
5813
|
// side before the call; what runs HERE is the half only this deployment can
|
|
5328
5814
|
// enforce: the scope's own permission check against its delivered
|
|
5329
5815
|
// `connection:<id>` tuple, in the scope's own DO. Fail closed — no grant, no
|
|
5330
|
-
// effect — exactly as for any other caller.
|
|
5816
|
+
// effect — exactly as for any other caller. And the lifecycle delivered to the
|
|
5817
|
+
// scope (#1713): a held scope's invoke and bytes doors refuse, as every other door does.
|
|
5331
5818
|
/** Invoke ONE operation in this deployment as a CONNECTION (#574). */
|
|
5332
5819
|
async connectorInvokeLocal(connectionId, tenantId, scopeId, operation, input) {
|
|
5820
|
+
await this.assertLive(tenantId, scopeId); // #1713: a held scope runs no connector work either
|
|
5333
5821
|
await this.migrateAndRecord(scopeId);
|
|
5334
5822
|
return this.buildStub(tenantId, scopeId, undefined, connectionId).invoke(operation, input);
|
|
5335
5823
|
}
|
|
5336
5824
|
/** Land provider bytes in this deployment as a CONNECTION — the bytes leg (#574). */
|
|
5337
5825
|
async connectorAttachmentUploadLocal(connectionId, tenantId, scopeId, upload) {
|
|
5826
|
+
// The lifecycle gate every attachments door takes (#1713), ahead of the bucket: a held
|
|
5827
|
+
// scope's provider bytes wait for the delivery's retry rather than land (#1995).
|
|
5828
|
+
await this.assertLive(tenantId, scopeId);
|
|
5338
5829
|
await this.migrateAndRecord(scopeId);
|
|
5339
5830
|
const store = await this.resolveAttachmentStore(tenantId);
|
|
5340
5831
|
return this.buildAttachmentSurface({ connectionId }, tenantId, scopeId, store).upload(upload);
|
|
@@ -5347,6 +5838,8 @@ export class CloudflareScopeHost {
|
|
|
5347
5838
|
* so a caller falls back rather than failing a dispatch over a missing file.
|
|
5348
5839
|
*/
|
|
5349
5840
|
async connectorAttachmentOpenLocal(connectionId, tenantId, scopeId, attachmentId, eventId) {
|
|
5841
|
+
// Same gate as the upload leg: a held scope's bytes are not read out of its bucket.
|
|
5842
|
+
await this.assertLive(tenantId, scopeId);
|
|
5350
5843
|
await this.migrateAndRecord(scopeId);
|
|
5351
5844
|
const store = await this.resolveAttachmentStore(tenantId);
|
|
5352
5845
|
return this.buildAttachmentSurface({ connectionId }, tenantId, scopeId, store,
|