@substrat-run/adapter-cloudflare 0.135.0 → 0.136.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/connector-calls.d.ts +11 -0
- package/dist/connector-calls.d.ts.map +1 -0
- package/dist/connector-calls.js +10 -0
- package/dist/connector-calls.js.map +1 -0
- package/dist/control-plane-do.d.ts +76 -0
- package/dist/control-plane-do.d.ts.map +1 -1
- package/dist/control-plane-do.js +172 -49
- package/dist/control-plane-do.js.map +1 -1
- package/dist/host.d.ts +193 -12
- package/dist/host.d.ts.map +1 -1
- package/dist/host.js +497 -46
- package/dist/host.js.map +1 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/live-reads.d.ts +5 -4
- package/dist/live-reads.d.ts.map +1 -1
- package/dist/live-reads.js +5 -4
- package/dist/live-reads.js.map +1 -1
- package/dist/r2.d.ts +2 -1
- package/dist/r2.d.ts.map +1 -1
- package/dist/r2.js.map +1 -1
- package/dist/route-resolver.d.ts +7 -0
- package/dist/route-resolver.d.ts.map +1 -1
- package/dist/route-resolver.js +4 -0
- package/dist/route-resolver.js.map +1 -1
- package/dist/scope-do.d.ts.map +1 -1
- package/dist/scope-do.js +632 -100
- package/dist/scope-do.js.map +1 -1
- package/dist/scope-sweeper-do.d.ts +32 -6
- package/dist/scope-sweeper-do.d.ts.map +1 -1
- package/dist/scope-sweeper-do.js +35 -5
- package/dist/scope-sweeper-do.js.map +1 -1
- package/package.json +10 -9
package/dist/host.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
|
+
import { env as ambientEnv } from 'cloudflare:workers';
|
|
1
2
|
import { isRewindRefusal, REWIND_REFUSED } from './rewind-refusal.js';
|
|
2
|
-
import { delegatedReadParams, fromWireFailure, exportReadInput, exportedBatch, importBatch, importResult, importState, importsOfManifestJson, exportsOfManifestJson, importCursorMove, importCursorMoveAt, importCursorMoved, accessLogEntry, adminLogEntry, opsFailureEntry, opsFailureFingerprint, issueEntry, sweepRunEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunsPayload, modelUsageEntry, attachmentRecord, createTenantInput, identityLink, identityPool, createOrgInput, promotionAcknowledgement, bindAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, registerVerticalInput, vertical as verticalSchema, verticalServingState, verticalChannel, verticalVersion, connection, capabilityExchange, capabilityGrant, principalId, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, systemSwitch, systemSwitchOutcome, systemScheduleEntry, systemSwitchRecord, entitlementGrant, entitlementGrantInput, capabilityFilter, instant, meterReading, subjectRef, createConnectionInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, org as orgSchema, orgMembership, resolvedIdentity, identityMembership, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, delegatedReadRecord, ownerTransferAudit, subjectShredReceipt, connectorDispatchKind, callsOfManifestJson, outboundOfManifestJson, substratError, redrainEventsInput, coverage, peerCoverage, peerGrantsEntry, peerSwitch, peerSwitchOutcome, verticalCaller, verticalResolution, verticalSlug, entityObjectRef, } from '@substrat-run/contracts';
|
|
3
|
+
import { delegatedReadParams, fromWireFailure, exportReadInput, exportedBatch, importBatch, importResult, importState, importsOfManifestJson, exportsOfManifestJson, importCursorMove, importCursorMoveAt, importCursorMoved, accessLogEntry, adminLogEntry, opsFailureEntry, opsFailureFingerprint, issueEntry, sweepRunEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunsPayload, modelUsageEntry, attachmentRecord, createTenantInput, identityLink, identityPool, createOrgInput, promotionAcknowledgement, bindAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, registerVerticalInput, vertical as verticalSchema, verticalServingState, verticalChannel, verticalVersion, connection, capabilityExchange, capabilityGrant, principalId, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, systemSwitch, systemSwitchOutcome, systemScheduleEntry, systemSwitchRecord, entitlementGrant, entitlementGrantInput, capabilityFilter, instant, meterReading, subjectRef, createConnectionInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, org as orgSchema, orgMembership, resolvedIdentity, identityMembership, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, delegatedReadRecord, ownerTransferAudit, copyMarkAudit, subjectShredReceipt, connectorDispatchKind, callsOfManifestJson, outboundOfManifestJson, substratError, redrainEventsInput, coverage, peerCoverage, peerGrantsEntry, peerSwitch, peerSwitchOutcome, verticalCaller, verticalResolution, verticalSlug, entityObjectRef, ATTACHMENT_BLOB_BINDING, blobStoreBindingName, lifecycleDelivery, scopeLifecycle, } from '@substrat-run/contracts';
|
|
3
4
|
import { normalizeHostname, toRouteTarget } from './route-resolver.js';
|
|
4
|
-
import { attachmentBlobKey, attachmentSha256, entitlementDenial, requiredEntitlementFor, foldMeterReading, parseValidationRecords, resolveScopeRecord, ulid, capabilityTokenHash, checkBecomeInput, plausibleSessionToken, foldModelUsage, backoffAt, resolveRetryPolicy, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, openSealed, ImpersonationRefused, assertSessionUsable, impersonationRowValues, mapImpersonationRow, newImpersonationSession, jobRunOf, runDueJobRuns, startJobRun, attachmentTextJob, assertAttachmentExtractors, assertJobRegistrable, isAttachmentTextRun, searchLimit, searchMatchExpression, globalFetch, assertRedrainWindow, platformRequestOf, undrainedEventsOf, inUnitMovesToAudit, staleCarryRevertRow, staleCarryReverts, withRecorded, systemSwitchedOffMessage, tenantSystemSwitchedOffMessage, CrossVerticalRegistry, importCursorSourceOf, exportBreaksOf, exportBreakRefusal, bindExportBreaksOf, bindExportBreakRefusal, collectPeers, peerSeats, connectorCallRecord, noopConnectorCallRecorder, recordConnectorCall, settleConnectionUse, splitManifestMigrations, unknownRoleError, assertRowLimit, assertRowOffset, } from '@substrat-run/kernel';
|
|
5
|
+
import { attachmentBlobKey, attachmentSha256, entitlementDenial, requiredEntitlementFor, foldMeterReading, parseValidationRecords, resolveScopeRecord, ulid, KEPT_COPY_REFUSAL, capabilityTokenHash, checkBecomeInput, plausibleSessionToken, foldModelUsage, backoffAt, resolveRetryPolicy, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, openSealed, ImpersonationRefused, assertSessionUsable, impersonationRowValues, mapImpersonationRow, newImpersonationSession, jobRunOf, runDueJobRuns, startJobRun, attachmentTextJob, assertAttachmentExtractors, assertJobRegistrable, isAttachmentTextRun, searchLimit, searchMatchExpression, globalFetch, assertRedrainWindow, platformRequestOf, undrainedEventsOf, inUnitMovesToAudit, staleCarryRevertRow, staleCarryReverts, withRecorded, systemSwitchedOffMessage, tenantSystemSwitchedOffMessage, CrossVerticalRegistry, importCursorSourceOf, exportBreaksOf, exportBreakRefusal, bindExportBreaksOf, bindExportBreakRefusal, collectPeers, peerSeats, connectorCallRecord, noopConnectorCallRecorder, recordConnectorCall, settleConnectionUse, splitManifestMigrations, unknownRoleError, assertRowLimit, assertRowOffset, INERT_SCOPE_REASON, lifecycleReceipt, lifecycleRefusal, isPrimaryScope, isPrimaryScopeRow, asyncInvocationId, asyncLinePass, } from '@substrat-run/kernel';
|
|
5
6
|
import { attributedHost } from '@substrat-run/kernel';
|
|
6
7
|
import { isOrangeToOrange, isUpgradeRequest, LIVE_MODE_HEADER, LIVE_PRINCIPAL_HEADER, LIVE_SCOPE_HEADER, LIVE_SUBSCRIBE_PATH, LIVE_TENANT_HEADER, } from './live-reads.js';
|
|
7
8
|
import { tenantStoreDatabaseName } from './d1.js';
|
|
@@ -30,12 +31,29 @@ function unwrapCapabilityReply(reply) {
|
|
|
30
31
|
throw fromWireFailure(reply.failure);
|
|
31
32
|
return reply.value;
|
|
32
33
|
}
|
|
34
|
+
/**
|
|
35
|
+
* How far ahead of the clock a held lifecycle epoch may be and still be learned past (#1713). An
|
|
36
|
+
* epoch is the time a directory restore minted it, so a legitimate one is behind its minter's
|
|
37
|
+
* clock; the skew allows for clocks that disagree, and bounds what a forged answer can move.
|
|
38
|
+
*/
|
|
39
|
+
export const LIFECYCLE_EPOCH_SKEW_MS = 24 * 60 * 60 * 1000;
|
|
40
|
+
/**
|
|
41
|
+
* Refuse to mark a scope a copy unless the request classifies it non-primary (#2005). The
|
|
42
|
+
* classification is the platform directory's; the vertical applies the same `isPrimaryScope` to
|
|
43
|
+
* it, so a primary can be marked only by a request that misstates its lineage, never by default.
|
|
44
|
+
*/
|
|
45
|
+
function assertCopyLineage(lineage) {
|
|
46
|
+
if (isPrimaryScope(lineage)) {
|
|
47
|
+
throw substratError('conflict', 'mark-copy refused: the directory classifies this scope as primary, not as a copy');
|
|
48
|
+
}
|
|
49
|
+
}
|
|
33
50
|
/**
|
|
34
51
|
* A control-plane stand-in for a CP-less vertical (scope-local-permissions.md Phase 3).
|
|
35
52
|
* The hot path a served scope actually touches becomes trust-the-upstream:
|
|
36
|
-
* - `scopeAccessRefusal` / `setMigrationState` → no-op: the router
|
|
37
|
-
*
|
|
38
|
-
*
|
|
53
|
+
* - `scopeAccessRefusal` / `setMigrationState` → no-op: the router gates the scope's
|
|
54
|
+
* lifecycle + tenancy from the shared directory, and the vertical trusts the asserted
|
|
55
|
+
* node rather than re-reading a directory it does not have. Its own half of the
|
|
56
|
+
* lifecycle gate reads the scope's storage instead (`assertLive`, #1713).
|
|
39
57
|
* - `tenantHoldsEntitlement` → true: the SKU was enforced on the shared control plane
|
|
40
58
|
* at provision (before `provisionInstance`), so a scope that EXISTS here was granted
|
|
41
59
|
* it upstream — a single-vertical deployment holds its own entitlements by construction.
|
|
@@ -106,6 +124,14 @@ export const SWITCH_HOLD_EXTRA_WAITS = 3;
|
|
|
106
124
|
* clock, against the releasing host's clock: minutes, not seconds.
|
|
107
125
|
*/
|
|
108
126
|
export const SWITCH_HOLD_PENDING_MAX_MS = 5 * 60_000;
|
|
127
|
+
/**
|
|
128
|
+
* The attachment bucket a host resolves when it was handed no resolver (#1995): the
|
|
129
|
+
* platform's per-tenant binding, read off this script's own env, named by the one shared
|
|
130
|
+
* encoding (`blobStoreBindingName`) for the tenant the caller passed in.
|
|
131
|
+
*/
|
|
132
|
+
function ambientAttachmentBucket(tenantId) {
|
|
133
|
+
return ambientEnv[blobStoreBindingName(ATTACHMENT_BLOB_BINDING, tenantId)] ?? null;
|
|
134
|
+
}
|
|
109
135
|
export class CloudflareScopeHost {
|
|
110
136
|
admin;
|
|
111
137
|
scopeNs;
|
|
@@ -145,7 +171,7 @@ export class CloudflareScopeHost {
|
|
|
145
171
|
tenantStores;
|
|
146
172
|
/** The live R2 client for per-tenant blob stores (#473); undefined ⇒ refuse loudly. */
|
|
147
173
|
blobStores;
|
|
148
|
-
/** Worker-side attachment-bucket resolver (#473);
|
|
174
|
+
/** Worker-side attachment-bucket resolver (#473); the ambient per-tenant binding unless overridden (#1995). */
|
|
149
175
|
attachmentBuckets;
|
|
150
176
|
/** The parsers attachment text is extracted with (K-43); the host's own, never imported here. */
|
|
151
177
|
attachmentExtractors;
|
|
@@ -180,6 +206,8 @@ export class CloudflareScopeHost {
|
|
|
180
206
|
eventDrainDelegation;
|
|
181
207
|
/** #1666: the schedule kill switch's reach into the deployment serving a scope. */
|
|
182
208
|
systemSwitchDelegation;
|
|
209
|
+
/** #1713: the lifecycle's reach into the deployment serving a scope. */
|
|
210
|
+
lifecycleDelegation;
|
|
183
211
|
/** #1706: the peer kill switch's reach into the deployment serving a scope. */
|
|
184
212
|
peerSwitchDelegation;
|
|
185
213
|
/** #1705 PR 3: the replay lever's reach into the deployment serving a consumer scope. */
|
|
@@ -211,7 +239,7 @@ export class CloudflareScopeHost {
|
|
|
211
239
|
this.secretBox = options.secretBox ?? unconfiguredSecretBox;
|
|
212
240
|
this.tenantStores = options.tenantStores;
|
|
213
241
|
this.blobStores = options.blobStores;
|
|
214
|
-
this.attachmentBuckets = options.attachmentBuckets;
|
|
242
|
+
this.attachmentBuckets = options.attachmentBuckets ?? ambientAttachmentBucket;
|
|
215
243
|
assertAttachmentExtractors(options.attachmentExtractors ?? []);
|
|
216
244
|
this.attachmentExtractors = options.attachmentExtractors ?? [];
|
|
217
245
|
this.fetchImpl = options.fetch ?? globalFetch;
|
|
@@ -225,6 +253,7 @@ export class CloudflareScopeHost {
|
|
|
225
253
|
this.connectorDelegation = options.connectorDelegation;
|
|
226
254
|
this.eventDrainDelegation = options.eventDrainDelegation;
|
|
227
255
|
this.systemSwitchDelegation = options.systemSwitchDelegation;
|
|
256
|
+
this.lifecycleDelegation = options.lifecycleDelegation;
|
|
228
257
|
this.peerSwitchDelegation = options.peerSwitchDelegation;
|
|
229
258
|
this.importCursorDelegation = options.importCursorDelegation;
|
|
230
259
|
this.admin = this.buildAdmin();
|
|
@@ -353,21 +382,78 @@ export class CloudflareScopeHost {
|
|
|
353
382
|
};
|
|
354
383
|
if (this.executors.size === 0)
|
|
355
384
|
return report;
|
|
385
|
+
// #1901: one line per attempt, capped per pass. Written on the coordinator, where the
|
|
386
|
+
// attempt runs — the DO only journals it.
|
|
387
|
+
const lines = asyncLinePass();
|
|
388
|
+
try {
|
|
389
|
+
await this.drainExecutorsPass(tenantId, scopeId, invocationId, report, lines);
|
|
390
|
+
}
|
|
391
|
+
finally {
|
|
392
|
+
lines.end();
|
|
393
|
+
}
|
|
394
|
+
return report;
|
|
395
|
+
}
|
|
396
|
+
async drainExecutorsPass(tenantId, scopeId, invocationId, report, lines) {
|
|
356
397
|
const stub = this.scopeStub(scopeId);
|
|
357
|
-
|
|
398
|
+
// #2005: a non-primary scope (a fork, a snapshot, a preview of either kind) causes no
|
|
399
|
+
// outbound effects, so its deliveries are journaled terminal with the reason and no
|
|
400
|
+
// handler runs. Asked of the directory where there is one. A CP-less host has none, so it
|
|
401
|
+
// asks the scope's own storage whether it is classified a copy (`_substrat_copy_origin`'s
|
|
402
|
+
// `is_copy`, #2009, set on the directory's word) — a preview and a snapshot reach a hosted
|
|
403
|
+
// vertical as restores the platform marks.
|
|
404
|
+
// Asked on the first due event, once per pass: most passes have none, and the directory
|
|
405
|
+
// is one global object.
|
|
406
|
+
let inert;
|
|
407
|
+
const isInert = () => (inert ??= this.cpLess
|
|
408
|
+
? stub.isCopy()
|
|
409
|
+
: this.cp.getScopeRecord(tenantId, scopeId).then((row) => !isPrimaryScopeRow(row)));
|
|
410
|
+
// #1713: a CP-less scope its lifecycle holds attempts nothing and journals nothing, so every
|
|
411
|
+
// due delivery stays due for the first pass after it is live again. Asked on the first due
|
|
412
|
+
// event, like `isInert`. Every caller passed `assertLive` already, so this is for the scope
|
|
413
|
+
// suspended between that gate and the drain: a request in flight, whose write commits while
|
|
414
|
+
// its effect waits. A host with a directory answers false without a read.
|
|
415
|
+
let held;
|
|
416
|
+
const isHeld = () => (held ??= this.lifecycleHeld(scopeId));
|
|
417
|
+
drain: for (const [id, executor] of this.executors) {
|
|
358
418
|
const deliveryId = `executor:${id}`;
|
|
359
419
|
const { events, undecodable } = await stub.pendingExecutorDeliveries(deliveryId, executor.eventType);
|
|
360
420
|
// #1636: an event the DO could not decode is dead-lettered for this executor at once,
|
|
361
421
|
// and its handler never sees it. Terminal on the FIRST failure, unlike a handler's:
|
|
362
422
|
// the decode is pure, so a retry cannot succeed. The rows behind it are delivered
|
|
363
423
|
// below — the decode used to throw the whole list, on every pass.
|
|
424
|
+
if ((undecodable.length > 0 || events.length > 0) && (await isHeld())) {
|
|
425
|
+
report.lifecycleHeld = true;
|
|
426
|
+
break drain;
|
|
427
|
+
}
|
|
428
|
+
// #1901: an attempt's line. The attempt number is the one the journal is about to record.
|
|
429
|
+
const unitOf = (eventId, attempt) => ({
|
|
430
|
+
kind: 'consumer',
|
|
431
|
+
tenantId,
|
|
432
|
+
scopeId,
|
|
433
|
+
invocationId: asyncInvocationId(invocationId),
|
|
434
|
+
operation: deliveryId,
|
|
435
|
+
eventType: executor.eventType,
|
|
436
|
+
eventId,
|
|
437
|
+
attempt,
|
|
438
|
+
startedAt: Date.now(),
|
|
439
|
+
});
|
|
364
440
|
for (const bad of undecodable) {
|
|
365
441
|
report.attempted += 1;
|
|
366
|
-
await stub.recordExecutorAttempt(bad.eventId, deliveryId, bad.error, null, invocationId);
|
|
442
|
+
const attempt = await stub.recordExecutorAttempt(bad.eventId, deliveryId, bad.error, null, invocationId);
|
|
367
443
|
report.deadLettered += 1;
|
|
444
|
+
lines.write({ ...unitOf(bad.eventId, attempt), outcome: 'dead-lettered' });
|
|
368
445
|
}
|
|
369
446
|
for (const event of events) {
|
|
370
447
|
report.attempted += 1;
|
|
448
|
+
const startedAt = Date.now();
|
|
449
|
+
if (await isInert()) {
|
|
450
|
+
// No next attempt, like an undecodable row: a scope does not become primary.
|
|
451
|
+
const attempt = await stub.recordExecutorAttempt(event.id, deliveryId, INERT_SCOPE_REASON, null, invocationId);
|
|
452
|
+
report.inert = (report.inert ?? 0) + 1;
|
|
453
|
+
// Its own outcome, never `delivered`: no handler ran (#2005).
|
|
454
|
+
lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'inert' });
|
|
455
|
+
continue;
|
|
456
|
+
}
|
|
371
457
|
this.causedBy = event.id;
|
|
372
458
|
try {
|
|
373
459
|
if (executor.kind === 'connector' && this.cpLess) {
|
|
@@ -381,16 +467,20 @@ export class CloudflareScopeHost {
|
|
|
381
467
|
// the at-least-once residue, as it already must in-process.
|
|
382
468
|
await stub.routeExecutorEventToPlatform(event.id, deliveryId, connectorDispatchKind(executor.provider), JSON.stringify({ executorId: id, event }), JSON.stringify({ system: 'connector-dispatch' }), invocationId);
|
|
383
469
|
report.routedToPlatform += 1;
|
|
470
|
+
// Handed on, not run here: the platform's drain owns its attempts from now on.
|
|
471
|
+
lines.write({ ...unitOf(event.id, 1), startedAt, outcome: 'routed' });
|
|
384
472
|
}
|
|
385
473
|
else if (executor.kind === 'connector') {
|
|
386
474
|
await executor.handler(await this.connectorContext(tenantId, scopeId, executor.timeoutMs, event.id), event);
|
|
387
|
-
await stub.recordExecutorAttempt(event.id, deliveryId, null, null, invocationId);
|
|
475
|
+
const attempt = await stub.recordExecutorAttempt(event.id, deliveryId, null, null, invocationId);
|
|
388
476
|
report.delivered += 1;
|
|
477
|
+
lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'delivered' });
|
|
389
478
|
}
|
|
390
479
|
else {
|
|
391
480
|
await executor.handler(this.admin, event);
|
|
392
|
-
await stub.recordExecutorAttempt(event.id, deliveryId, null, null, invocationId);
|
|
481
|
+
const attempt = await stub.recordExecutorAttempt(event.id, deliveryId, null, null, invocationId);
|
|
393
482
|
report.delivered += 1;
|
|
483
|
+
lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'delivered' });
|
|
394
484
|
}
|
|
395
485
|
}
|
|
396
486
|
catch (err) {
|
|
@@ -405,18 +495,23 @@ export class CloudflareScopeHost {
|
|
|
405
495
|
report.deadLettered += 1;
|
|
406
496
|
else
|
|
407
497
|
report.retrying += 1;
|
|
498
|
+
lines.write({
|
|
499
|
+
...unitOf(event.id, attempts),
|
|
500
|
+
startedAt,
|
|
501
|
+
outcome: exhausted ? 'dead-lettered' : 'retrying',
|
|
502
|
+
error: err,
|
|
503
|
+
});
|
|
408
504
|
}
|
|
409
505
|
finally {
|
|
410
506
|
this.causedBy = null;
|
|
411
507
|
}
|
|
412
508
|
}
|
|
413
509
|
}
|
|
414
|
-
return report;
|
|
415
510
|
}
|
|
416
511
|
async drainDue(tenantId, scopeId) {
|
|
417
512
|
// Same lifecycle gate `getScope` applies (K-3): a suspended or archived scope
|
|
418
513
|
// does not get its effects driven either.
|
|
419
|
-
await this.
|
|
514
|
+
await this.assertLive(tenantId, scopeId);
|
|
420
515
|
await this.migrateAndRecord(scopeId);
|
|
421
516
|
// #1525: null, and honestly so — a sweep is not a call. An attempt this pass makes
|
|
422
517
|
// records no invocation, which is what distinguishes it from the first attempt the
|
|
@@ -474,14 +569,14 @@ export class CloudflareScopeHost {
|
|
|
474
569
|
this.attachmentExtractors);
|
|
475
570
|
}
|
|
476
571
|
async startJobRun(tenantId, scopeId, input) {
|
|
477
|
-
await this.
|
|
572
|
+
await this.assertLive(tenantId, scopeId);
|
|
478
573
|
await this.migrateAndRecord(scopeId);
|
|
479
574
|
return jobRunOf(await startJobRun(this.jobStore(scopeId), input, ulid, () => new Date().toISOString()));
|
|
480
575
|
}
|
|
481
576
|
async runDueJobs(tenantId, scopeId, options) {
|
|
482
577
|
// Same lifecycle gate as `drainDue`: a suspended scope's runs wait rather than
|
|
483
578
|
// advance, and an archived one's never move again.
|
|
484
|
-
await this.
|
|
579
|
+
await this.assertLive(tenantId, scopeId);
|
|
485
580
|
await this.migrateAndRecord(scopeId);
|
|
486
581
|
// #1575: the kernel's extraction job is this host's own, bound to this scope — no
|
|
487
582
|
// deployment registers it, and none can shadow it.
|
|
@@ -506,7 +601,7 @@ export class CloudflareScopeHost {
|
|
|
506
601
|
// same context build as the in-process path, so the handler cannot tell which host
|
|
507
602
|
// ran it. On a CP-less host `connectorContext` throws from the null control plane:
|
|
508
603
|
// fail closed, exactly the hole routing exists to avoid.
|
|
509
|
-
await this.
|
|
604
|
+
await this.assertLive(tenantId, scopeId);
|
|
510
605
|
this.causedBy = event.id;
|
|
511
606
|
try {
|
|
512
607
|
await handler(await this.connectorContext(tenantId, scopeId, options?.timeoutMs ?? 30_000, event.id), event);
|
|
@@ -653,7 +748,9 @@ export class CloudflareScopeHost {
|
|
|
653
748
|
*/
|
|
654
749
|
async snapshotScopeLocal(sourceScopeId, destScopeId) {
|
|
655
750
|
const tables = await this.scopeStub(sourceScopeId).exportDump();
|
|
656
|
-
|
|
751
|
+
// #2009: a snapshot is a fork by construction (the control plane's row names `forkedFrom`, so
|
|
752
|
+
// the directory never calls it primary), and a load classifies nothing by itself: marked here.
|
|
753
|
+
await this.scopeStub(destScopeId).importDump(tables, destScopeId, { sourceScopeId, exact: true, markCopy: true });
|
|
657
754
|
return { tables: tables.length };
|
|
658
755
|
}
|
|
659
756
|
/**
|
|
@@ -666,14 +763,131 @@ export class CloudflareScopeHost {
|
|
|
666
763
|
/** #1742: the directory's recorded-off modules, switched off on THIS scope in the replay's
|
|
667
764
|
* own event — a dump from before a switch was pulled brings its grants back live. #1869:
|
|
668
765
|
* `sourceScopeId`, the scope the dump was captured from, narrows the grant re-point to it;
|
|
669
|
-
* `exact` says the platform exported the dump itself, so the re-point never falls back.
|
|
766
|
+
* `exact` says the platform exported the dump itself, so the re-point never falls back.
|
|
767
|
+
* #1722: `loadStamp`, the stamp a carry leaves on the copy it lands, for a fenced wipe later,
|
|
768
|
+
* and `expect`, the marker the carry read here: the load is refused if the store moved since.
|
|
769
|
+
* #2005: `markCopy` — the directory's classification of the scope, sent when it is not
|
|
770
|
+
* primary; the restore then marks it a copy. Refused if it classifies a primary. */
|
|
670
771
|
opts) {
|
|
671
|
-
|
|
772
|
+
if (opts?.markCopy)
|
|
773
|
+
assertCopyLineage(opts.markCopy);
|
|
774
|
+
const load = {
|
|
672
775
|
switchOff: opts?.switchedOff ? { moduleIds: opts.switchedOff, at: new Date().toISOString() } : undefined,
|
|
673
776
|
sourceScopeId: opts?.sourceScopeId,
|
|
674
777
|
exact: opts?.exact,
|
|
778
|
+
loadStamp: opts?.loadStamp,
|
|
779
|
+
markCopy: opts?.markCopy !== undefined,
|
|
780
|
+
};
|
|
781
|
+
// Both refusals come back as values (a throw over the RPC carries only its message) and are
|
|
782
|
+
// thrown here, in the vertical's own isolate, so its route answers 409 or 412, not a fault.
|
|
783
|
+
const out = await this.scopeStub(scopeId).importDumpChecked(tables, scopeId, {
|
|
784
|
+
...load,
|
|
785
|
+
...(opts?.expect ? { expect: opts.expect } : {}),
|
|
786
|
+
});
|
|
787
|
+
if (out.refused === false) {
|
|
788
|
+
return { tables: tables.length, ...(opts?.switchedOff ? { switchedOff: out.switchedOff } : {}) };
|
|
789
|
+
}
|
|
790
|
+
if (out.refused === 'kept')
|
|
791
|
+
throw substratError('conflict', KEPT_COPY_REFUSAL);
|
|
792
|
+
// A retry of a load that already committed (its answer was lost on the way back) is
|
|
793
|
+
// refused by the marker that load itself moved. The store holding THIS request's stamp
|
|
794
|
+
// says so: no other load writes it. Answered as applied, without `switchedOff`, so the
|
|
795
|
+
// caller's re-assert after the bind covers the OFF positions.
|
|
796
|
+
const now = await this.scopeStub(scopeId).loadMarker();
|
|
797
|
+
if (!opts?.loadStamp || now.loadStamp !== opts.loadStamp) {
|
|
798
|
+
throw substratError('precondition_failed', 'scope store changed since the carry read it; nothing was loaded (#1722)');
|
|
799
|
+
}
|
|
800
|
+
return { tables: tables.length };
|
|
801
|
+
}
|
|
802
|
+
/** The kept-copy marker of this scope's store in THIS deployment (#1722), or null. */
|
|
803
|
+
async keptCopyLocal(scopeId) {
|
|
804
|
+
return this.scopeStub(scopeId).keptCopy();
|
|
805
|
+
}
|
|
806
|
+
/**
|
|
807
|
+
* Discard the kept copy of a scope in THIS deployment (#1722), behind the vertical's
|
|
808
|
+
* `/internal/kept-copy/discard`: the staff resolution, only at the write revision the operator
|
|
809
|
+
* acted on. Answers whether it was discarded, or why not.
|
|
810
|
+
*/
|
|
811
|
+
async discardKeptCopyLocal(scopeId, revision, carriedAway,
|
|
812
|
+
/** #2005: the directory's classification, sent when the scope is not primary. */
|
|
813
|
+
markCopy,
|
|
814
|
+
/** #1722 (Codex #2008 r13): the load stamp read with `revision`; a replaced store is refused. */
|
|
815
|
+
loadStamp) {
|
|
816
|
+
if (markCopy)
|
|
817
|
+
assertCopyLineage(markCopy);
|
|
818
|
+
return this.scopeStub(scopeId).discardKeptCopy(scopeId, revision, carriedAway, markCopy !== undefined, ...(loadStamp !== undefined ? [loadStamp] : []));
|
|
819
|
+
}
|
|
820
|
+
/**
|
|
821
|
+
* What a carry's restore into this scope's store expects to find unchanged (#1722), behind
|
|
822
|
+
* the vertical's `/internal/load-marker`. Read before the carry checks the binding again; the
|
|
823
|
+
* restore then carries it as `expect`, and is refused if the store was loaded or written since.
|
|
824
|
+
*/
|
|
825
|
+
async loadMarkerLocal(scopeId) {
|
|
826
|
+
return this.scopeStub(scopeId).loadMarker();
|
|
827
|
+
}
|
|
828
|
+
/**
|
|
829
|
+
* Wipe the copy a carry left in THIS deployment (#1722), behind the vertical's
|
|
830
|
+
* `/internal/wipe-carried`: only if nothing was loaded into the scope's DO since the stamp
|
|
831
|
+
* the carry read (`expectLoadStamp`, null for a store no load has stamped). Non-terminal,
|
|
832
|
+
* so a later bind back to this version can restore into it. False when refused.
|
|
833
|
+
*/
|
|
834
|
+
async wipeCarriedLocal(scopeId, expectLoadStamp, carriedAway, opts = {}) {
|
|
835
|
+
if (opts.markCopy)
|
|
836
|
+
assertCopyLineage(opts.markCopy);
|
|
837
|
+
return this.scopeStub(scopeId).wipeCarried(scopeId, expectLoadStamp, carriedAway, {
|
|
838
|
+
...(opts.expectRevision !== undefined ? { expectRevision: opts.expectRevision } : {}),
|
|
839
|
+
...(opts.protectIfChanged !== undefined ? { protectIfChanged: opts.protectIfChanged } : {}),
|
|
840
|
+
markCopy: opts.markCopy !== undefined,
|
|
675
841
|
});
|
|
676
|
-
|
|
842
|
+
}
|
|
843
|
+
/** Release a kept copy that is the live store after all (#1722), at the revision read. */
|
|
844
|
+
async releaseKeptCopyLocal(scopeId, revision,
|
|
845
|
+
/** #2005: the directory's classification, sent when the scope is not primary. */
|
|
846
|
+
markCopy,
|
|
847
|
+
/** #1722 (Codex #2008 r13): the load stamp read with `revision`; a replaced store is refused. */
|
|
848
|
+
loadStamp) {
|
|
849
|
+
if (markCopy)
|
|
850
|
+
assertCopyLineage(markCopy);
|
|
851
|
+
return this.scopeStub(scopeId).releaseKeptCopy(revision, markCopy !== undefined, ...(loadStamp !== undefined ? [loadStamp] : []));
|
|
852
|
+
}
|
|
853
|
+
/**
|
|
854
|
+
* Store the lifecycle the platform delivered for one scope in THIS deployment (#1713), behind
|
|
855
|
+
* the vertical's `/internal/lifecycle`. Kept unless the scope already holds a newer one, so a
|
|
856
|
+
* push that arrives late cannot undo a later transition. `applied` is what the platform's heal
|
|
857
|
+
* sweep records as delivered; `changed` says the gate's answer moved.
|
|
858
|
+
*/
|
|
859
|
+
async setLifecycleLocal(scopeId, next) {
|
|
860
|
+
return lifecycleDelivery.parse(await this.scopeStub(scopeId).setLifecycle(scopeLifecycle.parse(next)));
|
|
861
|
+
}
|
|
862
|
+
/**
|
|
863
|
+
* Mark one scope in THIS deployment a copy (#2005), behind the vertical's `/internal/mark-copy`:
|
|
864
|
+
* the repair of a copy that predates the marker, which a CP-less coordinator reads for primacy.
|
|
865
|
+
* The control plane decides which scopes (its directory says they are not primary) and audits.
|
|
866
|
+
*/
|
|
867
|
+
async markCopyLocal(scopeId, lineage) {
|
|
868
|
+
assertCopyLineage(lineage);
|
|
869
|
+
return { marked: await this.scopeStub(scopeId).markCopy() };
|
|
870
|
+
}
|
|
871
|
+
/**
|
|
872
|
+
* Clear a mistaken copy classification from one scope in THIS deployment (#2005), behind the
|
|
873
|
+
* vertical's `/internal/clear-copy-mark`: staff's correction for a scope the directory says IS
|
|
874
|
+
* primary. Refused for a scope classified a copy. A load's copied-events mark is never touched
|
|
875
|
+
* (#2009), so a primary restored from another scope's backup runs its own effects again and
|
|
876
|
+
* still never runs the work it copied.
|
|
877
|
+
*/
|
|
878
|
+
async clearCopyMarkLocal(scopeId, lineage,
|
|
879
|
+
/** #1722 (Codex #2008 r12–r13): the store as the caller means it, its load stamp and write
|
|
880
|
+
* revision, compared in the clear's own transaction; refused (412) if either moved. The
|
|
881
|
+
* platform's reconcile of a carry's destination sends it; staff send none. */
|
|
882
|
+
expect) {
|
|
883
|
+
if (!isPrimaryScope(lineage)) {
|
|
884
|
+
throw substratError('conflict', 'clear-copy-mark refused: the directory classifies this scope as a copy (a preview or a fork)');
|
|
885
|
+
}
|
|
886
|
+
const outcome = await this.scopeStub(scopeId).clearCopyMark(...(expect ? [expect] : []));
|
|
887
|
+
if (outcome === 'changed') {
|
|
888
|
+
throw substratError('precondition_failed', `clear-copy-mark refused: scope ${scopeId}'s store changed since its revision was read`);
|
|
889
|
+
}
|
|
890
|
+
return { cleared: outcome === 'cleared' };
|
|
677
891
|
}
|
|
678
892
|
/**
|
|
679
893
|
* Re-apply the vertical's OWN role definitions to one scope, CP-lessly — the
|
|
@@ -710,6 +924,13 @@ export class CloudflareScopeHost {
|
|
|
710
924
|
async exportScopeLocal(scopeId) {
|
|
711
925
|
return this.scopeStub(scopeId).exportDump();
|
|
712
926
|
}
|
|
927
|
+
/**
|
|
928
|
+
* `exportScopeLocal` with the store's load stamp, read in the same DO call (#1722): what a
|
|
929
|
+
* carry's fenced wipe of the copy it leaves here expects. Behind the vertical's `/internal/export`.
|
|
930
|
+
*/
|
|
931
|
+
async exportScopeStampedLocal(scopeId) {
|
|
932
|
+
return this.scopeStub(scopeId).exportDumpStamped();
|
|
933
|
+
}
|
|
713
934
|
/** Facet this host's own scope's outbox (#1239) — the vertical-host read. */
|
|
714
935
|
async facetEventsLocal(scopeId, input) {
|
|
715
936
|
return this.scopeStub(scopeId).facetEvents(input);
|
|
@@ -1065,7 +1286,7 @@ export class CloudflareScopeHost {
|
|
|
1065
1286
|
// gates and metadata facts live in the ScopeDO (per-scope serialization, spine event
|
|
1066
1287
|
// in the same transaction); bytes go straight to the per-tenant R2 bucket through the
|
|
1067
1288
|
// binding the vertical's worker resolved — never through the DO.
|
|
1068
|
-
await this.
|
|
1289
|
+
await this.assertLive(tenantId, scopeId);
|
|
1069
1290
|
await this.migrateAndRecord(scopeId);
|
|
1070
1291
|
const store = await this.resolveAttachmentStore(tenantId);
|
|
1071
1292
|
return this.buildAttachmentSurface({ principal }, tenantId, scopeId, store);
|
|
@@ -1138,17 +1359,14 @@ export class CloudflareScopeHost {
|
|
|
1138
1359
|
const store = await this.resolveAttachmentStore(conn.tenant_id);
|
|
1139
1360
|
return this.buildAttachmentSurface({ connectionId }, conn.tenant_id, scopeId, store, forEvent);
|
|
1140
1361
|
}
|
|
1141
|
-
/** Resolve the per-tenant R2 blob store, or fail closed
|
|
1362
|
+
/** Resolve the per-tenant R2 blob store, or fail closed. */
|
|
1142
1363
|
async resolveAttachmentStore(tenantId) {
|
|
1143
|
-
if (!this.attachmentBuckets) {
|
|
1144
|
-
throw new Error(`attachments are not configured on this host (#473): pass ` +
|
|
1145
|
-
`CloudflareScopeHostOptions.attachmentBuckets — (tenantId) => ` +
|
|
1146
|
-
`env[blobStoreBindingName('<BINDING>', tenantId)] for the vertical's declared blob store`);
|
|
1147
|
-
}
|
|
1148
1364
|
const bucket = await this.attachmentBuckets(tenantId);
|
|
1149
1365
|
if (!bucket) {
|
|
1150
|
-
throw new Error(`no attachment bucket resolved for tenant ${tenantId} (#473) —
|
|
1151
|
-
|
|
1366
|
+
throw new Error(`no attachment bucket resolved for tenant ${tenantId} (#473, #1995) — this script has no ` +
|
|
1367
|
+
`${blobStoreBindingName(ATTACHMENT_BLOB_BINDING, tenantId)} binding. A vertical whose modules declare ` +
|
|
1368
|
+
`attachmentTargets gets one per installed tenant once it is pushed with a CLI that declares the ` +
|
|
1369
|
+
`store (re-push); off the platform, pass CloudflareScopeHostOptions.attachmentBuckets.`);
|
|
1152
1370
|
}
|
|
1153
1371
|
return r2TenantBlobStore(bucket);
|
|
1154
1372
|
}
|
|
@@ -1381,7 +1599,7 @@ export class CloudflareScopeHost {
|
|
|
1381
1599
|
async getScope(principal, tenantId, scopeId, options) {
|
|
1382
1600
|
// Lifecycle gates (control-plane.md §4.1/§4.2), the K-3 fail-closed path,
|
|
1383
1601
|
// evaluated durably in the ControlPlaneDO. A throw propagates here.
|
|
1384
|
-
await this.
|
|
1602
|
+
await this.assertLive(tenantId, scopeId);
|
|
1385
1603
|
await this.migrateAndRecord(scopeId);
|
|
1386
1604
|
return this.buildStub(tenantId, scopeId, principal, undefined, undefined, options);
|
|
1387
1605
|
}
|
|
@@ -1456,7 +1674,7 @@ export class CloudflareScopeHost {
|
|
|
1456
1674
|
const record = await this.resolveImpersonation(session, tenantId, scopeId);
|
|
1457
1675
|
// The same lifecycle gate the principal door applies: a suspended tenant
|
|
1458
1676
|
// refuses a support session exactly as it refuses a user.
|
|
1459
|
-
await this.
|
|
1677
|
+
await this.assertLive(tenantId, scopeId);
|
|
1460
1678
|
await this.migrateAndRecord(scopeId);
|
|
1461
1679
|
return this.buildStub(tenantId, scopeId, record.principal, undefined, undefined, options, record.id);
|
|
1462
1680
|
}
|
|
@@ -1470,7 +1688,7 @@ export class CloudflareScopeHost {
|
|
|
1470
1688
|
if (!this.moduleIds.has(moduleId)) {
|
|
1471
1689
|
throw new Error(`module not registered on this host: ${moduleId}`);
|
|
1472
1690
|
}
|
|
1473
|
-
await this.
|
|
1691
|
+
await this.assertLive(tenantId, scopeId);
|
|
1474
1692
|
await this.migrateAndRecord(scopeId);
|
|
1475
1693
|
return this.buildStub(tenantId, scopeId, undefined, undefined, moduleId);
|
|
1476
1694
|
}
|
|
@@ -1503,7 +1721,7 @@ export class CloudflareScopeHost {
|
|
|
1503
1721
|
* capability row lives. The answer is re-parsed on this side of the RPC.
|
|
1504
1722
|
*/
|
|
1505
1723
|
async exchangeCapability(tenantId, scopeId, secret, options) {
|
|
1506
|
-
await this.
|
|
1724
|
+
await this.assertLive(tenantId, scopeId);
|
|
1507
1725
|
await this.migrateAndRecord(scopeId);
|
|
1508
1726
|
const outcome = await this.scopeStub(scopeId).exchangeCapability(secret, tenantId, scopeId, options?.mode);
|
|
1509
1727
|
return capabilityExchange.nullable().parse(outcome);
|
|
@@ -1517,7 +1735,7 @@ export class CloudflareScopeHost {
|
|
|
1517
1735
|
if (!plausibleSessionToken(sessionToken)) {
|
|
1518
1736
|
throw substratError('unauthenticated', 'not a capability session token');
|
|
1519
1737
|
}
|
|
1520
|
-
await this.
|
|
1738
|
+
await this.assertLive(tenantId, scopeId);
|
|
1521
1739
|
await this.migrateAndRecord(scopeId);
|
|
1522
1740
|
const hash = await capabilityTokenHash(sessionToken);
|
|
1523
1741
|
return this.buildStub(tenantId, scopeId, undefined, undefined, undefined, options, undefined, hash);
|
|
@@ -1593,7 +1811,7 @@ export class CloudflareScopeHost {
|
|
|
1593
1811
|
}
|
|
1594
1812
|
this.assertServedHere(record, scopeId, verb);
|
|
1595
1813
|
}
|
|
1596
|
-
await this.
|
|
1814
|
+
await this.assertLive(tenantId, scopeId);
|
|
1597
1815
|
await this.migrateAndRecord(scopeId);
|
|
1598
1816
|
}
|
|
1599
1817
|
/**
|
|
@@ -1606,7 +1824,7 @@ export class CloudflareScopeHost {
|
|
|
1606
1824
|
if (!plausibleSessionToken(sessionToken)) {
|
|
1607
1825
|
throw substratError('unauthenticated', 'not a capability session token');
|
|
1608
1826
|
}
|
|
1609
|
-
await this.
|
|
1827
|
+
await this.assertLive(tenantId, scopeId);
|
|
1610
1828
|
await this.migrateAndRecord(scopeId);
|
|
1611
1829
|
const store = await this.resolveAttachmentStore(tenantId);
|
|
1612
1830
|
const capabilitySession = await capabilityTokenHash(sessionToken);
|
|
@@ -1703,6 +1921,10 @@ export class CloudflareScopeHost {
|
|
|
1703
1921
|
if (!rec || rec.status !== 'active')
|
|
1704
1922
|
return report;
|
|
1705
1923
|
}
|
|
1924
|
+
else if (await this.lifecycleHeld(scopeId)) {
|
|
1925
|
+
// #1713: a held scope emits nothing, so judging it stale would only raise a false alarm.
|
|
1926
|
+
return report;
|
|
1927
|
+
}
|
|
1706
1928
|
const stub = this.scopeStub(scopeId);
|
|
1707
1929
|
const windows = new Map();
|
|
1708
1930
|
for (const specs of this.moduleFreshness.values()) {
|
|
@@ -1741,14 +1963,23 @@ export class CloudflareScopeHost {
|
|
|
1741
1963
|
return report;
|
|
1742
1964
|
// Only run on a live scope of this tenant; a scope archived between the sweep's
|
|
1743
1965
|
// enumeration and here simply has nothing due. A CP-less host has no directory
|
|
1744
|
-
// to ask (#461)
|
|
1745
|
-
// whole request path, and lifecycle for its scopes lives wherever provisioning
|
|
1746
|
-
// does, so the grant check below is the only gate it can and does enforce.
|
|
1966
|
+
// to ask (#461), so it reads the lifecycle the platform delivered to the scope (#1713).
|
|
1747
1967
|
if (!this.cpLess) {
|
|
1748
1968
|
const rec = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
1749
1969
|
if (!rec || rec.status !== 'active')
|
|
1750
1970
|
return report;
|
|
1751
1971
|
}
|
|
1972
|
+
else if (await this.lifecycleHeld(scopeId)) {
|
|
1973
|
+
// #1713: the lifecycle the platform delivered holds the scope. Every schedule is
|
|
1974
|
+
// `skipped` and no cadence row moves, as under the kill switch, so a schedule that
|
|
1975
|
+
// came due meanwhile fires once on the first pass after the scope is live again.
|
|
1976
|
+
for (const schedule of schedules) {
|
|
1977
|
+
report.skipped += 1;
|
|
1978
|
+
report.runs.push({ operation: schedule.operation, outcome: 'skipped' });
|
|
1979
|
+
}
|
|
1980
|
+
report.lifecycleHeld = true;
|
|
1981
|
+
return report;
|
|
1982
|
+
}
|
|
1752
1983
|
const stub = this.scopeStub(scopeId);
|
|
1753
1984
|
// The grant IS the switch (#383), and the kill switch is its lever (#1666): the
|
|
1754
1985
|
// kernel's `systemScheduleState`, the one predicate both adapters run. A scope that
|
|
@@ -1772,6 +2003,7 @@ export class CloudflareScopeHost {
|
|
|
1772
2003
|
return report;
|
|
1773
2004
|
}
|
|
1774
2005
|
const now = Date.now();
|
|
2006
|
+
const lines = asyncLinePass();
|
|
1775
2007
|
for (const schedule of schedules) {
|
|
1776
2008
|
const last = await stub.scheduleLastRun(schedule.operation);
|
|
1777
2009
|
const lastRun = last ? Date.parse(last) : null;
|
|
@@ -1786,24 +2018,43 @@ export class CloudflareScopeHost {
|
|
|
1786
2018
|
// produces either way carries it, and so does the row below.
|
|
1787
2019
|
const invocationId = ulid();
|
|
1788
2020
|
let status = 'ok';
|
|
2021
|
+
// #1901: the run's line, under the call's own id — the one its `ctx.log` lines carry.
|
|
2022
|
+
const startedAt = Date.now();
|
|
2023
|
+
let emitted;
|
|
2024
|
+
let failure;
|
|
1789
2025
|
try {
|
|
1790
2026
|
const scope = await this.getSystemScope(moduleId, tenantId, scopeId);
|
|
1791
|
-
await scope.invoke(schedule.operation, schedule.input, { invocationId });
|
|
2027
|
+
await scope.invoke(schedule.operation, schedule.input, { invocationId, onEmitted: (r) => (emitted = r) });
|
|
1792
2028
|
report.fired += 1;
|
|
1793
2029
|
}
|
|
1794
2030
|
catch (err) {
|
|
1795
2031
|
status = 'failed';
|
|
2032
|
+
failure = { error: err };
|
|
1796
2033
|
report.failed += 1;
|
|
1797
2034
|
report.errors.push({
|
|
1798
2035
|
operation: schedule.operation,
|
|
1799
2036
|
error: err instanceof Error ? err.message : String(err),
|
|
1800
2037
|
});
|
|
1801
2038
|
}
|
|
2039
|
+
lines.write({
|
|
2040
|
+
kind: 'schedule',
|
|
2041
|
+
tenantId,
|
|
2042
|
+
scopeId,
|
|
2043
|
+
invocationId,
|
|
2044
|
+
operation: schedule.operation,
|
|
2045
|
+
startedAt,
|
|
2046
|
+
outcome: status,
|
|
2047
|
+
...failure,
|
|
2048
|
+
dueAt: lastRun === null ? null : new Date(dueAt).toISOString(),
|
|
2049
|
+
latenessMs: lastRun === null ? null : now - dueAt,
|
|
2050
|
+
...(emitted ? { emitted } : {}),
|
|
2051
|
+
});
|
|
1802
2052
|
// #1288: 'schedule', whatever this operation happens to be called — including
|
|
1803
2053
|
// `freshness:<something>`, which is exactly the row the evaluator no longer eats.
|
|
1804
2054
|
await stub.recordScheduleRun(schedule.operation, new Date(now).toISOString(), status, 'schedule', invocationId);
|
|
1805
2055
|
report.runs.push({ operation: schedule.operation, outcome: status === 'ok' ? 'ok' : 'failed' });
|
|
1806
2056
|
}
|
|
2057
|
+
lines.end();
|
|
1807
2058
|
return report;
|
|
1808
2059
|
}
|
|
1809
2060
|
/** The stub body, shared by the principal, connection, system and impersonation doors. */
|
|
@@ -2125,6 +2376,8 @@ export class CloudflareScopeHost {
|
|
|
2125
2376
|
// returns it with the previous status, so the trail cannot disagree with
|
|
2126
2377
|
// the directory about which deployment the action touched.
|
|
2127
2378
|
await this.recordAdmin(actor, action, { tenantId, scopeId, vertical: before.vertical }, { status: before.status }, { status: to, ...afterExtra });
|
|
2379
|
+
// #1713: after the directory moved and the move was audited, never before or instead.
|
|
2380
|
+
await this.deliverLifecycles(actor, { scopeId });
|
|
2128
2381
|
};
|
|
2129
2382
|
const writeGrant = async (subject, permission, node, entity, expiresAt) => {
|
|
2130
2383
|
if (entity) {
|
|
@@ -3126,7 +3379,7 @@ export class CloudflareScopeHost {
|
|
|
3126
3379
|
if (channel === 'prod' && !incoming.deployment_ref) {
|
|
3127
3380
|
const owning = await this.cp.readVertical(verticalSlug);
|
|
3128
3381
|
if (owning && owning.owner_tenant !== null && !owning.listed) {
|
|
3129
|
-
const bound = (await this.cp.listScopes({ tenantId: owning.owner_tenant, vertical: verticalSlug, status: ['active'] })).filter(
|
|
3382
|
+
const bound = (await this.cp.listScopes({ tenantId: owning.owner_tenant, vertical: verticalSlug, status: ['active'] })).filter(isPrimaryScopeRow);
|
|
3130
3383
|
for (const s of bound) {
|
|
3131
3384
|
if (s.vertical_version_id === versionId)
|
|
3132
3385
|
continue;
|
|
@@ -3284,6 +3537,12 @@ export class CloudflareScopeHost {
|
|
|
3284
3537
|
await this.cp.setScopeServingRef(scopeId, servingRef);
|
|
3285
3538
|
await this.recordAdmin(actor, 'setScopeServingRef', { tenantId, scopeId }, { servingRef: scope.serving_ref ?? null }, { servingRef, ...(ack.exportBreak ? { acknowledged: ack } : {}) });
|
|
3286
3539
|
},
|
|
3540
|
+
recordKeptCopyResolution: async (actor, tenantId, scopeId, r) => {
|
|
3541
|
+
const scope = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
3542
|
+
if (!scope)
|
|
3543
|
+
throw substratError('not_found', `unknown scope ${scopeId} in tenant ${tenantId}`);
|
|
3544
|
+
await this.recordAdmin(actor, 'resolveKeptCopy', { tenantId, scopeId }, { script: r.script, keptAt: r.keptAt, revision: r.revisionBefore }, { action: r.action, liveScript: r.liveScript, revision: r.revisionAfter });
|
|
3545
|
+
},
|
|
3287
3546
|
setScopeExpiresAt: async (actor, tenantId, scopeId, expiresAt) => {
|
|
3288
3547
|
const scope = await this.cp.getScopeRecord(tenantId, scopeId);
|
|
3289
3548
|
if (!scope)
|
|
@@ -3388,6 +3647,8 @@ export class CloudflareScopeHost {
|
|
|
3388
3647
|
setTenantStatus: async (actor, tenantId, status) => {
|
|
3389
3648
|
const before = await this.cp.setTenantStatus(tenantId, status);
|
|
3390
3649
|
await this.recordAdmin(actor, 'setTenantStatus', { tenantId }, { status: before }, { status });
|
|
3650
|
+
// #1713: every hosted scope under the tenant holds or resumes its work by this.
|
|
3651
|
+
await this.deliverLifecycles(actor, { tenantId });
|
|
3391
3652
|
},
|
|
3392
3653
|
setTenantName: async (actor, tenantId, name) => {
|
|
3393
3654
|
const before = await this.cp.setTenantName(tenantId, name);
|
|
@@ -4260,6 +4521,11 @@ export class CloudflareScopeHost {
|
|
|
4260
4521
|
const { tenantId, scopeId, ...after } = ownerTransferAudit.parse(entry);
|
|
4261
4522
|
await this.recordAdmin(actor, 'transferOwner', { tenantId, scopeId }, null, after);
|
|
4262
4523
|
},
|
|
4524
|
+
/** #2005: one change to a scope's copy marker, written around the vertical's own change. */
|
|
4525
|
+
recordCopyMark: async (actor, entry) => {
|
|
4526
|
+
const { tenantId, scopeId, action, ...after } = copyMarkAudit.parse(entry);
|
|
4527
|
+
await this.recordAdmin(actor, action === 'mark' ? 'markScopeCopy' : 'clearScopeCopyMark', { tenantId, scopeId }, null, after);
|
|
4528
|
+
},
|
|
4263
4529
|
accessLog: async (actor, filter) => {
|
|
4264
4530
|
const rows = await this.cp.accessLog({
|
|
4265
4531
|
actor: filter?.actor,
|
|
@@ -4481,6 +4747,184 @@ export class CloudflareScopeHost {
|
|
|
4481
4747
|
return;
|
|
4482
4748
|
throw refusal.code ? substratError(refusal.code, refusal.message) : new Error(refusal.message);
|
|
4483
4749
|
}
|
|
4750
|
+
/**
|
|
4751
|
+
* THE lifecycle gate (#1713) at every door that runs a scope's work: a request's stubs and
|
|
4752
|
+
* attachments, a capability or impersonation session, a peer call or delivery, a subscription,
|
|
4753
|
+
* and the entry points no request asked for — the retry driver, the job runner, a schedule's
|
|
4754
|
+
* system stub, a routed connector delivery.
|
|
4755
|
+
*
|
|
4756
|
+
* `validateScopeAccess` first, which is the directory's gate where there is a directory. A
|
|
4757
|
+
* CP-less host has none, so it then reads the lifecycle the platform delivered to the scope's
|
|
4758
|
+
* own storage (`/internal/lifecycle`), judged by the kernel's `lifecycleRefusal` in the
|
|
4759
|
+
* directory's own words. The router refuses a suspended scope's requests first (#1730); this
|
|
4760
|
+
* is the deployment's own half, and the only one a timer or a retry ever meets.
|
|
4761
|
+
*
|
|
4762
|
+
* The admin and diagnostic reads (dead letters, job runs, platform requests, delivered
|
|
4763
|
+
* grants) keep `validateScopeAccess` alone: an operator still reads a suspended scope.
|
|
4764
|
+
*/
|
|
4765
|
+
async assertLive(tenantId, scopeId) {
|
|
4766
|
+
await this.validateScopeAccess(tenantId, scopeId);
|
|
4767
|
+
const refusal = await this.cpLessRefusal(scopeId, { tenantId, scopeId });
|
|
4768
|
+
if (refusal)
|
|
4769
|
+
throw new Error(refusal);
|
|
4770
|
+
}
|
|
4771
|
+
/** The delivered lifecycle's refusal on a CP-less host (#1713); always null with a directory. */
|
|
4772
|
+
async cpLessRefusal(scopeId, ids) {
|
|
4773
|
+
if (!this.cpLess)
|
|
4774
|
+
return null;
|
|
4775
|
+
return lifecycleRefusal(await this.scopeStub(scopeId).lifecycle(), ids);
|
|
4776
|
+
}
|
|
4777
|
+
/**
|
|
4778
|
+
* Deliver the directory's lifecycle to the deployments serving the matching scopes (#1713):
|
|
4779
|
+
* one scope after its transition, a tenant's scopes after the tenant's, or the heal sweep's
|
|
4780
|
+
* drift. Each delivery carries the directory's revisions, read in the same query as the
|
|
4781
|
+
* statuses, so a late one cannot undo a later transition: the deployment keeps only a delivery
|
|
4782
|
+
* strictly newer than the one it holds.
|
|
4783
|
+
*
|
|
4784
|
+
* Never throws. A transition is the operator's lever in an incident, and the router refuses
|
|
4785
|
+
* the scope's requests as soon as the directory moves (#1730), so a deployment that cannot be
|
|
4786
|
+
* reached must not refuse the lever. Each failure is an ops-failure row, no receipt is written,
|
|
4787
|
+
* and the heal sweep delivers again.
|
|
4788
|
+
*/
|
|
4789
|
+
async deliverLifecycles(actor, filter) {
|
|
4790
|
+
const report = { attempted: 0, delivered: 0, failed: 0 };
|
|
4791
|
+
const delegation = this.lifecycleDelegation;
|
|
4792
|
+
if (!delegation || this.cpLess)
|
|
4793
|
+
return report;
|
|
4794
|
+
let targets;
|
|
4795
|
+
try {
|
|
4796
|
+
targets = await this.cp.lifecycleTargets(filter);
|
|
4797
|
+
}
|
|
4798
|
+
catch (err) {
|
|
4799
|
+
console.error('substrat: could not read the scopes to deliver a lifecycle to (#1713)', err);
|
|
4800
|
+
return report;
|
|
4801
|
+
}
|
|
4802
|
+
for (const first of targets) {
|
|
4803
|
+
const tenantId = first.tenant_id;
|
|
4804
|
+
const scopeId = first.scope_id;
|
|
4805
|
+
let t = first;
|
|
4806
|
+
let lifecycle = this.lifecycleOfTarget(t);
|
|
4807
|
+
if (!lifecycle) {
|
|
4808
|
+
// A status this code does not know (a newer directory): delivering a guess could lift a hold.
|
|
4809
|
+
report.failed += 1;
|
|
4810
|
+
console.error(`substrat: scope ${scopeId} has a lifecycle this code cannot read (#1713)`);
|
|
4811
|
+
continue;
|
|
4812
|
+
}
|
|
4813
|
+
// A live scope whose deployment already acknowledged exactly this has nothing to receive,
|
|
4814
|
+
// and neither does one with no receipt in a directory never restored: a deployment holding
|
|
4815
|
+
// no lifecycle runs live. That keeps an activation, and every transition before a
|
|
4816
|
+
// deployment carries the route, from posting a delivery that changes nothing. After a
|
|
4817
|
+
// restore (epoch > 0) no receipt says nothing about what the deployment holds, so it is
|
|
4818
|
+
// delivered. A HOLD is always delivered.
|
|
4819
|
+
if (lifecycleRefusal(lifecycle) === null && (t.delivered === null ? t.epoch === 0 : t.delivered === lifecycleReceipt(lifecycle))) {
|
|
4820
|
+
continue;
|
|
4821
|
+
}
|
|
4822
|
+
report.attempted += 1;
|
|
4823
|
+
try {
|
|
4824
|
+
let answer = await delegation.deliver({ tenantId, scopeId, lifecycle });
|
|
4825
|
+
// SINGLE AUTHORITY (the invariant this rests on): an environment has exactly one directory,
|
|
4826
|
+
// the singleton `CONTROL_PLANE.idFromName('control-plane')`. A "fresh directory" restore is
|
|
4827
|
+
// that same object restored after its storage was lost, never a second live writer, and
|
|
4828
|
+
// the raise below always re-reads the CURRENT store and delivers what it says. Two control
|
|
4829
|
+
// planes healing one dispatch namespace at once is a split brain that corrupts far more
|
|
4830
|
+
// than lifecycle, and is out of scope here; fencing it in lifecycle alone would guarantee
|
|
4831
|
+
// nothing.
|
|
4832
|
+
//
|
|
4833
|
+
// The scope refused us as OLDER while holding a revision this directory's history did not
|
|
4834
|
+
// write: an epoch ahead of ours, or ours with other counters. Only another history of the
|
|
4835
|
+
// directory delivers those — the one a fresh-directory restore replaced, whose epoch a
|
|
4836
|
+
// clock running behind (or a restore in the same millisecond) cannot outrank. A scope only
|
|
4837
|
+
// ever holds epochs a directory minted, so this directory learns past it: its epoch is
|
|
4838
|
+
// raised above the one held (monotonic, one statement), and the scope is delivered again.
|
|
4839
|
+
const held = answer.lifecycle.revision;
|
|
4840
|
+
const foreign = !answer.applied &&
|
|
4841
|
+
held !== undefined &&
|
|
4842
|
+
(held.epoch > lifecycle.revision.epoch ||
|
|
4843
|
+
(held.epoch === lifecycle.revision.epoch && lifecycleReceipt(answer.lifecycle) !== lifecycleReceipt(lifecycle)));
|
|
4844
|
+
// Bounded, because the deployment answering is the vertical's own code: only a directory
|
|
4845
|
+
// that has been restored can meet another history (epoch > 0), and a legitimate epoch is
|
|
4846
|
+
// a mint time, so one further ahead than the skew is forged or broken. Either way nothing
|
|
4847
|
+
// is raised; the refusal is an ops failure for an operator to look at.
|
|
4848
|
+
if (foreign && (lifecycle.revision.epoch === 0 || held.epoch > Date.now() + LIFECYCLE_EPOCH_SKEW_MS)) {
|
|
4849
|
+
report.failed += 1;
|
|
4850
|
+
await this.admin
|
|
4851
|
+
.recordOpsFailure({
|
|
4852
|
+
actor,
|
|
4853
|
+
operation: 'scope.lifecycle',
|
|
4854
|
+
stage: 'foreign-epoch',
|
|
4855
|
+
tenantId,
|
|
4856
|
+
scopeId,
|
|
4857
|
+
message: `the deployment refused this directory's lifecycle while holding epoch ${held.epoch} ` +
|
|
4858
|
+
`(this directory is at ${lifecycle.revision.epoch}); not raised past it — ` +
|
|
4859
|
+
(lifecycle.revision.epoch === 0
|
|
4860
|
+
? 'this directory has never been restored, so no other history should have written it'
|
|
4861
|
+
: 'it is further ahead of the clock than any directory mints'),
|
|
4862
|
+
})
|
|
4863
|
+
.catch((e) => console.error('substrat: could not record a foreign lifecycle epoch (#1713)', e));
|
|
4864
|
+
continue;
|
|
4865
|
+
}
|
|
4866
|
+
if (foreign) {
|
|
4867
|
+
await this.cp.raiseLifecycleEpoch(held.epoch + 1);
|
|
4868
|
+
const [again] = await this.cp.lifecycleTargets({ scopeId });
|
|
4869
|
+
const next = again ? this.lifecycleOfTarget(again) : null;
|
|
4870
|
+
if (again && next) {
|
|
4871
|
+
t = again;
|
|
4872
|
+
lifecycle = next;
|
|
4873
|
+
answer = await delegation.deliver({ tenantId, scopeId, lifecycle });
|
|
4874
|
+
}
|
|
4875
|
+
}
|
|
4876
|
+
// The receipt is what the scope HOLDS: a newer delivery it kept instead is the truth.
|
|
4877
|
+
await this.cp.recordLifecycleReceipt(scopeId, lifecycleReceipt(answer.lifecycle), new Date().toISOString());
|
|
4878
|
+
report.delivered += 1;
|
|
4879
|
+
}
|
|
4880
|
+
catch (err) {
|
|
4881
|
+
report.failed += 1;
|
|
4882
|
+
try {
|
|
4883
|
+
await this.admin.recordOpsFailure({
|
|
4884
|
+
actor,
|
|
4885
|
+
operation: 'scope.lifecycle',
|
|
4886
|
+
stage: 'deliver',
|
|
4887
|
+
tenantId,
|
|
4888
|
+
scopeId,
|
|
4889
|
+
message: `the lifecycle (scope ${t.scope_status}, tenant ${t.tenant_status}) did not reach the ` +
|
|
4890
|
+
`deployment serving this scope; the heal sweep will deliver it again: ${err instanceof Error ? err.message : String(err)}`,
|
|
4891
|
+
});
|
|
4892
|
+
}
|
|
4893
|
+
catch (recordErr) {
|
|
4894
|
+
console.error('substrat: could not record a lifecycle delivery failure (#1713)', recordErr);
|
|
4895
|
+
}
|
|
4896
|
+
}
|
|
4897
|
+
}
|
|
4898
|
+
return report;
|
|
4899
|
+
}
|
|
4900
|
+
/** One directory row as the lifecycle a delivery carries (#1713), or null for a status this code does not know. */
|
|
4901
|
+
lifecycleOfTarget(t) {
|
|
4902
|
+
const parsed = scopeLifecycle.safeParse({
|
|
4903
|
+
scope: t.scope_status,
|
|
4904
|
+
tenant: t.tenant_status,
|
|
4905
|
+
at: new Date().toISOString(),
|
|
4906
|
+
revision: { epoch: t.epoch, scope: t.scope_rev, tenant: t.tenant_rev },
|
|
4907
|
+
});
|
|
4908
|
+
return parsed.success ? parsed.data : null;
|
|
4909
|
+
}
|
|
4910
|
+
/**
|
|
4911
|
+
* The heal sweep (#1713): deliver the lifecycle to every hosted scope whose deployment's
|
|
4912
|
+
* acknowledged state differs from the directory, and again to every scope held now — which is
|
|
4913
|
+
* what puts a hold back on a store a carry or a restore landed without it. Bounded per pass
|
|
4914
|
+
* (`limit`), drifted scopes first. Run from the shared control plane's cron.
|
|
4915
|
+
*/
|
|
4916
|
+
async healLifecycles(actor, opts) {
|
|
4917
|
+
return this.deliverLifecycles(actor, { drift: true, limit: opts?.limit ?? 200 });
|
|
4918
|
+
}
|
|
4919
|
+
/**
|
|
4920
|
+
* Whether a CP-less host holds this scope's work (#1713): `assertLive`'s predicate, answered
|
|
4921
|
+
* rather than thrown, for the entry points that defer instead of refusing — a sweep pass, a
|
|
4922
|
+
* schedule run, an executor drain. Always false on a host with a directory, whose entry
|
|
4923
|
+
* points ask the directory themselves.
|
|
4924
|
+
*/
|
|
4925
|
+
async lifecycleHeld(scopeId) {
|
|
4926
|
+
return (await this.cpLessRefusal(scopeId)) !== null;
|
|
4927
|
+
}
|
|
4484
4928
|
/**
|
|
4485
4929
|
* K-3's cross-check on its own: the (tenant, scope) pair must exist and agree before a
|
|
4486
4930
|
* subject-key operation touches anything. Without it a caller could reach another
|
|
@@ -4997,16 +5441,16 @@ export class CloudflareScopeHost {
|
|
|
4997
5441
|
* suspended or archiving scope could still be addressed and handed a 101 — a
|
|
4998
5442
|
* scope that refuses every ordinary read while quietly holding an open socket.
|
|
4999
5443
|
*
|
|
5000
|
-
*
|
|
5001
|
-
*
|
|
5002
|
-
*
|
|
5444
|
+
* On a CP-less vertical, `validateScopeAccess` is a no-op on the null control
|
|
5445
|
+
* plane, and `assertLive` reads the lifecycle the platform delivered to the
|
|
5446
|
+
* scope's own storage instead (#1713).
|
|
5003
5447
|
*
|
|
5004
5448
|
* `migrateAndRecord`, not the DO's own `ensureMigrations`: the DO migrates
|
|
5005
5449
|
* itself when the socket opens either way, but only this reports the applied
|
|
5006
5450
|
* count to the directory — so a scope whose first contact after a deploy is a
|
|
5007
5451
|
* subscription does not go dark in the migration fleet view.
|
|
5008
5452
|
*/
|
|
5009
|
-
await this.
|
|
5453
|
+
await this.assertLive(tenantId, scopeId);
|
|
5010
5454
|
await this.migrateAndRecord(scopeId);
|
|
5011
5455
|
// Asserted, not carried through from the client: the principal is the
|
|
5012
5456
|
// vertical's own resolution of its session, and the tenant and scope are the
|
|
@@ -5369,14 +5813,19 @@ export class CloudflareScopeHost {
|
|
|
5369
5813
|
// side before the call; what runs HERE is the half only this deployment can
|
|
5370
5814
|
// enforce: the scope's own permission check against its delivered
|
|
5371
5815
|
// `connection:<id>` tuple, in the scope's own DO. Fail closed — no grant, no
|
|
5372
|
-
// effect — exactly as for any other caller.
|
|
5816
|
+
// effect — exactly as for any other caller. And the lifecycle delivered to the
|
|
5817
|
+
// scope (#1713): a held scope's invoke and bytes doors refuse, as every other door does.
|
|
5373
5818
|
/** Invoke ONE operation in this deployment as a CONNECTION (#574). */
|
|
5374
5819
|
async connectorInvokeLocal(connectionId, tenantId, scopeId, operation, input) {
|
|
5820
|
+
await this.assertLive(tenantId, scopeId); // #1713: a held scope runs no connector work either
|
|
5375
5821
|
await this.migrateAndRecord(scopeId);
|
|
5376
5822
|
return this.buildStub(tenantId, scopeId, undefined, connectionId).invoke(operation, input);
|
|
5377
5823
|
}
|
|
5378
5824
|
/** Land provider bytes in this deployment as a CONNECTION — the bytes leg (#574). */
|
|
5379
5825
|
async connectorAttachmentUploadLocal(connectionId, tenantId, scopeId, upload) {
|
|
5826
|
+
// The lifecycle gate every attachments door takes (#1713), ahead of the bucket: a held
|
|
5827
|
+
// scope's provider bytes wait for the delivery's retry rather than land (#1995).
|
|
5828
|
+
await this.assertLive(tenantId, scopeId);
|
|
5380
5829
|
await this.migrateAndRecord(scopeId);
|
|
5381
5830
|
const store = await this.resolveAttachmentStore(tenantId);
|
|
5382
5831
|
return this.buildAttachmentSurface({ connectionId }, tenantId, scopeId, store).upload(upload);
|
|
@@ -5389,6 +5838,8 @@ export class CloudflareScopeHost {
|
|
|
5389
5838
|
* so a caller falls back rather than failing a dispatch over a missing file.
|
|
5390
5839
|
*/
|
|
5391
5840
|
async connectorAttachmentOpenLocal(connectionId, tenantId, scopeId, attachmentId, eventId) {
|
|
5841
|
+
// Same gate as the upload leg: a held scope's bytes are not read out of its bucket.
|
|
5842
|
+
await this.assertLive(tenantId, scopeId);
|
|
5392
5843
|
await this.migrateAndRecord(scopeId);
|
|
5393
5844
|
const store = await this.resolveAttachmentStore(tenantId);
|
|
5394
5845
|
return this.buildAttachmentSurface({ connectionId }, tenantId, scopeId, store,
|