@substrat-run/adapter-cloudflare 0.135.0 → 0.137.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/dist/checker.d.ts.map +1 -1
  2. package/dist/checker.js +3 -1
  3. package/dist/checker.js.map +1 -1
  4. package/dist/connector-calls.d.ts +11 -0
  5. package/dist/connector-calls.d.ts.map +1 -0
  6. package/dist/connector-calls.js +10 -0
  7. package/dist/connector-calls.js.map +1 -0
  8. package/dist/control-plane-do.d.ts +77 -3
  9. package/dist/control-plane-do.d.ts.map +1 -1
  10. package/dist/control-plane-do.js +250 -35
  11. package/dist/control-plane-do.js.map +1 -1
  12. package/dist/host.d.ts +297 -16
  13. package/dist/host.d.ts.map +1 -1
  14. package/dist/host.js +895 -132
  15. package/dist/host.js.map +1 -1
  16. package/dist/index.d.ts +3 -1
  17. package/dist/index.d.ts.map +1 -1
  18. package/dist/index.js +2 -0
  19. package/dist/index.js.map +1 -1
  20. package/dist/live-reads.d.ts +5 -4
  21. package/dist/live-reads.d.ts.map +1 -1
  22. package/dist/live-reads.js +5 -4
  23. package/dist/live-reads.js.map +1 -1
  24. package/dist/r2.d.ts +2 -1
  25. package/dist/r2.d.ts.map +1 -1
  26. package/dist/r2.js.map +1 -1
  27. package/dist/route-resolver.d.ts +7 -0
  28. package/dist/route-resolver.d.ts.map +1 -1
  29. package/dist/route-resolver.js +4 -0
  30. package/dist/route-resolver.js.map +1 -1
  31. package/dist/scope-do.d.ts +2 -0
  32. package/dist/scope-do.d.ts.map +1 -1
  33. package/dist/scope-do.js +896 -140
  34. package/dist/scope-do.js.map +1 -1
  35. package/dist/scope-sweeper-do.d.ts +32 -6
  36. package/dist/scope-sweeper-do.d.ts.map +1 -1
  37. package/dist/scope-sweeper-do.js +35 -5
  38. package/dist/scope-sweeper-do.js.map +1 -1
  39. package/dist/system-door.d.ts +21 -0
  40. package/dist/system-door.d.ts.map +1 -0
  41. package/dist/system-door.js +13 -0
  42. package/dist/system-door.js.map +1 -0
  43. package/package.json +10 -9
package/dist/host.js CHANGED
@@ -1,7 +1,9 @@
1
+ import { env as ambientEnv } from 'cloudflare:workers';
1
2
  import { isRewindRefusal, REWIND_REFUSED } from './rewind-refusal.js';
2
- import { delegatedReadParams, fromWireFailure, exportReadInput, exportedBatch, importBatch, importResult, importState, importsOfManifestJson, exportsOfManifestJson, importCursorMove, importCursorMoveAt, importCursorMoved, accessLogEntry, adminLogEntry, opsFailureEntry, opsFailureFingerprint, issueEntry, sweepRunEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunsPayload, modelUsageEntry, attachmentRecord, createTenantInput, identityLink, identityPool, createOrgInput, promotionAcknowledgement, bindAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, registerVerticalInput, vertical as verticalSchema, verticalServingState, verticalChannel, verticalVersion, connection, capabilityExchange, capabilityGrant, principalId, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, systemSwitch, systemSwitchOutcome, systemScheduleEntry, systemSwitchRecord, entitlementGrant, entitlementGrantInput, capabilityFilter, instant, meterReading, subjectRef, createConnectionInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, org as orgSchema, orgMembership, resolvedIdentity, identityMembership, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, delegatedReadRecord, ownerTransferAudit, subjectShredReceipt, connectorDispatchKind, callsOfManifestJson, outboundOfManifestJson, substratError, redrainEventsInput, coverage, peerCoverage, peerGrantsEntry, peerSwitch, peerSwitchOutcome, verticalCaller, verticalResolution, verticalSlug, entityObjectRef, } from '@substrat-run/contracts';
3
+ import { isSystemDoorMoved, SYSTEM_DOOR_MOVED, SYSTEM_DOOR_REGATES } from './system-door.js';
4
+ import { delegatedReadParams, fromWireFailure, exportReadInput, exportedBatch, importBatch, importResult, importState, importsOfManifestJson, exportsOfManifestJson, importCursorMove, importCursorMoveAt, importCursorMoved, accessLogEntry, adminLogEntry, opsFailureEntry, opsFailureFingerprint, issueEntry, sweepRunEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunsPayload, modelUsageEntry, attachmentRecord, createTenantInput, identityLink, identityPool, createOrgInput, promotionAcknowledgement, bindAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, registerVerticalInput, vertical as verticalSchema, verticalServingState, verticalChannel, verticalVersion, connection, capabilityExchange, mintedCapability, capabilityGrant, principalId, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, systemSwitch, systemSwitchOutcome, systemScheduleEntry, systemSwitchRecord, entitlementGrant, entitlementGrantInput, capabilityFilter, instant, meterReading, subjectRef, createConnectionInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, org as orgSchema, orgMembership, resolvedIdentity, identityMembership, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, delegatedReadRecord, ownerTransferAudit, copyMarkAudit, subjectShredReceipt, connectorDispatchKind, callsOfManifestJson, outboundOfManifestJson, substratError, redrainEventsInput, coverage, peerCoverage, peerGrantsEntry, peerSwitch, peerSwitchOutcome, verticalCaller, verticalResolution, verticalSlug, entityObjectRef, ATTACHMENT_BLOB_BINDING, blobStoreBindingName, lifecycleDelivery, scopeLifecycle, } from '@substrat-run/contracts';
3
5
  import { normalizeHostname, toRouteTarget } from './route-resolver.js';
4
- import { attachmentBlobKey, attachmentSha256, entitlementDenial, requiredEntitlementFor, foldMeterReading, parseValidationRecords, resolveScopeRecord, ulid, capabilityTokenHash, checkBecomeInput, plausibleSessionToken, foldModelUsage, backoffAt, resolveRetryPolicy, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, openSealed, ImpersonationRefused, assertSessionUsable, impersonationRowValues, mapImpersonationRow, newImpersonationSession, jobRunOf, runDueJobRuns, startJobRun, attachmentTextJob, assertAttachmentExtractors, assertJobRegistrable, isAttachmentTextRun, searchLimit, searchMatchExpression, globalFetch, assertRedrainWindow, platformRequestOf, undrainedEventsOf, inUnitMovesToAudit, staleCarryRevertRow, staleCarryReverts, withRecorded, systemSwitchedOffMessage, tenantSystemSwitchedOffMessage, CrossVerticalRegistry, importCursorSourceOf, exportBreaksOf, exportBreakRefusal, bindExportBreaksOf, bindExportBreakRefusal, collectPeers, peerSeats, connectorCallRecord, noopConnectorCallRecorder, recordConnectorCall, settleConnectionUse, splitManifestMigrations, unknownRoleError, assertRowLimit, assertRowOffset, } from '@substrat-run/kernel';
6
+ import { attachmentBlobKey, attachmentSha256, entitlementDenial, requiredEntitlementFor, foldMeterReading, parseValidationRecords, resolveScopeRecord, ulid, KEPT_COPY_REFUSAL, capabilityTokenHash, checkBecomeInput, plausibleSessionToken, foldModelUsage, backoffAt, resolveRetryPolicy, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, openSealed, ImpersonationRefused, assertSessionUsable, impersonationRowValues, mapImpersonationRow, newImpersonationSession, jobRunOf, runDueJobRuns, SYSTEM_DOOR_WAIT, startJobRun, attachmentTextJob, assertAttachmentExtractors, assertJobRegistrable, isAttachmentTextRun, searchLimit, searchMatchExpression, globalFetch, assertRedrainWindow, platformRequestOf, undrainedEventsOf, inUnitMovesToAudit, staleCarryRevertRow, staleCarryReverts, withRecorded, systemSwitchedOffMessage, tenantSystemSwitchedOffMessage, CrossVerticalRegistry, importCursorSourceOf, exportBreaksOf, exportBreakRefusal, bindExportBreaksOf, bindExportBreakRefusal, collectPeers, peerSeats, connectorCallRecord, noopConnectorCallRecorder, recordConnectorCall, settleConnectionUse, splitManifestMigrations, unknownRoleError, assertRowLimit, assertRowOffset, INERT_SCOPE_REASON, lifecycleReceipt, lifecycleRefusal, isPrimaryScope, isPrimaryScopeRow, asyncInvocationId, asyncLinePass, } from '@substrat-run/kernel';
5
7
  import { attributedHost } from '@substrat-run/kernel';
6
8
  import { isOrangeToOrange, isUpgradeRequest, LIVE_MODE_HEADER, LIVE_PRINCIPAL_HEADER, LIVE_SCOPE_HEADER, LIVE_SUBSCRIBE_PATH, LIVE_TENANT_HEADER, } from './live-reads.js';
7
9
  import { tenantStoreDatabaseName } from './d1.js';
@@ -30,12 +32,52 @@ function unwrapCapabilityReply(reply) {
30
32
  throw fromWireFailure(reply.failure);
31
33
  return reply.value;
32
34
  }
35
+ /**
36
+ * How far ahead of the clock a held lifecycle epoch may be and still be learned past (#1713). An
37
+ * epoch is the time a directory restore minted it, so a legitimate one is behind its minter's
38
+ * clock; the skew allows for clocks that disagree, and bounds what a forged answer can move.
39
+ */
40
+ export const LIFECYCLE_EPOCH_SKEW_MS = 24 * 60 * 60 * 1000;
41
+ /**
42
+ * #2016: how many served scopes one heal pass asks for their tenant record, beyond the drift and the
43
+ * holds. Rotated least-recently-asked first, so a deployment that cannot answer yet (a build from
44
+ * before the record) costs at most this many deliveries a pass and starves nothing.
45
+ */
46
+ export const TENANT_UNRECORDED_PER_PASS = 50;
47
+ /**
48
+ * K-3's refusal of a (tenant, scope) pair the host does not hold — the same answer for a scope that
49
+ * does not exist and for one of another tenant (#1718), in one wording.
50
+ */
51
+ const unknownScopeForTenant = (tenantId, scopeId) => substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
52
+ /**
53
+ * #2016: the line a CP-less host writes each time it admits a pair to a scope with no tenant record
54
+ * and no role rows — nothing to hold the pair against. Ids only.
55
+ */
56
+ export const tenantUnrecordedLine = (tenantId, scopeId) => ({
57
+ substrat: 'tenant-unrecorded',
58
+ level: 'warn',
59
+ tenantId,
60
+ scopeId,
61
+ message: 'admitted to a scope whose tenant is not recorded; a reconcile or lifecycle delivery records it',
62
+ });
63
+ /**
64
+ * Refuse to mark a scope a copy unless the request classifies it non-primary (#2005). The
65
+ * classification is the platform directory's; the vertical applies the same `isPrimaryScope` to
66
+ * it, so a primary can be marked only by a request that misstates its lineage, never by default.
67
+ */
68
+ function assertCopyLineage(lineage) {
69
+ if (isPrimaryScope(lineage)) {
70
+ throw substratError('conflict', 'mark-copy refused: the directory classifies this scope as primary, not as a copy');
71
+ }
72
+ }
33
73
  /**
34
74
  * A control-plane stand-in for a CP-less vertical (scope-local-permissions.md Phase 3).
35
75
  * The hot path a served scope actually touches becomes trust-the-upstream:
36
- * - `scopeAccessRefusal` / `setMigrationState` → no-op: the router already gated the
37
- * scope's lifecycle + tenancy from the shared directory, so the vertical trusts the
38
- * asserted node rather than re-reading a directory it does not have.
76
+ * - `scopeAccessRefusal` / `setMigrationState` → no-op: the router gates the scope's
77
+ * lifecycle + tenancy from the shared directory, and the vertical does not re-read a
78
+ * directory it does not have. Its own half of both gates reads the scope's storage
79
+ * instead (`validateScopeAccess`): the tenant the scope was provisioned for (#2016) and the lifecycle
80
+ * the platform delivered (#1713).
39
81
  * - `tenantHoldsEntitlement` → true: the SKU was enforced on the shared control plane
40
82
  * at provision (before `provisionInstance`), so a scope that EXISTS here was granted
41
83
  * it upstream — a single-vertical deployment holds its own entitlements by construction.
@@ -53,12 +95,14 @@ function nullControlPlane() {
53
95
  // #1674: the schedule switch's record is a DIRECTORY store, and a CP-less host has no
54
96
  // directory — the shared control plane that delegates here keeps it. Same posture as
55
97
  // `recordAdmin`: nothing to write, nothing recorded to read back.
56
- recordSystemSwitchedOff: noop,
98
+ recordSystemSwitchedOff: async () => null,
57
99
  recordSystemSwitchedOn: async () => null,
58
100
  restoreSystemSwitchRecord: noop,
59
101
  listSystemSwitches: async () => [],
60
102
  systemSwitchRecordsOf: async () => [],
61
103
  switchedOffModulesOf: async () => [],
104
+ // #1823: no tenant tuples here either — the platform reads them and sends `tenantHeld`.
105
+ tenantHeldSystemModules: async () => [],
62
106
  };
63
107
  return new Proxy({}, {
64
108
  get: (_t, prop) => typeof prop === 'string' && prop in passthrough
@@ -106,6 +150,28 @@ export const SWITCH_HOLD_EXTRA_WAITS = 3;
106
150
  * clock, against the releasing host's clock: minutes, not seconds.
107
151
  */
108
152
  export const SWITCH_HOLD_PENDING_MAX_MS = 5 * 60_000;
153
+ /**
154
+ * An OFF of a tenant-held module whose far end did not attest `deniesTenantGrants` (#1823): it
155
+ * switched, but its evaluator still authorizes the module's tenant-level grant on the scope.
156
+ * Carries the far end's outcome, so the operator's OFF knows whether there is a move to put back.
157
+ */
158
+ class UnattestedSwitchOff extends Error {
159
+ outcome;
160
+ constructor(scopeId, moduleId, outcome) {
161
+ super(`the deployment serving scope ${scopeId} predates the kill switch's tenant-grant denial (#1823): ` +
162
+ `module '${moduleId}' holds a tenant-level grant that its OFF would leave authorizing there. ` +
163
+ `Redeploy the vertical, then retry.`);
164
+ this.outcome = outcome;
165
+ }
166
+ }
167
+ /**
168
+ * The attachment bucket a host resolves when it was handed no resolver (#1995): the
169
+ * platform's per-tenant binding, read off this script's own env, named by the one shared
170
+ * encoding (`blobStoreBindingName`) for the tenant the caller passed in.
171
+ */
172
+ function ambientAttachmentBucket(tenantId) {
173
+ return ambientEnv[blobStoreBindingName(ATTACHMENT_BLOB_BINDING, tenantId)] ?? null;
174
+ }
109
175
  export class CloudflareScopeHost {
110
176
  admin;
111
177
  scopeNs;
@@ -145,7 +211,7 @@ export class CloudflareScopeHost {
145
211
  tenantStores;
146
212
  /** The live R2 client for per-tenant blob stores (#473); undefined ⇒ refuse loudly. */
147
213
  blobStores;
148
- /** Worker-side attachment-bucket resolver (#473); undefined ⇒ attachments() refuses. */
214
+ /** Worker-side attachment-bucket resolver (#473); the ambient per-tenant binding unless overridden (#1995). */
149
215
  attachmentBuckets;
150
216
  /** The parsers attachment text is extracted with (K-43); the host's own, never imported here. */
151
217
  attachmentExtractors;
@@ -180,6 +246,8 @@ export class CloudflareScopeHost {
180
246
  eventDrainDelegation;
181
247
  /** #1666: the schedule kill switch's reach into the deployment serving a scope. */
182
248
  systemSwitchDelegation;
249
+ /** #1713: the lifecycle's reach into the deployment serving a scope. */
250
+ lifecycleDelegation;
183
251
  /** #1706: the peer kill switch's reach into the deployment serving a scope. */
184
252
  peerSwitchDelegation;
185
253
  /** #1705 PR 3: the replay lever's reach into the deployment serving a consumer scope. */
@@ -211,7 +279,7 @@ export class CloudflareScopeHost {
211
279
  this.secretBox = options.secretBox ?? unconfiguredSecretBox;
212
280
  this.tenantStores = options.tenantStores;
213
281
  this.blobStores = options.blobStores;
214
- this.attachmentBuckets = options.attachmentBuckets;
282
+ this.attachmentBuckets = options.attachmentBuckets ?? ambientAttachmentBucket;
215
283
  assertAttachmentExtractors(options.attachmentExtractors ?? []);
216
284
  this.attachmentExtractors = options.attachmentExtractors ?? [];
217
285
  this.fetchImpl = options.fetch ?? globalFetch;
@@ -225,6 +293,7 @@ export class CloudflareScopeHost {
225
293
  this.connectorDelegation = options.connectorDelegation;
226
294
  this.eventDrainDelegation = options.eventDrainDelegation;
227
295
  this.systemSwitchDelegation = options.systemSwitchDelegation;
296
+ this.lifecycleDelegation = options.lifecycleDelegation;
228
297
  this.peerSwitchDelegation = options.peerSwitchDelegation;
229
298
  this.importCursorDelegation = options.importCursorDelegation;
230
299
  this.admin = this.buildAdmin();
@@ -353,21 +422,78 @@ export class CloudflareScopeHost {
353
422
  };
354
423
  if (this.executors.size === 0)
355
424
  return report;
425
+ // #1901: one line per attempt, capped per pass. Written on the coordinator, where the
426
+ // attempt runs — the DO only journals it.
427
+ const lines = asyncLinePass();
428
+ try {
429
+ await this.drainExecutorsPass(tenantId, scopeId, invocationId, report, lines);
430
+ }
431
+ finally {
432
+ lines.end();
433
+ }
434
+ return report;
435
+ }
436
+ async drainExecutorsPass(tenantId, scopeId, invocationId, report, lines) {
356
437
  const stub = this.scopeStub(scopeId);
357
- for (const [id, executor] of this.executors) {
438
+ // #2005: a non-primary scope (a fork, a snapshot, a preview of either kind) causes no
439
+ // outbound effects, so its deliveries are journaled terminal with the reason and no
440
+ // handler runs. Asked of the directory where there is one. A CP-less host has none, so it
441
+ // asks the scope's own storage whether it is classified a copy (`_substrat_copy_origin`'s
442
+ // `is_copy`, #2009, set on the directory's word) — a preview and a snapshot reach a hosted
443
+ // vertical as restores the platform marks.
444
+ // Asked on the first due event, once per pass: most passes have none, and the directory
445
+ // is one global object.
446
+ let inert;
447
+ const isInert = () => (inert ??= this.cpLess
448
+ ? stub.isCopy()
449
+ : this.cp.getScopeRecord(tenantId, scopeId).then((row) => !isPrimaryScopeRow(row)));
450
+ // #1713: a CP-less scope its lifecycle holds attempts nothing and journals nothing, so every
451
+ // due delivery stays due for the first pass after it is live again. Asked on the first due
452
+ // event, like `isInert`. Every caller passed `assertLive` already, so this is for the scope
453
+ // suspended between that gate and the drain: a request in flight, whose write commits while
454
+ // its effect waits. A host with a directory answers false without a read.
455
+ let held;
456
+ const isHeld = () => (held ??= this.lifecycleHeld(scopeId));
457
+ drain: for (const [id, executor] of this.executors) {
358
458
  const deliveryId = `executor:${id}`;
359
459
  const { events, undecodable } = await stub.pendingExecutorDeliveries(deliveryId, executor.eventType);
360
460
  // #1636: an event the DO could not decode is dead-lettered for this executor at once,
361
461
  // and its handler never sees it. Terminal on the FIRST failure, unlike a handler's:
362
462
  // the decode is pure, so a retry cannot succeed. The rows behind it are delivered
363
463
  // below — the decode used to throw the whole list, on every pass.
464
+ if ((undecodable.length > 0 || events.length > 0) && (await isHeld())) {
465
+ report.lifecycleHeld = true;
466
+ break drain;
467
+ }
468
+ // #1901: an attempt's line. The attempt number is the one the journal is about to record.
469
+ const unitOf = (eventId, attempt) => ({
470
+ kind: 'consumer',
471
+ tenantId,
472
+ scopeId,
473
+ invocationId: asyncInvocationId(invocationId),
474
+ operation: deliveryId,
475
+ eventType: executor.eventType,
476
+ eventId,
477
+ attempt,
478
+ startedAt: Date.now(),
479
+ });
364
480
  for (const bad of undecodable) {
365
481
  report.attempted += 1;
366
- await stub.recordExecutorAttempt(bad.eventId, deliveryId, bad.error, null, invocationId);
482
+ const attempt = await stub.recordExecutorAttempt(bad.eventId, deliveryId, bad.error, null, invocationId);
367
483
  report.deadLettered += 1;
484
+ lines.write({ ...unitOf(bad.eventId, attempt), outcome: 'dead-lettered' });
368
485
  }
369
486
  for (const event of events) {
370
487
  report.attempted += 1;
488
+ const startedAt = Date.now();
489
+ if (await isInert()) {
490
+ // No next attempt, like an undecodable row: a scope does not become primary.
491
+ const attempt = await stub.recordExecutorAttempt(event.id, deliveryId, INERT_SCOPE_REASON, null, invocationId);
492
+ report.inert = (report.inert ?? 0) + 1;
493
+ // Its own outcome, never `delivered`: no handler ran (#2005).
494
+ lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'inert' });
495
+ continue;
496
+ }
371
497
  this.causedBy = event.id;
372
498
  try {
373
499
  if (executor.kind === 'connector' && this.cpLess) {
@@ -381,16 +507,20 @@ export class CloudflareScopeHost {
381
507
  // the at-least-once residue, as it already must in-process.
382
508
  await stub.routeExecutorEventToPlatform(event.id, deliveryId, connectorDispatchKind(executor.provider), JSON.stringify({ executorId: id, event }), JSON.stringify({ system: 'connector-dispatch' }), invocationId);
383
509
  report.routedToPlatform += 1;
510
+ // Handed on, not run here: the platform's drain owns its attempts from now on.
511
+ lines.write({ ...unitOf(event.id, 1), startedAt, outcome: 'routed' });
384
512
  }
385
513
  else if (executor.kind === 'connector') {
386
514
  await executor.handler(await this.connectorContext(tenantId, scopeId, executor.timeoutMs, event.id), event);
387
- await stub.recordExecutorAttempt(event.id, deliveryId, null, null, invocationId);
515
+ const attempt = await stub.recordExecutorAttempt(event.id, deliveryId, null, null, invocationId);
388
516
  report.delivered += 1;
517
+ lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'delivered' });
389
518
  }
390
519
  else {
391
520
  await executor.handler(this.admin, event);
392
- await stub.recordExecutorAttempt(event.id, deliveryId, null, null, invocationId);
521
+ const attempt = await stub.recordExecutorAttempt(event.id, deliveryId, null, null, invocationId);
393
522
  report.delivered += 1;
523
+ lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'delivered' });
394
524
  }
395
525
  }
396
526
  catch (err) {
@@ -405,18 +535,23 @@ export class CloudflareScopeHost {
405
535
  report.deadLettered += 1;
406
536
  else
407
537
  report.retrying += 1;
538
+ lines.write({
539
+ ...unitOf(event.id, attempts),
540
+ startedAt,
541
+ outcome: exhausted ? 'dead-lettered' : 'retrying',
542
+ error: err,
543
+ });
408
544
  }
409
545
  finally {
410
546
  this.causedBy = null;
411
547
  }
412
548
  }
413
549
  }
414
- return report;
415
550
  }
416
551
  async drainDue(tenantId, scopeId) {
417
552
  // Same lifecycle gate `getScope` applies (K-3): a suspended or archived scope
418
553
  // does not get its effects driven either.
419
- await this.validateScopeAccess(tenantId, scopeId);
554
+ await this.assertLive(tenantId, scopeId);
420
555
  await this.migrateAndRecord(scopeId);
421
556
  // #1525: null, and honestly so — a sweep is not a call. An attempt this pass makes
422
557
  // records no invocation, which is what distinguishes it from the first attempt the
@@ -448,7 +583,7 @@ export class CloudflareScopeHost {
448
583
  return {
449
584
  startOrJoin: (key, row) => stub.jobRunStartOrJoin(key.moduleId, key.job, key.instance, row),
450
585
  get: (id) => stub.jobRunById(id),
451
- due: (now, limit, afterId) => stub.jobRunsDue(now, limit, afterId),
586
+ dueKeys: (now, max) => stub.jobRunsDueKeys(now, max),
452
587
  list: (filter) => stub.jobRunList(filter),
453
588
  patch: (id, patch) => stub.jobRunPatch(id, patch),
454
589
  commitPass: (id, patch) => stub.jobCommitPass(id, patch),
@@ -474,14 +609,14 @@ export class CloudflareScopeHost {
474
609
  this.attachmentExtractors);
475
610
  }
476
611
  async startJobRun(tenantId, scopeId, input) {
477
- await this.validateScopeAccess(tenantId, scopeId);
612
+ await this.assertLive(tenantId, scopeId);
478
613
  await this.migrateAndRecord(scopeId);
479
614
  return jobRunOf(await startJobRun(this.jobStore(scopeId), input, ulid, () => new Date().toISOString()));
480
615
  }
481
616
  async runDueJobs(tenantId, scopeId, options) {
482
617
  // Same lifecycle gate as `drainDue`: a suspended scope's runs wait rather than
483
618
  // advance, and an archived one's never move again.
484
- await this.validateScopeAccess(tenantId, scopeId);
619
+ await this.assertLive(tenantId, scopeId);
485
620
  await this.migrateAndRecord(scopeId);
486
621
  // #1575: the kernel's extraction job is this host's own, bound to this scope — no
487
622
  // deployment registers it, and none can shadow it.
@@ -490,7 +625,10 @@ export class CloudflareScopeHost {
490
625
  store: this.jobStore(scopeId),
491
626
  handlerFor: (run) => isAttachmentTextRun(run) ? attachmentText : this.jobs.get(`${run.module_id}/${run.job}`),
492
627
  now: () => new Date().toISOString(),
493
- openScope: (run) => this.getSystemScope(run.module_id, tenantId, scopeId),
628
+ // #1834: the door is opened FOR this pass, so its "not now" is tied to this pass alone.
629
+ openScope: async (run, pass) => this.buildStub(tenantId, scopeId, undefined, undefined, await this.openSystemDoor(run.module_id, tenantId, scopeId, undefined, pass)),
630
+ // #1834: only this host's own door refusals defer a pass, recognised by identity.
631
+ deferral: (err, pass) => this.takeDoorWait(err, pass),
494
632
  maxPasses: options?.maxPasses,
495
633
  limit: options?.limit,
496
634
  });
@@ -506,7 +644,7 @@ export class CloudflareScopeHost {
506
644
  // same context build as the in-process path, so the handler cannot tell which host
507
645
  // ran it. On a CP-less host `connectorContext` throws from the null control plane:
508
646
  // fail closed, exactly the hole routing exists to avoid.
509
- await this.validateScopeAccess(tenantId, scopeId);
647
+ await this.assertLive(tenantId, scopeId);
510
648
  this.causedBy = event.id;
511
649
  try {
512
650
  await handler(await this.connectorContext(tenantId, scopeId, options?.timeoutMs ?? 30_000, event.id), event);
@@ -570,7 +708,7 @@ export class CloudflareScopeHost {
570
708
  const rec = await this.cp.getScopeRecord(tenantId, scopeId);
571
709
  // K-3: a scope under another tenant is indistinguishable from one that does not exist.
572
710
  if (!rec)
573
- throw substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
711
+ throw unknownScopeForTenant(tenantId, scopeId);
574
712
  if (rec.status !== 'active' && rec.status !== 'provisioning') {
575
713
  throw new Error(`scope not migratable (status: ${rec.status}): ${scopeId}`);
576
714
  }
@@ -642,6 +780,31 @@ export class CloudflareScopeHost {
642
780
  async listCapabilitiesLocal(scopeId, filter) {
643
781
  return this.scopeStub(scopeId).listCapabilities(filter);
644
782
  }
783
+ /**
784
+ * The platform's `become` mint on the CP-less path (#1686) — `HostAdmin.mintCapability`'s
785
+ * ScopeDO half, for the deployment that serves the scope. Its one caller is an owner claim
786
+ * link, minted on the platform's instruction behind the vertical's platform-gated
787
+ * `/internal/owner-claim`; the control plane made the K-3 check and keeps the admin log before
788
+ * it called. Not a module verb: a module mints through `ctx.capabilities`, which mints `act`
789
+ * capabilities only. Same lifecycle gate as the exchange, so a held scope mints nothing.
790
+ */
791
+ async mintCapabilityLocal(tenantId, scopeId, input, actor) {
792
+ // Checked on this side too, as `HostAdmin.mintCapability` does: a typed refusal thrown
793
+ // across the RPC arrives as a bare message.
794
+ checkBecomeInput(input, new Date().toISOString());
795
+ await this.assertLive(tenantId, scopeId);
796
+ await this.migrateAndRecord(scopeId);
797
+ return mintedCapability.parse(await this.scopeStub(scopeId).mintBecomeCapability(input, actor));
798
+ }
799
+ /**
800
+ * The platform's revoke on the CP-less path (#1686) — how a re-minted claim link retires the
801
+ * previous one. True when the scope held the capability (revoked now, or already); false when
802
+ * it holds no such capability. No lifecycle gate: a revoke only narrows, so a held scope still
803
+ * takes one. Idempotent.
804
+ */
805
+ async revokeCapabilityLocal(scopeId, capabilityId, actor) {
806
+ return (await this.scopeStub(scopeId).revokeCapabilityAsPlatform(capabilityId, actor)) !== null;
807
+ }
645
808
  /**
646
809
  * Copy one scope's data into a fresh scope DO, entirely within THIS deployment —
647
810
  * the data half of an orchestrated snapshot (preview-and-snapshots.md §9). Like the
@@ -650,10 +813,24 @@ export class CloudflareScopeHost {
650
813
  * activation, version bind) stays on the control plane's side. Because source and
651
814
  * destination sit in the same SCOPE namespace, no scope bytes ever leave the
652
815
  * deployment — the §9 property the trust line rests on.
816
+ *
817
+ * #2016: `tenantId`, the tenant the platform snapshots for (absent from an older platform). The
818
+ * source must not be foreign to it — a copy never reads another tenant's scope — and the fork
819
+ * records it as its own receipt, not the source's.
653
820
  */
654
- async snapshotScopeLocal(sourceScopeId, destScopeId) {
821
+ async snapshotScopeLocal(sourceScopeId, destScopeId, tenantId) {
822
+ if (tenantId !== undefined && (await this.scopeStub(sourceScopeId).admission(tenantId)).verdict === 'foreign') {
823
+ throw unknownScopeForTenant(tenantId, sourceScopeId);
824
+ }
655
825
  const tables = await this.scopeStub(sourceScopeId).exportDump();
656
- await this.scopeStub(destScopeId).importDump(tables, destScopeId, { sourceScopeId, exact: true });
826
+ // #2009: a snapshot is a fork by construction (the control plane's row names `forkedFrom`, so
827
+ // the directory never calls it primary), and a load classifies nothing by itself: marked here.
828
+ await this.scopeStub(destScopeId).importDump(tables, destScopeId, {
829
+ sourceScopeId,
830
+ exact: true,
831
+ markCopy: true,
832
+ provisionedFor: tenantId,
833
+ });
657
834
  return { tables: tables.length };
658
835
  }
659
836
  /**
@@ -666,14 +843,145 @@ export class CloudflareScopeHost {
666
843
  /** #1742: the directory's recorded-off modules, switched off on THIS scope in the replay's
667
844
  * own event — a dump from before a switch was pulled brings its grants back live. #1869:
668
845
  * `sourceScopeId`, the scope the dump was captured from, narrows the grant re-point to it;
669
- * `exact` says the platform exported the dump itself, so the re-point never falls back. */
846
+ * `exact` says the platform exported the dump itself, so the re-point never falls back.
847
+ * #1722: `loadStamp`, the stamp a carry leaves on the copy it lands, for a fenced wipe later,
848
+ * and `expect`, the marker the carry read here: the load is refused if the store moved since.
849
+ * #2005: `markCopy` — the directory's classification of the scope, sent when it is not
850
+ * primary; the restore then marks it a copy. Refused if it classifies a primary.
851
+ * #2016: `tenantId`, the tenant the platform restores for: recorded as the scope's receipt in
852
+ * the load's own transaction, and a store provisioned for another tenant refuses the load. */
670
853
  opts) {
671
- const switchedOff = await this.scopeStub(scopeId).importDump(tables, scopeId, {
672
- switchOff: opts?.switchedOff ? { moduleIds: opts.switchedOff, at: new Date().toISOString() } : undefined,
854
+ if (opts?.markCopy)
855
+ assertCopyLineage(opts.markCopy);
856
+ const load = {
857
+ switchOff: opts?.switchedOff
858
+ ? { moduleIds: opts.switchedOff, at: new Date().toISOString(), tenantHeld: opts.tenantHeld }
859
+ : undefined,
673
860
  sourceScopeId: opts?.sourceScopeId,
674
861
  exact: opts?.exact,
862
+ loadStamp: opts?.loadStamp,
863
+ markCopy: opts?.markCopy !== undefined,
864
+ provisionedFor: opts?.tenantId,
865
+ };
866
+ // The refusals come back as values (a throw over the RPC carries only its message) and are
867
+ // thrown here, in the vertical's own isolate, so its route answers 409 or 412, not a fault.
868
+ const out = await this.scopeStub(scopeId).importDumpChecked(tables, scopeId, {
869
+ ...load,
870
+ ...(opts?.expect ? { expect: opts.expect } : {}),
675
871
  });
676
- return { tables: tables.length, ...(opts?.switchedOff ? { switchedOff } : {}) };
872
+ if (out.refused === false) {
873
+ return { tables: tables.length, ...(opts?.switchedOff ? { switchedOff: out.switchedOff } : {}) };
874
+ }
875
+ if (out.refused === 'kept')
876
+ throw substratError('conflict', KEPT_COPY_REFUSAL);
877
+ if (out.refused === 'tenant')
878
+ throw substratError('conflict', out.message);
879
+ // A retry of a load that already committed (its answer was lost on the way back) is
880
+ // refused by the marker that load itself moved. The store holding THIS request's stamp
881
+ // says so: no other load writes it. Answered as applied, without `switchedOff`, so the
882
+ // caller's re-assert after the bind covers the OFF positions.
883
+ const now = await this.scopeStub(scopeId).loadMarker();
884
+ if (!opts?.loadStamp || now.loadStamp !== opts.loadStamp) {
885
+ throw substratError('precondition_failed', 'scope store changed since the carry read it; nothing was loaded (#1722)');
886
+ }
887
+ return { tables: tables.length };
888
+ }
889
+ /** The kept-copy marker of this scope's store in THIS deployment (#1722), or null. */
890
+ async keptCopyLocal(scopeId) {
891
+ return this.scopeStub(scopeId).keptCopy();
892
+ }
893
+ /**
894
+ * Discard the kept copy of a scope in THIS deployment (#1722), behind the vertical's
895
+ * `/internal/kept-copy/discard`: the staff resolution, only at the write revision the operator
896
+ * acted on. Answers whether it was discarded, or why not.
897
+ */
898
+ async discardKeptCopyLocal(scopeId, revision, carriedAway,
899
+ /** #2005: the directory's classification, sent when the scope is not primary. */
900
+ markCopy,
901
+ /** #1722 (Codex #2008 r13): the load stamp read with `revision`; a replaced store is refused. */
902
+ loadStamp) {
903
+ if (markCopy)
904
+ assertCopyLineage(markCopy);
905
+ return this.scopeStub(scopeId).discardKeptCopy(scopeId, revision, carriedAway, markCopy !== undefined, ...(loadStamp !== undefined ? [loadStamp] : []));
906
+ }
907
+ /**
908
+ * What a carry's restore into this scope's store expects to find unchanged (#1722), behind
909
+ * the vertical's `/internal/load-marker`. Read before the carry checks the binding again; the
910
+ * restore then carries it as `expect`, and is refused if the store was loaded or written since.
911
+ */
912
+ async loadMarkerLocal(scopeId) {
913
+ return this.scopeStub(scopeId).loadMarker();
914
+ }
915
+ /**
916
+ * Wipe the copy a carry left in THIS deployment (#1722), behind the vertical's
917
+ * `/internal/wipe-carried`: only if nothing was loaded into the scope's DO since the stamp
918
+ * the carry read (`expectLoadStamp`, null for a store no load has stamped). Non-terminal,
919
+ * so a later bind back to this version can restore into it. False when refused.
920
+ */
921
+ async wipeCarriedLocal(scopeId, expectLoadStamp, carriedAway, opts = {}) {
922
+ if (opts.markCopy)
923
+ assertCopyLineage(opts.markCopy);
924
+ return this.scopeStub(scopeId).wipeCarried(scopeId, expectLoadStamp, carriedAway, {
925
+ ...(opts.expectRevision !== undefined ? { expectRevision: opts.expectRevision } : {}),
926
+ ...(opts.protectIfChanged !== undefined ? { protectIfChanged: opts.protectIfChanged } : {}),
927
+ markCopy: opts.markCopy !== undefined,
928
+ });
929
+ }
930
+ /** Release a kept copy that is the live store after all (#1722), at the revision read. */
931
+ async releaseKeptCopyLocal(scopeId, revision,
932
+ /** #2005: the directory's classification, sent when the scope is not primary. */
933
+ markCopy,
934
+ /** #1722 (Codex #2008 r13): the load stamp read with `revision`; a replaced store is refused. */
935
+ loadStamp) {
936
+ if (markCopy)
937
+ assertCopyLineage(markCopy);
938
+ return this.scopeStub(scopeId).releaseKeptCopy(revision, markCopy !== undefined, ...(loadStamp !== undefined ? [loadStamp] : []));
939
+ }
940
+ /**
941
+ * Store the lifecycle the platform delivered for one scope in THIS deployment (#1713), behind
942
+ * the vertical's `/internal/lifecycle`. Kept unless the scope already holds a newer one, so a
943
+ * push that arrives late cannot undo a later transition. `applied` is what the platform's heal
944
+ * sweep records as delivered; `changed` says the gate's answer moved.
945
+ *
946
+ * #2016: `tenantId`, the tenant the directory delivered it for (absent from an older platform).
947
+ * A scope provisioned for another tenant refuses the delivery (`conflict`) and stores nothing;
948
+ * a scope provisioned before the tenant receipt existed records it here, its role rows agreeing.
949
+ */
950
+ async setLifecycleLocal(scopeId, next, tenantId) {
951
+ const out = await this.scopeStub(scopeId).setLifecycle(scopeLifecycle.parse(next), tenantId);
952
+ if ('refused' in out)
953
+ throw substratError('conflict', out.message);
954
+ return lifecycleDelivery.parse(out);
955
+ }
956
+ /**
957
+ * Mark one scope in THIS deployment a copy (#2005), behind the vertical's `/internal/mark-copy`:
958
+ * the repair of a copy that predates the marker, which a CP-less coordinator reads for primacy.
959
+ * The control plane decides which scopes (its directory says they are not primary) and audits.
960
+ */
961
+ async markCopyLocal(scopeId, lineage) {
962
+ assertCopyLineage(lineage);
963
+ return { marked: await this.scopeStub(scopeId).markCopy() };
964
+ }
965
+ /**
966
+ * Clear a mistaken copy classification from one scope in THIS deployment (#2005), behind the
967
+ * vertical's `/internal/clear-copy-mark`: staff's correction for a scope the directory says IS
968
+ * primary. Refused for a scope classified a copy. A load's copied-events mark is never touched
969
+ * (#2009), so a primary restored from another scope's backup runs its own effects again and
970
+ * still never runs the work it copied.
971
+ */
972
+ async clearCopyMarkLocal(scopeId, lineage,
973
+ /** #1722 (Codex #2008 r12–r13): the store as the caller means it, its load stamp and write
974
+ * revision, compared in the clear's own transaction; refused (412) if either moved. The
975
+ * platform's reconcile of a carry's destination sends it; staff send none. */
976
+ expect) {
977
+ if (!isPrimaryScope(lineage)) {
978
+ throw substratError('conflict', 'clear-copy-mark refused: the directory classifies this scope as a copy (a preview or a fork)');
979
+ }
980
+ const outcome = await this.scopeStub(scopeId).clearCopyMark(...(expect ? [expect] : []));
981
+ if (outcome === 'changed') {
982
+ throw substratError('precondition_failed', `clear-copy-mark refused: scope ${scopeId}'s store changed since its revision was read`);
983
+ }
984
+ return { cleared: outcome === 'cleared' };
677
985
  }
678
986
  /**
679
987
  * Re-apply the vertical's OWN role definitions to one scope, CP-lessly — the
@@ -710,6 +1018,13 @@ export class CloudflareScopeHost {
710
1018
  async exportScopeLocal(scopeId) {
711
1019
  return this.scopeStub(scopeId).exportDump();
712
1020
  }
1021
+ /**
1022
+ * `exportScopeLocal` with the store's load stamp, read in the same DO call (#1722): what a
1023
+ * carry's fenced wipe of the copy it leaves here expects. Behind the vertical's `/internal/export`.
1024
+ */
1025
+ async exportScopeStampedLocal(scopeId) {
1026
+ return this.scopeStub(scopeId).exportDumpStamped();
1027
+ }
713
1028
  /** Facet this host's own scope's outbox (#1239) — the vertical-host read. */
714
1029
  async facetEventsLocal(scopeId, input) {
715
1030
  return this.scopeStub(scopeId).facetEvents(input);
@@ -924,8 +1239,15 @@ export class CloudflareScopeHost {
924
1239
  // after it then finds them off, and audits what the unit moved.
925
1240
  const ownStore = this.cpLess || record.vertical === null;
926
1241
  const recordedOff = ownStore ? await this.cp.switchedOffModulesOf(input.tenantId, input.scopeId) : [];
1242
+ const seatAt = new Date().toISOString();
927
1243
  const switchedOff = await this.scopeStub(input.scopeId).seatTuples(seats, recordedOff.length
928
- ? { scopeId: input.scopeId, moduleIds: recordedOff, at: new Date().toISOString() }
1244
+ ? {
1245
+ scopeId: input.scopeId,
1246
+ moduleIds: recordedOff,
1247
+ at: seatAt,
1248
+ // #1823: a recorded-off module held only by a tenant grant goes back off too.
1249
+ tenantHeld: await this.cp.tenantHeldSystemModules(input.tenantId, recordedOff, seatAt),
1250
+ }
929
1251
  : undefined);
930
1252
  if (ownStore) {
931
1253
  await this.admin.reassertSystemSwitches(actor, { tenantId: input.tenantId, scopeId: input.scopeId }, { appliedInUnit: switchedOff });
@@ -1065,21 +1387,21 @@ export class CloudflareScopeHost {
1065
1387
  // gates and metadata facts live in the ScopeDO (per-scope serialization, spine event
1066
1388
  // in the same transaction); bytes go straight to the per-tenant R2 bucket through the
1067
1389
  // binding the vertical's worker resolved — never through the DO.
1068
- await this.validateScopeAccess(tenantId, scopeId);
1390
+ await this.assertLive(tenantId, scopeId);
1069
1391
  await this.migrateAndRecord(scopeId);
1070
1392
  const store = await this.resolveAttachmentStore(tenantId);
1071
1393
  return this.buildAttachmentSurface({ principal }, tenantId, scopeId, store);
1072
1394
  }
1073
1395
  async getSystemAttachments(moduleId, tenantId, scopeId) {
1074
- // Reuse the system invoke door's module, tenant and lifecycle gate. The DO
1075
- // authorizes each open against the attachment target's read permission as
1396
+ // Reuse the system invoke door's module, tenant and lifecycle gate, and its rewind gate
1397
+ // (#1834). The DO authorizes each open against the attachment target's read permission as
1076
1398
  // system:<moduleId>; the worker reads bytes only after that gate succeeds.
1077
- await this.getSystemScope(moduleId, tenantId, scopeId);
1399
+ const door = await this.openSystemDoor(moduleId, tenantId, scopeId);
1078
1400
  const store = await this.resolveAttachmentStore(tenantId);
1079
1401
  const stub = this.scopeStub(scopeId);
1080
1402
  return {
1081
1403
  open: async (attachmentId) => {
1082
- const record = await stub.systemAttachmentAuthorize(attachmentId, moduleId, tenantId, scopeId);
1404
+ const record = await door.through((instance) => stub.systemAttachmentAuthorize(attachmentId, moduleId, tenantId, scopeId, instance));
1083
1405
  return record ? this.openAttachmentBytes(store, scopeId, record) : null;
1084
1406
  },
1085
1407
  };
@@ -1138,17 +1460,14 @@ export class CloudflareScopeHost {
1138
1460
  const store = await this.resolveAttachmentStore(conn.tenant_id);
1139
1461
  return this.buildAttachmentSurface({ connectionId }, conn.tenant_id, scopeId, store, forEvent);
1140
1462
  }
1141
- /** Resolve the per-tenant R2 blob store, or fail closed exactly as `attachments` did. */
1463
+ /** Resolve the per-tenant R2 blob store, or fail closed. */
1142
1464
  async resolveAttachmentStore(tenantId) {
1143
- if (!this.attachmentBuckets) {
1144
- throw new Error(`attachments are not configured on this host (#473): pass ` +
1145
- `CloudflareScopeHostOptions.attachmentBuckets — (tenantId) => ` +
1146
- `env[blobStoreBindingName('<BINDING>', tenantId)] for the vertical's declared blob store`);
1147
- }
1148
1465
  const bucket = await this.attachmentBuckets(tenantId);
1149
1466
  if (!bucket) {
1150
- throw new Error(`no attachment bucket resolved for tenant ${tenantId} (#473) — is the per-tenant blob ` +
1151
- `store provisioned and its r2_bucket binding attached to the serving script?`);
1467
+ throw new Error(`no attachment bucket resolved for tenant ${tenantId} (#473, #1995) — this script has no ` +
1468
+ `${blobStoreBindingName(ATTACHMENT_BLOB_BINDING, tenantId)} binding. A vertical whose modules declare ` +
1469
+ `attachmentTargets gets one per installed tenant once it is pushed with a CLI that declares the ` +
1470
+ `store (re-push); off the platform, pass CloudflareScopeHostOptions.attachmentBuckets.`);
1152
1471
  }
1153
1472
  return r2TenantBlobStore(bucket);
1154
1473
  }
@@ -1288,8 +1607,16 @@ export class CloudflareScopeHost {
1288
1607
  // restored in its deployment, which the platform carries the same list to.
1289
1608
  const ownStore = this.cpLess || existing.vertical === null;
1290
1609
  const recordedOff = ownStore ? await this.cp.switchedOffModulesOf(tenantId, scopeId) : [];
1610
+ const replayAt = new Date().toISOString();
1291
1611
  const switchedOff = await this.scopeStub(scopeId).importDump(dump.tables, scopeId, {
1292
- switchOff: recordedOff.length ? { moduleIds: recordedOff, at: new Date().toISOString() } : undefined,
1612
+ switchOff: recordedOff.length
1613
+ ? {
1614
+ moduleIds: recordedOff,
1615
+ at: replayAt,
1616
+ // #1823: a recorded-off module held only by a tenant grant goes back off too.
1617
+ tenantHeld: await this.cp.tenantHeldSystemModules(tenantId, recordedOff, replayAt),
1618
+ }
1619
+ : undefined,
1293
1620
  sourceScopeId: opts?.sourceScopeId ?? dump.scopeId,
1294
1621
  });
1295
1622
  await this.recordAdmin(actor, 'restoreScope', { tenantId, scopeId }, null, { sourceScopeId: dump.scopeId, tables: dump.tables.length, capturedAt: dump.capturedAt });
@@ -1381,7 +1708,7 @@ export class CloudflareScopeHost {
1381
1708
  async getScope(principal, tenantId, scopeId, options) {
1382
1709
  // Lifecycle gates (control-plane.md §4.1/§4.2), the K-3 fail-closed path,
1383
1710
  // evaluated durably in the ControlPlaneDO. A throw propagates here.
1384
- await this.validateScopeAccess(tenantId, scopeId);
1711
+ await this.assertLive(tenantId, scopeId);
1385
1712
  await this.migrateAndRecord(scopeId);
1386
1713
  return this.buildStub(tenantId, scopeId, principal, undefined, undefined, options);
1387
1714
  }
@@ -1456,7 +1783,7 @@ export class CloudflareScopeHost {
1456
1783
  const record = await this.resolveImpersonation(session, tenantId, scopeId);
1457
1784
  // The same lifecycle gate the principal door applies: a suspended tenant
1458
1785
  // refuses a support session exactly as it refuses a user.
1459
- await this.validateScopeAccess(tenantId, scopeId);
1786
+ await this.assertLive(tenantId, scopeId);
1460
1787
  await this.migrateAndRecord(scopeId);
1461
1788
  return this.buildStub(tenantId, scopeId, record.principal, undefined, undefined, options, record.id);
1462
1789
  }
@@ -1467,12 +1794,93 @@ export class CloudflareScopeHost {
1467
1794
  * against `system:<moduleId>` grants inside the stub.
1468
1795
  */
1469
1796
  async getSystemScope(moduleId, tenantId, scopeId) {
1797
+ return this.buildStub(tenantId, scopeId, undefined, undefined, await this.openSystemDoor(moduleId, tenantId, scopeId));
1798
+ }
1799
+ /**
1800
+ * #1834: the system door — the ONE place every entry acting as `system:<moduleId>` passes
1801
+ * through: a schedule's fire, a job run's `pass.scope()`, a module's attachment open. The
1802
+ * module must be registered here and the scope live, as before. Then the door GATES: the
1803
+ * scope's own state read for the module, and after it the rewind hold (#1819), in the order
1804
+ * `SWITCH_HOLD_SETTLE_MS` relies on. A module the hold keeps off is refused here, as a
1805
+ * schedule pass skips it.
1806
+ *
1807
+ * A gate read is about the instance that answered it, and a door can be held for as long as
1808
+ * a job pass runs. So every call through the door is PINNED to that instance: the DO refuses
1809
+ * one that lands on another (`SYSTEM_DOOR_MOVED`), since a PITR restore always restarts the
1810
+ * object, and the door gates again and retries. A restart within one call's round trip is
1811
+ * rare, so past `SYSTEM_DOOR_REGATES` it fails closed (`unavailable`). No clock is involved:
1812
+ * the gate and the call meet the same storage, or the call does not run.
1813
+ */
1814
+ async openSystemDoor(moduleId, tenantId, scopeId, gated,
1815
+ /** #1834: the job pass this door is opened for; its "not now" refusals are tied to it. */
1816
+ pass) {
1470
1817
  if (!this.moduleIds.has(moduleId)) {
1471
1818
  throw new Error(`module not registered on this host: ${moduleId}`);
1472
1819
  }
1473
- await this.validateScopeAccess(tenantId, scopeId);
1820
+ await this.assertLive(tenantId, scopeId);
1474
1821
  await this.migrateAndRecord(scopeId);
1475
- return this.buildStub(tenantId, scopeId, undefined, undefined, moduleId);
1822
+ // A gate this door reads itself reports a hold it could not read; one handed in was the
1823
+ // caller's to report (`runDueSchedules` puts it on its pass report).
1824
+ const regate = async () => {
1825
+ const fresh = await this.systemDoorGate(scopeId, moduleId);
1826
+ if (fresh.error)
1827
+ console.error(`substrat: system door ${moduleId} on ${scopeId}: ${fresh.error}`);
1828
+ return fresh;
1829
+ };
1830
+ let gate = gated ?? (await regate());
1831
+ return {
1832
+ moduleId,
1833
+ through: async (call) => {
1834
+ for (let regates = 0;; regates += 1) {
1835
+ if (gate.held) {
1836
+ // The hold ends when the switch is applied again: a job run waits it out without
1837
+ // spending its retries, because the driver defers on THIS refusal (`doorWait`).
1838
+ throw this.doorWait(pass, substratError('forbidden', `module '${moduleId}' is held off on this scope: it was rewound to before its schedule ` +
1839
+ 'switch was turned off, and it stays off until the switch is applied again', { reason: SYSTEM_DOOR_WAIT }));
1840
+ }
1841
+ const answer = await call(gate.instance);
1842
+ if (!isSystemDoorMoved(answer))
1843
+ return answer;
1844
+ if (regates >= SYSTEM_DOOR_REGATES) {
1845
+ throw this.doorWait(pass, substratError('unavailable', `the scope kept restarting under the system door of module '${moduleId}' ` +
1846
+ `(${regates + 1} attempts); nothing ran. Retry later`, { reason: SYSTEM_DOOR_WAIT }));
1847
+ }
1848
+ gate = await regate();
1849
+ }
1850
+ },
1851
+ };
1852
+ }
1853
+ /**
1854
+ * #1834: the refusals this host's system door threw that mean "not now" (a hold, a scope that
1855
+ * kept restarting), each tied by identity to the job pass whose door threw it. The job driver
1856
+ * defers a pass only on one of these, never on an error's shape, which any step or operation
1857
+ * could copy. A mark is ORIGIN and FRESHNESS together: it is consumed when the driver takes it,
1858
+ * and only the pass it names can take it, so a handler that keeps a refusal and throws it again
1859
+ * on a later pass is an ordinary failure there. Held weakly: an error nobody holds goes.
1860
+ */
1861
+ doorWaits = new WeakMap();
1862
+ /** #1834: mark this door's own refusal as "not now" for the pass it was opened for, and hand it back. */
1863
+ doorWait(pass, err) {
1864
+ if (pass)
1865
+ this.doorWaits.set(err, pass);
1866
+ return err;
1867
+ }
1868
+ /** #1834: is `err` a "not now" this host's door threw on `pass`? Consumes the mark when it is. */
1869
+ takeDoorWait(err, pass) {
1870
+ if (typeof err !== 'object' || err === null || this.doorWaits.get(err) !== pass)
1871
+ return false;
1872
+ this.doorWaits.delete(err);
1873
+ return true;
1874
+ }
1875
+ /**
1876
+ * #1834: the system door's gate — the scope's state read for the module, the instance that
1877
+ * answered it, and then (only for a module the scope has on) the rewind hold (#1819). The
1878
+ * hold is read AFTER the state, which is the order `switchHeld` needs.
1879
+ */
1880
+ async systemDoorGate(scopeId, moduleId) {
1881
+ const { state, instance } = await this.scopeStub(scopeId).systemDoorState(moduleId);
1882
+ const hold = state === 'on' ? await this.switchHeld(scopeId, moduleId) : { held: false };
1883
+ return { state, instance, held: hold.held, ...(hold.error ? { error: hold.error } : {}) };
1476
1884
  }
1477
1885
  /**
1478
1886
  * #1705: the consumer half. The same lifecycle gate as every door (K-3), then the ScopeDO
@@ -1503,7 +1911,7 @@ export class CloudflareScopeHost {
1503
1911
  * capability row lives. The answer is re-parsed on this side of the RPC.
1504
1912
  */
1505
1913
  async exchangeCapability(tenantId, scopeId, secret, options) {
1506
- await this.validateScopeAccess(tenantId, scopeId);
1914
+ await this.assertLive(tenantId, scopeId);
1507
1915
  await this.migrateAndRecord(scopeId);
1508
1916
  const outcome = await this.scopeStub(scopeId).exchangeCapability(secret, tenantId, scopeId, options?.mode);
1509
1917
  return capabilityExchange.nullable().parse(outcome);
@@ -1517,7 +1925,7 @@ export class CloudflareScopeHost {
1517
1925
  if (!plausibleSessionToken(sessionToken)) {
1518
1926
  throw substratError('unauthenticated', 'not a capability session token');
1519
1927
  }
1520
- await this.validateScopeAccess(tenantId, scopeId);
1928
+ await this.assertLive(tenantId, scopeId);
1521
1929
  await this.migrateAndRecord(scopeId);
1522
1930
  const hash = await capabilityTokenHash(sessionToken);
1523
1931
  return this.buildStub(tenantId, scopeId, undefined, undefined, undefined, options, undefined, hash);
@@ -1554,7 +1962,7 @@ export class CloudflareScopeHost {
1554
1962
  // provisioning receipt is K-3 here (#1738), read before anything migrates the DO, and the
1555
1963
  // refusal reads as the pure host's does: an unknown scope, not an unknown role.
1556
1964
  if (this.cpLess && !(await this.scopeStub(scopeId).servesTenant(tenantId))) {
1557
- throw substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
1965
+ throw unknownScopeForTenant(tenantId, scopeId);
1558
1966
  }
1559
1967
  await this.peerScopeGate(tenantId, scopeId, 'canAssign');
1560
1968
  const bound = await this.scopeStub(scopeId).canAssignFor(tenantId, scopeId, principalId.parse(principal), roleKey);
@@ -1564,7 +1972,7 @@ export class CloudflareScopeHost {
1564
1972
  }
1565
1973
  async assignScopeRoleBounded(tenantId, scopeId, caller, assignee, roleKey) {
1566
1974
  if (this.cpLess && !(await this.scopeStub(scopeId).servesTenant(tenantId))) {
1567
- throw substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
1975
+ throw unknownScopeForTenant(tenantId, scopeId);
1568
1976
  }
1569
1977
  await this.peerScopeGate(tenantId, scopeId, 'assignScopeRoleBounded');
1570
1978
  const bound = await this.scopeStub(scopeId).assignScopeRoleBoundedFor(tenantId, scopeId, principalId.parse(caller), principalId.parse(assignee), roleKey);
@@ -1589,11 +1997,11 @@ export class CloudflareScopeHost {
1589
1997
  if (!this.cpLess) {
1590
1998
  const record = await this.cp.getScopeRecord(tenantId, scopeId);
1591
1999
  if (!record) {
1592
- throw substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
2000
+ throw unknownScopeForTenant(tenantId, scopeId);
1593
2001
  }
1594
2002
  this.assertServedHere(record, scopeId, verb);
1595
2003
  }
1596
- await this.validateScopeAccess(tenantId, scopeId);
2004
+ await this.assertLive(tenantId, scopeId);
1597
2005
  await this.migrateAndRecord(scopeId);
1598
2006
  }
1599
2007
  /**
@@ -1606,7 +2014,7 @@ export class CloudflareScopeHost {
1606
2014
  if (!plausibleSessionToken(sessionToken)) {
1607
2015
  throw substratError('unauthenticated', 'not a capability session token');
1608
2016
  }
1609
- await this.validateScopeAccess(tenantId, scopeId);
2017
+ await this.assertLive(tenantId, scopeId);
1610
2018
  await this.migrateAndRecord(scopeId);
1611
2019
  const store = await this.resolveAttachmentStore(tenantId);
1612
2020
  const capabilitySession = await capabilityTokenHash(sessionToken);
@@ -1657,7 +2065,7 @@ export class CloudflareScopeHost {
1657
2065
  async capabilityScopeStub(tenantId, scopeId, verb) {
1658
2066
  const rec = await this.cp.getScopeRecord(tenantId, scopeId);
1659
2067
  if (!rec)
1660
- throw substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
2068
+ throw unknownScopeForTenant(tenantId, scopeId);
1661
2069
  if (this.servesScopesElsewhereNow && rec.vertical !== null) {
1662
2070
  throw substratError('unavailable', `${verb} cannot reach scope ${scopeId}: it is served by the '${rec.vertical}' ` +
1663
2071
  'deployment, and platform capability verbs are not delegated there yet');
@@ -1703,6 +2111,12 @@ export class CloudflareScopeHost {
1703
2111
  if (!rec || rec.status !== 'active')
1704
2112
  return report;
1705
2113
  }
2114
+ else if (lifecycleRefusal(await this.validateScopeAccess(tenantId, scopeId)) !== null) {
2115
+ // #2016: the pair is held to the scope's own record first, so a sweep roster entry whose
2116
+ // tenant disagrees with the scope is refused before the probe reads or writes anything.
2117
+ // #1713: a held scope emits nothing, so judging it stale would only raise a false alarm.
2118
+ return report;
2119
+ }
1706
2120
  const stub = this.scopeStub(scopeId);
1707
2121
  const windows = new Map();
1708
2122
  for (const specs of this.moduleFreshness.values()) {
@@ -1741,29 +2155,40 @@ export class CloudflareScopeHost {
1741
2155
  return report;
1742
2156
  // Only run on a live scope of this tenant; a scope archived between the sweep's
1743
2157
  // enumeration and here simply has nothing due. A CP-less host has no directory
1744
- // to ask (#461) — it already trusts the router-asserted (tenant, scope) for the
1745
- // whole request path, and lifecycle for its scopes lives wherever provisioning
1746
- // does, so the grant check below is the only gate it can and does enforce.
2158
+ // to ask (#461), so it reads the lifecycle the platform delivered to the scope (#1713), in
2159
+ // the same call that holds the pair to the scope's own record (#2016): a roster entry whose
2160
+ // tenant disagrees is refused before a grant, a cadence row or a run is read or written.
1747
2161
  if (!this.cpLess) {
1748
2162
  const rec = await this.cp.getScopeRecord(tenantId, scopeId);
1749
2163
  if (!rec || rec.status !== 'active')
1750
2164
  return report;
1751
2165
  }
2166
+ else if (lifecycleRefusal(await this.validateScopeAccess(tenantId, scopeId)) !== null) {
2167
+ // #1713: the lifecycle the platform delivered holds the scope. Every schedule is
2168
+ // `skipped` and no cadence row moves, as under the kill switch, so a schedule that
2169
+ // came due meanwhile fires once on the first pass after the scope is live again.
2170
+ for (const schedule of schedules) {
2171
+ report.skipped += 1;
2172
+ report.runs.push({ operation: schedule.operation, outcome: 'skipped' });
2173
+ }
2174
+ report.lifecycleHeld = true;
2175
+ return report;
2176
+ }
1752
2177
  const stub = this.scopeStub(scopeId);
1753
2178
  // The grant IS the switch (#383), and the kill switch is its lever (#1666): the
1754
2179
  // kernel's `systemScheduleState`, the one predicate both adapters run. A scope that
1755
2180
  // never held the module's grant (a foreign vertical's) is a quiet no-op, exactly as
1756
2181
  // before. A scope switched OFF reports every schedule `skipped`, never `failed`, and
1757
2182
  // does not touch its cadence rows — so a restore fires a due schedule on the next pass.
1758
- const state = await stub.systemScheduleState(moduleId);
1759
- if (state === 'ungranted')
2183
+ // #1819: a scope rewound to before its switch was pulled answers `on`, and the rewind held
2184
+ // the module outside the scope. Both are read by the system door's own gate (#1834), which
2185
+ // every fire below then goes through, pinned to the instance this read came from.
2186
+ const gate = await this.systemDoorGate(scopeId, moduleId);
2187
+ if (gate.state === 'ungranted')
1760
2188
  return report;
1761
- // #1819: a scope rewound to before its switch was pulled answers `on`. The rewind held the
1762
- // module outside the scope; read AFTER the state, which is the order `switchHeld` needs.
1763
- const hold = state === 'on' ? await this.switchHeld(scopeId, moduleId) : { held: false };
1764
- if (hold.error)
1765
- report.errors.push({ operation: 'switch-hold', error: hold.error });
1766
- if (state === 'off' || hold.held) {
2189
+ if (gate.error)
2190
+ report.errors.push({ operation: 'switch-hold', error: gate.error });
2191
+ if (gate.state === 'off' || gate.held) {
1767
2192
  for (const schedule of schedules) {
1768
2193
  report.skipped += 1;
1769
2194
  report.runs.push({ operation: schedule.operation, outcome: 'skipped' });
@@ -1772,6 +2197,8 @@ export class CloudflareScopeHost {
1772
2197
  return report;
1773
2198
  }
1774
2199
  const now = Date.now();
2200
+ const lines = asyncLinePass();
2201
+ let door;
1775
2202
  for (const schedule of schedules) {
1776
2203
  const last = await stub.scheduleLastRun(schedule.operation);
1777
2204
  const lastRun = last ? Date.parse(last) : null;
@@ -1786,28 +2213,55 @@ export class CloudflareScopeHost {
1786
2213
  // produces either way carries it, and so does the row below.
1787
2214
  const invocationId = ulid();
1788
2215
  let status = 'ok';
2216
+ // #1901: the run's line, under the call's own id — the one its `ctx.log` lines carry.
2217
+ const startedAt = Date.now();
2218
+ let emitted;
2219
+ let failure;
1789
2220
  try {
1790
- const scope = await this.getSystemScope(moduleId, tenantId, scopeId);
1791
- await scope.invoke(schedule.operation, schedule.input, { invocationId });
2221
+ // The gate above already answered for this pass; a fire that meets a restarted scope
2222
+ // is gated again by the door (#1834).
2223
+ door ??= await this.openSystemDoor(moduleId, tenantId, scopeId, gate);
2224
+ const scope = this.buildStub(tenantId, scopeId, undefined, undefined, door);
2225
+ await scope.invoke(schedule.operation, schedule.input, { invocationId, onEmitted: (r) => (emitted = r) });
1792
2226
  report.fired += 1;
1793
2227
  }
1794
2228
  catch (err) {
1795
2229
  status = 'failed';
2230
+ failure = { error: err };
1796
2231
  report.failed += 1;
1797
2232
  report.errors.push({
1798
2233
  operation: schedule.operation,
1799
2234
  error: err instanceof Error ? err.message : String(err),
1800
2235
  });
1801
2236
  }
2237
+ lines.write({
2238
+ kind: 'schedule',
2239
+ tenantId,
2240
+ scopeId,
2241
+ invocationId,
2242
+ operation: schedule.operation,
2243
+ startedAt,
2244
+ outcome: status,
2245
+ ...failure,
2246
+ dueAt: lastRun === null ? null : new Date(dueAt).toISOString(),
2247
+ latenessMs: lastRun === null ? null : now - dueAt,
2248
+ ...(emitted ? { emitted } : {}),
2249
+ });
1802
2250
  // #1288: 'schedule', whatever this operation happens to be called — including
1803
2251
  // `freshness:<something>`, which is exactly the row the evaluator no longer eats.
1804
2252
  await stub.recordScheduleRun(schedule.operation, new Date(now).toISOString(), status, 'schedule', invocationId);
1805
2253
  report.runs.push({ operation: schedule.operation, outcome: status === 'ok' ? 'ok' : 'failed' });
1806
2254
  }
2255
+ lines.end();
1807
2256
  return report;
1808
2257
  }
1809
2258
  /** The stub body, shared by the principal, connection, system and impersonation doors. */
1810
- buildStub(tenantId, scopeId, principal, connectionId, systemModuleId, options,
2259
+ buildStub(tenantId, scopeId, principal, connectionId,
2260
+ /**
2261
+ * #1834: the system door this stub acts through. The ONLY way to make a stub acting as
2262
+ * `system:<moduleId>`: the module is the door's, and every invoke is gated and pinned by it.
2263
+ */
2264
+ systemDoor, options,
1811
2265
  /**
1812
2266
  * K-42: the session this stub acts under. Re-read from the directory on
1813
2267
  * EVERY invoke rather than captured here — a stub is a capability and
@@ -1828,6 +2282,7 @@ export class CloudflareScopeHost {
1828
2282
  */
1829
2283
  verticalCaller) {
1830
2284
  const stub = this.scopeStub(scopeId);
2285
+ const systemModuleId = systemDoor?.moduleId;
1831
2286
  const cp = this.cp;
1832
2287
  const operationEntitlement = this.operationEntitlement;
1833
2288
  // The DO needs SOME principal-shaped value for `ctx.principal`; for a
@@ -1882,7 +2337,15 @@ export class CloudflareScopeHost {
1882
2337
  }
1883
2338
  // K-42: resolved per invoke, so expiry and `endImpersonation` both bite.
1884
2339
  const session = sessionId === undefined ? undefined : await this.resolveImpersonation(sessionId, tenantId, scopeId);
1885
- const envelope = await stub.invoke(operation, input, asPrincipalId, tenantId, scopeId, connectionId, requiredKey, systemModuleId, true, invokeOptions, session, capabilitySession, verticalCaller);
2340
+ const send = (systemDoorInstance) => stub.invoke(operation, input, asPrincipalId, tenantId, scopeId, connectionId, requiredKey, systemModuleId, true, invokeOptions, session, capabilitySession, verticalCaller, systemDoorInstance);
2341
+ // #1834: through the system door, pinned to the instance its gate read. The DO's
2342
+ // refusal comes back as a failure envelope, so it is rethrown for the door to re-gate.
2343
+ const envelope = systemDoor
2344
+ ? await systemDoor.through(async (instance) => {
2345
+ const sent = await send(instance);
2346
+ return sent.systemDoorMoved === true ? SYSTEM_DOOR_MOVED : sent;
2347
+ })
2348
+ : await send();
1886
2349
  // The operation failed and the DO handed the error back as DATA — so it still
1887
2350
  // has its code and extensions, which a throw across this boundary would have
1888
2351
  // stripped down to a message (#113 §3). Rethrown here, where the caller expects
@@ -1931,6 +2394,12 @@ export class CloudflareScopeHost {
1931
2394
  throw substratError('unavailable', `${operation} ran on a scope host that did not understand peer callers. ` +
1932
2395
  'Retry once the scope has been migrated');
1933
2396
  }
2397
+ // #1834, the same shape: a DO too old to know the system door's pin ran the call
2398
+ // without checking it met the storage the door's gate read.
2399
+ if (systemDoor !== undefined && envelope.systemDoor?.honoured !== true) {
2400
+ throw substratError('unavailable', `${operation} ran on a scope host that did not understand the system door's pin. ` +
2401
+ 'Retry once the scope has been migrated');
2402
+ }
1934
2403
  if (session !== undefined && envelope.impersonation?.honoured !== true) {
1935
2404
  throw substratError('unavailable', `${operation} ran on a scope host that did not understand impersonation — the ` +
1936
2405
  'operation may have run unrecorded and unbounded. Retry once the scope has ' +
@@ -2125,6 +2594,8 @@ export class CloudflareScopeHost {
2125
2594
  // returns it with the previous status, so the trail cannot disagree with
2126
2595
  // the directory about which deployment the action touched.
2127
2596
  await this.recordAdmin(actor, action, { tenantId, scopeId, vertical: before.vertical }, { status: before.status }, { status: to, ...afterExtra });
2597
+ // #1713: after the directory moved and the move was audited, never before or instead.
2598
+ await this.deliverLifecycles(actor, { scopeId });
2128
2599
  };
2129
2600
  const writeGrant = async (subject, permission, node, entity, expiresAt) => {
2130
2601
  if (entity) {
@@ -2161,7 +2632,7 @@ export class CloudflareScopeHost {
2161
2632
  if (!this.cpLess) {
2162
2633
  const rec = await this.cp.getScopeRecord(tenantId, scopeId);
2163
2634
  if (!rec)
2164
- throw substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
2635
+ throw unknownScopeForTenant(tenantId, scopeId);
2165
2636
  vertical = rec.vertical;
2166
2637
  }
2167
2638
  const action = to === 'off' ? 'revokeFromPeer' : 'restoreToPeer';
@@ -2231,21 +2702,50 @@ export class CloudflareScopeHost {
2231
2702
  if (!this.cpLess) {
2232
2703
  const rec = await this.cp.getScopeRecord(tenantId, scopeId);
2233
2704
  if (!rec)
2234
- throw substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
2705
+ throw unknownScopeForTenant(tenantId, scopeId);
2235
2706
  vertical = rec.vertical;
2236
2707
  }
2237
2708
  const delegation = this.cpLess || vertical !== null ? this.systemSwitchDelegation : undefined;
2238
2709
  // #1819: a scope whose store is here was rewound here, so `switchInScope` releases its
2239
2710
  // hold here too. A delegated move releases in the deployment serving the scope.
2240
- const move = (moduleId, to, at) => delegation
2241
- ? delegation.switch({ tenantId, scopeId, moduleId, to })
2242
- : this.switchInScope(scopeId, moduleId, to, at);
2243
- return { vertical, move };
2711
+ // #1823: a module whose only authority on the scope is a TENANT-level grant has nothing in
2712
+ // the scope's storage for the switch to find, so the directory says whether it is held.
2713
+ // A caller moving several modules reads them in one call and passes each answer in.
2714
+ // An OFF of a tenant-held module is refused, after the far end answers, unless that answer
2715
+ // attests the evaluator that denies a tenant-level grant (`deniesTenantGrants`). A
2716
+ // deployment built before #1823 drops `tenantHeld`, still answers `held: true` (the module's
2717
+ // scope-level grants were there to switch), and keeps authorizing the tenant-level grant —
2718
+ // so its `held` alone would have the platform record a scope off that is not. Every OFF the
2719
+ // platform drives moves through here: an operator's (`switchSystem`, which then puts the
2720
+ // switch back) and a re-assert's, which every carry ends with.
2721
+ const move = async (moduleId, to, at, tenantHeld) => {
2722
+ const held = tenantHeld ?? (await this.cp.tenantHeldSystemModules(tenantId, [moduleId], at)).length > 0;
2723
+ const outcome = await (delegation
2724
+ ? delegation.switch({ tenantId, scopeId, moduleId, to, tenantHeld: held })
2725
+ : this.switchInScope(scopeId, moduleId, to, at, held));
2726
+ if (to === 'off' && held && outcome.held && outcome.deniesTenantGrants !== true) {
2727
+ throw new UnattestedSwitchOff(scopeId, moduleId, outcome);
2728
+ }
2729
+ return outcome;
2730
+ };
2731
+ /**
2732
+ * Is the module switched off on the scope right now? Read from wherever `move` writes — the
2733
+ * deployment's status read or this scope's own — for a move that threw (#1823): the switch
2734
+ * may already have moved before the throw (`switchInScope` moves it, then works the rewind
2735
+ * hold), and then the directory's record must stay.
2736
+ */
2737
+ const isOff = async (moduleId) => {
2738
+ const rows = delegation
2739
+ ? await delegation.status({ tenantId, scopeId })
2740
+ : systemScheduleEntry.array().parse(await this.scopeStub(scopeId).systemGrantsStatus());
2741
+ return rows.some((r) => r.moduleId === moduleId && r.schedules === 'off');
2742
+ };
2743
+ return { vertical, move, isOff };
2244
2744
  };
2245
2745
  const switchSystem = async (actor, raw, to) => {
2246
2746
  const input = systemSwitch.parse(raw);
2247
2747
  const { tenantId, scopeId } = input.node;
2248
- const { vertical, move } = await systemSwitchTarget(tenantId, scopeId);
2748
+ const { vertical, move, isOff } = await systemSwitchTarget(tenantId, scopeId);
2249
2749
  // AUDIT FIRST (#1666 review): the intent row lands before anything moves, and the
2250
2750
  // outcome row after — every attempt, a repeat included. The scope's store and the
2251
2751
  // admin log are separate, so no order makes the pair atomic; this one fails toward
@@ -2256,37 +2756,37 @@ export class CloudflareScopeHost {
2256
2756
  const target = { tenantId, scopeId, vertical };
2257
2757
  const base = { operationId, moduleId: input.moduleId, schedules: to };
2258
2758
  await this.recordAdmin(actor, action, target, null, { ...base, phase: 'intent', reason: input.reason });
2259
- // The directory's record (#1674): ON before the scope moves, OFF after it held — see
2260
- // `recordSystemSwitchedOn` for why that order is the safe one.
2759
+ // The directory's record (#1674), written BEFORE the scope moves, both ways, and undone
2760
+ // if the move throws or holds nothing — see `recordSystemSwitchedOn` and (#1823)
2761
+ // `recordSystemSwitchedOff` for why that order is the safe one.
2261
2762
  const at = new Date().toISOString();
2262
2763
  const record = { tenantId, scopeId, moduleId: input.moduleId, actor, reason: input.reason, operationId, at };
2263
2764
  const errorOf = (err) => (err instanceof Error ? err.message : String(err));
2264
- let prior = null;
2265
- if (to === 'on') {
2266
- try {
2267
- prior = await this.cp.recordSystemSwitchedOn(record);
2268
- }
2269
- catch (err) {
2270
- // Nothing has moved: fail the call here, audited, rather than switch a scope on
2271
- // whose record still says off (the next reconcile would switch it back off).
2272
- await this.recordAdmin(actor, action, target, null, { ...base, phase: 'failed', error: errorOf(err) }).catch(() => undefined);
2273
- throw err;
2274
- }
2765
+ let prior;
2766
+ try {
2767
+ prior = to === 'on' ? await this.cp.recordSystemSwitchedOn(record) : await this.cp.recordSystemSwitchedOff(record);
2768
+ }
2769
+ catch (err) {
2770
+ // Nothing has moved: fail the call here, audited. An ON must not switch a scope on
2771
+ // whose record still says off (the next reconcile would switch it back off), and an
2772
+ // OFF must not leave a scope off that the record does not know of (#1823).
2773
+ await this.recordAdmin(actor, action, target, null, { ...base, phase: 'failed', error: errorOf(err) }).catch(() => undefined);
2774
+ throw err;
2275
2775
  }
2276
2776
  /**
2277
- * A record write AFTER the scope moved (review #5): retried once, and a failure of both
2278
- * is answered rather than swallowed — it lands on the outcome row as `recordError`, and
2279
- * the call does not report plain success. A lost OFF record is exactly what a wipe then
2280
- * turns into a module running again.
2777
+ * The record's undo, after the scope's move threw or held nothing: retried once, and a
2778
+ * failure of both is answered rather than swallowed — it lands on the outcome row as
2779
+ * `recordError`, and the call says so.
2281
2780
  */
2282
- const recordWrite = async (write) => {
2781
+ const undoRecord = async () => {
2782
+ const undo = () => this.cp.restoreSystemSwitchRecord(record, prior);
2283
2783
  try {
2284
- await write();
2784
+ await undo();
2285
2785
  return null;
2286
2786
  }
2287
2787
  catch {
2288
2788
  try {
2289
- await write();
2789
+ await undo();
2290
2790
  return null;
2291
2791
  }
2292
2792
  catch (err) {
@@ -2299,25 +2799,54 @@ export class CloudflareScopeHost {
2299
2799
  outcome = await move(input.moduleId, to, at);
2300
2800
  }
2301
2801
  catch (err) {
2302
- const recordError = prior ? await recordWrite(() => this.cp.restoreSystemSwitchRecord(record, prior)) : null;
2802
+ if (err instanceof UnattestedSwitchOff) {
2803
+ // The far end switched, and cannot hold it against the tenant-level grant. Refused:
2804
+ // ON gives back exactly what this OFF took (nothing, when it changed nothing), and
2805
+ // then the record goes back too — neither the scope nor the directory says off. An ON
2806
+ // that fails here leaves the scope off, and the readback rule below keeps its record.
2807
+ const putBack = err.outcome.changed
2808
+ ? await move(input.moduleId, 'on', new Date().toISOString(), true).then(() => true, () => false)
2809
+ : true;
2810
+ if (putBack) {
2811
+ const recordError = await undoRecord();
2812
+ await this.recordAdmin(actor, action, target, null, {
2813
+ ...base,
2814
+ phase: 'refused',
2815
+ error: err.message,
2816
+ ...(recordError ? { recordError } : {}),
2817
+ }).catch(() => undefined);
2818
+ throw substratError('precondition_failed', `${err.message} Nothing was switched` +
2819
+ (recordError ? `, but its directory record could not be put back (${recordError}).` : '.'));
2820
+ }
2821
+ }
2822
+ // A throw does not say the scope did not move: the move crosses the scope's DO and then
2823
+ // the rewind hold, and a throw from the second leaves the scope's switch moved. So the
2824
+ // record is undone only when the scope reads back in the OTHER position — an OFF that
2825
+ // reads not off, an ON that still reads off — and kept when it reads back as asked.
2826
+ // Undoing an ON that landed would put the record back to `off` beside a scope that is
2827
+ // on, and the next re-assert would switch the operator's ON back off. A read that fails
2828
+ // too counts as off, both ways, so what is left fails toward off: an OFF keeps its
2829
+ // record (never a scope switched off with no record — a wipe or restore would have
2830
+ // nothing to re-assert, and #1743's refusal nothing to read), and an ON is undone back
2831
+ // to `off`, which the next re-assert completes if the scope did move.
2832
+ const off = await isOff(input.moduleId).catch(() => true);
2833
+ const recordKept = off === (to === 'off');
2834
+ const recordError = recordKept ? null : await undoRecord();
2303
2835
  // Best effort: the original error is what the caller must see, and the intent row
2304
2836
  // already says an attempt was made.
2305
2837
  await this.recordAdmin(actor, action, target, null, {
2306
2838
  ...base,
2307
2839
  phase: 'failed',
2308
2840
  error: errorOf(err),
2841
+ ...(recordKept ? { recordKept: true } : {}),
2309
2842
  ...(recordError ? { recordError } : {}),
2310
2843
  }).catch(() => undefined);
2311
2844
  throw err;
2312
2845
  }
2313
- // OFF is recorded once the scope held it. A refused ON moved nothing, so its record
2314
- // write is undone: left `on`, the next reconcile of a wiped scope would leave the
2315
- // module running.
2316
- const recordError = to === 'off' && outcome.held
2317
- ? await recordWrite(() => this.cp.recordSystemSwitchedOff(record))
2318
- : to === 'on' && !outcome.held && prior
2319
- ? await recordWrite(() => this.cp.restoreSystemSwitchRecord(record, prior))
2320
- : null;
2846
+ // A call that held nothing moved nothing, so its record write is undone: left `off`, it
2847
+ // would switch the module off the day it is installed; left `on`, the next reconcile of
2848
+ // a wiped scope would leave the module running.
2849
+ const recordError = outcome.held ? null : await undoRecord();
2321
2850
  await this.recordAdmin(actor, action, target, null, {
2322
2851
  ...base,
2323
2852
  phase: outcome.held ? 'applied' : 'refused',
@@ -2330,11 +2859,6 @@ export class CloudflareScopeHost {
2330
2859
  `(check the module id: it is the module's manifest id, e.g. '@substrat-run/engine-absence')` +
2331
2860
  (recordError ? `; and its directory record could not be put back (${recordError})` : ''));
2332
2861
  }
2333
- if (recordError) {
2334
- throw substratError('unavailable', `module '${input.moduleId}' is switched ${to} on scope ${scopeId}, but the directory's record of it ` +
2335
- `could not be written (${recordError}) — repeat the call to record it; a wipe of this scope would ` +
2336
- `otherwise lose the switch`);
2337
- }
2338
2862
  return {
2339
2863
  operationId,
2340
2864
  moduleId: input.moduleId,
@@ -2398,7 +2922,7 @@ export class CloudflareScopeHost {
2398
2922
  if (!this.cpLess) {
2399
2923
  const rec = await this.cp.getScopeRecord(tenantId, scopeId);
2400
2924
  if (!rec)
2401
- throw substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
2925
+ throw unknownScopeForTenant(tenantId, scopeId);
2402
2926
  vertical = rec.vertical;
2403
2927
  }
2404
2928
  // A hosted scope (a vertical bound, on a non-cpLess host) has no local storage worth
@@ -2485,8 +3009,9 @@ export class CloudflareScopeHost {
2485
3009
  });
2486
3010
  }
2487
3011
  const results = [];
3012
+ const tenantHeld = new Set(modules.length ? await this.cp.tenantHeldSystemModules(tenantId, modules, at) : []);
2488
3013
  for (const moduleId of modules) {
2489
- const outcome = await move(moduleId, 'off', at);
3014
+ const outcome = await move(moduleId, 'off', at, tenantHeld.has(moduleId));
2490
3015
  if (outcome.changed) {
2491
3016
  await this.recordAdmin(actor, 'reassertSystemSwitch', { tenantId, scopeId, vertical }, null, {
2492
3017
  operationId: ulid(),
@@ -2516,7 +3041,7 @@ export class CloudflareScopeHost {
2516
3041
  if (!this.cpLess) {
2517
3042
  const rec = await this.cp.getScopeRecord(tenantId, scopeId);
2518
3043
  if (!rec)
2519
- throw substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
3044
+ throw unknownScopeForTenant(tenantId, scopeId);
2520
3045
  vertical = rec.vertical;
2521
3046
  }
2522
3047
  if (!this.cpLess && vertical !== null && !this.peerSwitchDelegation) {
@@ -2715,6 +3240,13 @@ export class CloudflareScopeHost {
2715
3240
  await this.recordAccess(actor, 'listSystemSwitches', { tenantId: filter?.tenantId ?? null }, filter ?? null, rows.length);
2716
3241
  return rows.map((r) => systemSwitchRecord.parse(r));
2717
3242
  },
3243
+ tenantHeldSystemModules: async (actor, tenantId, moduleIds) => {
3244
+ const held = moduleIds.length
3245
+ ? await this.cp.tenantHeldSystemModules(tenantId, moduleIds, new Date().toISOString())
3246
+ : [];
3247
+ await this.recordAccess(actor, 'tenantHeldSystemModules', { tenantId }, { moduleIds: [...moduleIds] }, held.length);
3248
+ return held;
3249
+ },
2718
3250
  reassertSystemSwitches: reassertSystemSwitchesOf,
2719
3251
  // #1672 — the platform's two capability verbs. Audited AFTER the write, on both, and
2720
3252
  // the failure each leaves is the safe one: a mint whose audit row did not land never
@@ -3126,7 +3658,7 @@ export class CloudflareScopeHost {
3126
3658
  if (channel === 'prod' && !incoming.deployment_ref) {
3127
3659
  const owning = await this.cp.readVertical(verticalSlug);
3128
3660
  if (owning && owning.owner_tenant !== null && !owning.listed) {
3129
- const bound = (await this.cp.listScopes({ tenantId: owning.owner_tenant, vertical: verticalSlug, status: ['active'] })).filter((s) => !s.forked_from && s.kind !== 'preview');
3661
+ const bound = (await this.cp.listScopes({ tenantId: owning.owner_tenant, vertical: verticalSlug, status: ['active'] })).filter(isPrimaryScopeRow);
3130
3662
  for (const s of bound) {
3131
3663
  if (s.vertical_version_id === versionId)
3132
3664
  continue;
@@ -3284,6 +3816,12 @@ export class CloudflareScopeHost {
3284
3816
  await this.cp.setScopeServingRef(scopeId, servingRef);
3285
3817
  await this.recordAdmin(actor, 'setScopeServingRef', { tenantId, scopeId }, { servingRef: scope.serving_ref ?? null }, { servingRef, ...(ack.exportBreak ? { acknowledged: ack } : {}) });
3286
3818
  },
3819
+ recordKeptCopyResolution: async (actor, tenantId, scopeId, r) => {
3820
+ const scope = await this.cp.getScopeRecord(tenantId, scopeId);
3821
+ if (!scope)
3822
+ throw substratError('not_found', `unknown scope ${scopeId} in tenant ${tenantId}`);
3823
+ await this.recordAdmin(actor, 'resolveKeptCopy', { tenantId, scopeId }, { script: r.script, keptAt: r.keptAt, revision: r.revisionBefore }, { action: r.action, liveScript: r.liveScript, revision: r.revisionAfter });
3824
+ },
3287
3825
  setScopeExpiresAt: async (actor, tenantId, scopeId, expiresAt) => {
3288
3826
  const scope = await this.cp.getScopeRecord(tenantId, scopeId);
3289
3827
  if (!scope)
@@ -3388,6 +3926,8 @@ export class CloudflareScopeHost {
3388
3926
  setTenantStatus: async (actor, tenantId, status) => {
3389
3927
  const before = await this.cp.setTenantStatus(tenantId, status);
3390
3928
  await this.recordAdmin(actor, 'setTenantStatus', { tenantId }, { status: before }, { status });
3929
+ // #1713: every hosted scope under the tenant holds or resumes its work by this.
3930
+ await this.deliverLifecycles(actor, { tenantId });
3391
3931
  },
3392
3932
  setTenantName: async (actor, tenantId, name) => {
3393
3933
  const before = await this.cp.setTenantName(tenantId, name);
@@ -4260,6 +4800,11 @@ export class CloudflareScopeHost {
4260
4800
  const { tenantId, scopeId, ...after } = ownerTransferAudit.parse(entry);
4261
4801
  await this.recordAdmin(actor, 'transferOwner', { tenantId, scopeId }, null, after);
4262
4802
  },
4803
+ /** #2005: one change to a scope's copy marker, written around the vertical's own change. */
4804
+ recordCopyMark: async (actor, entry) => {
4805
+ const { tenantId, scopeId, action, ...after } = copyMarkAudit.parse(entry);
4806
+ await this.recordAdmin(actor, action === 'mark' ? 'markScopeCopy' : 'clearScopeCopyMark', { tenantId, scopeId }, null, after);
4807
+ },
4263
4808
  accessLog: async (actor, filter) => {
4264
4809
  const rows = await this.cp.accessLog({
4265
4810
  actor: filter?.actor,
@@ -4472,14 +5017,225 @@ export class CloudflareScopeHost {
4472
5017
  * The getScope gate (control-plane.md §4.1/§4.2) — the pair check, then the tenant's and
4473
5018
  * the scope's lifecycle — thrown on THIS side of the RPC (#1718). The ControlPlaneDO
4474
5019
  * answers its refusal as data, because an error thrown there arrives here flattened, with
4475
- * no code; a record crosses intact. A CP-less host's null control plane answers nothing,
4476
- * so the gate passes: the router already made it from the shared directory.
5020
+ * no code; a record crosses intact.
5021
+ *
5022
+ * A CP-less host's null control plane answers nothing: the router made the pair check from
5023
+ * the shared directory. The scope's own storage then makes it again (#2016), against the
5024
+ * tenant it was provisioned for (`ScopeDO.admission`), so a vertical that passes a scope id
5025
+ * under the wrong tenant meets K-3's refusal at the door instead of relying on the permission
5026
+ * gate's grant data to deny it.
5027
+ *
5028
+ * Answers with the lifecycle the CP-less scope holds (#1713), read in the same DO call as the
5029
+ * pair check, so an `assertLive` door still costs one round-trip; a door that reads the scope
5030
+ * without the lifecycle half pays the one DO call the pair check needs. Null with a directory,
5031
+ * whose refusal already covered the lifecycle.
4477
5032
  */
4478
5033
  async validateScopeAccess(tenantId, scopeId) {
4479
5034
  const refusal = await this.cp.scopeAccessRefusal(tenantId, scopeId);
4480
- if (!refusal)
4481
- return;
4482
- throw refusal.code ? substratError(refusal.code, refusal.message) : new Error(refusal.message);
5035
+ if (refusal)
5036
+ throw refusal.code ? substratError(refusal.code, refusal.message) : new Error(refusal.message);
5037
+ if (!this.cpLess)
5038
+ return null;
5039
+ const { verdict, lifecycle } = await this.scopeStub(scopeId).admission(tenantId);
5040
+ if (verdict === 'foreign')
5041
+ throw unknownScopeForTenant(tenantId, scopeId);
5042
+ // A scope with no tenant record and no role rows has nothing to hold the pair against, so it is
5043
+ // let through — a compatibility state, not a steady one: the next reconcile (every push runs
5044
+ // one against the scopes behind it, #1172) or lifecycle delivery records the directory's tenant.
5045
+ // Logged on every admission, so an operator sees a scope that has not converged.
5046
+ if (verdict === 'unknown')
5047
+ console.warn(JSON.stringify(tenantUnrecordedLine(tenantId, scopeId)));
5048
+ return lifecycle;
5049
+ }
5050
+ /**
5051
+ * THE lifecycle gate (#1713) at every door that runs a scope's work: a request's stubs and
5052
+ * attachments, a capability or impersonation session, a peer call or delivery, a subscription,
5053
+ * and the entry points no request asked for — the retry driver, the job runner, a schedule's
5054
+ * system stub, a routed connector delivery.
5055
+ *
5056
+ * `validateScopeAccess` first, which is the directory's gate where there is a directory. A
5057
+ * CP-less host has none, so it then reads the lifecycle the platform delivered to the scope's
5058
+ * own storage (`/internal/lifecycle`), judged by the kernel's `lifecycleRefusal` in the
5059
+ * directory's own words. The router refuses a suspended scope's requests first (#1730); this
5060
+ * is the deployment's own half, and the only one a timer or a retry ever meets.
5061
+ *
5062
+ * The admin and diagnostic reads (dead letters, job runs, platform requests, delivered
5063
+ * grants) keep `validateScopeAccess` alone: an operator still reads a suspended scope.
5064
+ */
5065
+ async assertLive(tenantId, scopeId) {
5066
+ const refusal = lifecycleRefusal(await this.validateScopeAccess(tenantId, scopeId), { tenantId, scopeId });
5067
+ if (refusal)
5068
+ throw new Error(refusal);
5069
+ }
5070
+ /**
5071
+ * Deliver the directory's lifecycle to the deployments serving the matching scopes (#1713):
5072
+ * one scope after its transition, a tenant's scopes after the tenant's, or the heal sweep's
5073
+ * drift. Each delivery carries the directory's revisions, read in the same query as the
5074
+ * statuses, so a late one cannot undo a later transition: the deployment keeps only a delivery
5075
+ * strictly newer than the one it holds.
5076
+ *
5077
+ * Never throws. A transition is the operator's lever in an incident, and the router refuses
5078
+ * the scope's requests as soon as the directory moves (#1730), so a deployment that cannot be
5079
+ * reached must not refuse the lever. Each failure is an ops-failure row, no receipt is written,
5080
+ * and the heal sweep delivers again.
5081
+ */
5082
+ async deliverLifecycles(actor, filter) {
5083
+ const report = { attempted: 0, delivered: 0, failed: 0 };
5084
+ const delegation = this.lifecycleDelegation;
5085
+ if (!delegation || this.cpLess)
5086
+ return report;
5087
+ let targets;
5088
+ // #2016: the heal also asks the served scopes whose deployment has not answered that it holds a
5089
+ // record of the scope's tenant, a copy as much as a primary (no push reconciles a copy), until
5090
+ // each answers that it does. A bounded slice per pass, after the drift and the holds.
5091
+ const askTenant = new Set();
5092
+ try {
5093
+ targets = await this.cp.lifecycleTargets(filter);
5094
+ if (filter.drift) {
5095
+ const unrecorded = await this.cp.lifecycleTargets({ unrecorded: true, limit: TENANT_UNRECORDED_PER_PASS });
5096
+ // Noted before asking, so an ask that fails (or a deployment that never answers) rotates
5097
+ // behind the scopes not asked yet, and the next pass reaches them.
5098
+ if (unrecorded.length > 0)
5099
+ await this.cp.recordTenantAsks(unrecorded.map((u) => u.scope_id), new Date().toISOString());
5100
+ const seen = new Set(targets.map((t) => t.scope_id));
5101
+ for (const u of unrecorded) {
5102
+ askTenant.add(u.scope_id);
5103
+ if (!seen.has(u.scope_id))
5104
+ targets.push(u);
5105
+ }
5106
+ }
5107
+ }
5108
+ catch (err) {
5109
+ console.error('substrat: could not read the scopes to deliver a lifecycle to (#1713)', err);
5110
+ return report;
5111
+ }
5112
+ for (const first of targets) {
5113
+ const tenantId = first.tenant_id;
5114
+ const scopeId = first.scope_id;
5115
+ let t = first;
5116
+ let lifecycle = this.lifecycleOfTarget(t);
5117
+ if (!lifecycle) {
5118
+ // A status this code does not know (a newer directory): delivering a guess could lift a hold.
5119
+ report.failed += 1;
5120
+ console.error(`substrat: scope ${scopeId} has a lifecycle this code cannot read (#1713)`);
5121
+ continue;
5122
+ }
5123
+ // A live scope whose deployment already acknowledged exactly this has nothing to receive,
5124
+ // and neither does one with no receipt in a directory never restored: a deployment holding
5125
+ // no lifecycle runs live. That keeps an activation, and every transition before a
5126
+ // deployment carries the route, from posting a delivery that changes nothing. After a
5127
+ // restore (epoch > 0) no receipt says nothing about what the deployment holds, so it is
5128
+ // delivered. A HOLD is always delivered, and so is a scope the heal asks for its tenant record.
5129
+ if (!askTenant.has(scopeId) &&
5130
+ lifecycleRefusal(lifecycle) === null &&
5131
+ (t.delivered === null ? t.epoch === 0 : t.delivered === lifecycleReceipt(lifecycle))) {
5132
+ continue;
5133
+ }
5134
+ report.attempted += 1;
5135
+ try {
5136
+ let answer = await delegation.deliver({ tenantId, scopeId, lifecycle });
5137
+ // SINGLE AUTHORITY (the invariant this rests on): an environment has exactly one directory,
5138
+ // the singleton `CONTROL_PLANE.idFromName('control-plane')`. A "fresh directory" restore is
5139
+ // that same object restored after its storage was lost, never a second live writer, and
5140
+ // the raise below always re-reads the CURRENT store and delivers what it says. Two control
5141
+ // planes healing one dispatch namespace at once is a split brain that corrupts far more
5142
+ // than lifecycle, and is out of scope here; fencing it in lifecycle alone would guarantee
5143
+ // nothing.
5144
+ //
5145
+ // The scope refused us as OLDER while holding a revision this directory's history did not
5146
+ // write: an epoch ahead of ours, or ours with other counters. Only another history of the
5147
+ // directory delivers those — the one a fresh-directory restore replaced, whose epoch a
5148
+ // clock running behind (or a restore in the same millisecond) cannot outrank. A scope only
5149
+ // ever holds epochs a directory minted, so this directory learns past it: its epoch is
5150
+ // raised above the one held (monotonic, one statement), and the scope is delivered again.
5151
+ const held = answer.lifecycle.revision;
5152
+ const foreign = !answer.applied &&
5153
+ held !== undefined &&
5154
+ (held.epoch > lifecycle.revision.epoch ||
5155
+ (held.epoch === lifecycle.revision.epoch && lifecycleReceipt(answer.lifecycle) !== lifecycleReceipt(lifecycle)));
5156
+ // Bounded, because the deployment answering is the vertical's own code: only a directory
5157
+ // that has been restored can meet another history (epoch > 0), and a legitimate epoch is
5158
+ // a mint time, so one further ahead than the skew is forged or broken. Either way nothing
5159
+ // is raised; the refusal is an ops failure for an operator to look at.
5160
+ if (foreign && (lifecycle.revision.epoch === 0 || held.epoch > Date.now() + LIFECYCLE_EPOCH_SKEW_MS)) {
5161
+ report.failed += 1;
5162
+ await this.admin
5163
+ .recordOpsFailure({
5164
+ actor,
5165
+ operation: 'scope.lifecycle',
5166
+ stage: 'foreign-epoch',
5167
+ tenantId,
5168
+ scopeId,
5169
+ message: `the deployment refused this directory's lifecycle while holding epoch ${held.epoch} ` +
5170
+ `(this directory is at ${lifecycle.revision.epoch}); not raised past it — ` +
5171
+ (lifecycle.revision.epoch === 0
5172
+ ? 'this directory has never been restored, so no other history should have written it'
5173
+ : 'it is further ahead of the clock than any directory mints'),
5174
+ })
5175
+ .catch((e) => console.error('substrat: could not record a foreign lifecycle epoch (#1713)', e));
5176
+ continue;
5177
+ }
5178
+ if (foreign) {
5179
+ await this.cp.raiseLifecycleEpoch(held.epoch + 1);
5180
+ const [again] = await this.cp.lifecycleTargets({ scopeId });
5181
+ const next = again ? this.lifecycleOfTarget(again) : null;
5182
+ if (again && next) {
5183
+ t = again;
5184
+ lifecycle = next;
5185
+ answer = await delegation.deliver({ tenantId, scopeId, lifecycle });
5186
+ }
5187
+ }
5188
+ // The receipt is what the scope HOLDS: a newer delivery it kept instead is the truth.
5189
+ await this.cp.recordLifecycleReceipt(scopeId, lifecycleReceipt(answer.lifecycle), new Date().toISOString(), answer.tenantRecorded === true);
5190
+ report.delivered += 1;
5191
+ }
5192
+ catch (err) {
5193
+ report.failed += 1;
5194
+ try {
5195
+ await this.admin.recordOpsFailure({
5196
+ actor,
5197
+ operation: 'scope.lifecycle',
5198
+ stage: 'deliver',
5199
+ tenantId,
5200
+ scopeId,
5201
+ message: `the lifecycle (scope ${t.scope_status}, tenant ${t.tenant_status}) did not reach the ` +
5202
+ `deployment serving this scope; the heal sweep will deliver it again: ${err instanceof Error ? err.message : String(err)}`,
5203
+ });
5204
+ }
5205
+ catch (recordErr) {
5206
+ console.error('substrat: could not record a lifecycle delivery failure (#1713)', recordErr);
5207
+ }
5208
+ }
5209
+ }
5210
+ return report;
5211
+ }
5212
+ /** One directory row as the lifecycle a delivery carries (#1713), or null for a status this code does not know. */
5213
+ lifecycleOfTarget(t) {
5214
+ const parsed = scopeLifecycle.safeParse({
5215
+ scope: t.scope_status,
5216
+ tenant: t.tenant_status,
5217
+ at: new Date().toISOString(),
5218
+ revision: { epoch: t.epoch, scope: t.scope_rev, tenant: t.tenant_rev },
5219
+ });
5220
+ return parsed.success ? parsed.data : null;
5221
+ }
5222
+ /**
5223
+ * The heal sweep (#1713): deliver the lifecycle to every hosted scope whose deployment's
5224
+ * acknowledged state differs from the directory, and again to every scope held now — which is
5225
+ * what puts a hold back on a store a carry or a restore landed without it. Bounded per pass
5226
+ * (`limit`), drifted scopes first. Run from the shared control plane's cron.
5227
+ */
5228
+ async healLifecycles(actor, opts) {
5229
+ return this.deliverLifecycles(actor, { drift: true, limit: opts?.limit ?? 200 });
5230
+ }
5231
+ /**
5232
+ * Whether a CP-less host holds this scope's work (#1713): `assertLive`'s predicate, answered
5233
+ * rather than thrown, for the entry points that defer instead of refusing — a sweep pass, a
5234
+ * schedule run, an executor drain. Always false on a host with a directory, whose entry
5235
+ * points ask the directory themselves.
5236
+ */
5237
+ async lifecycleHeld(scopeId) {
5238
+ return this.cpLess && lifecycleRefusal(await this.scopeStub(scopeId).lifecycle()) !== null;
4483
5239
  }
4484
5240
  /**
4485
5241
  * K-3's cross-check on its own: the (tenant, scope) pair must exist and agree before a
@@ -4489,7 +5245,7 @@ export class CloudflareScopeHost {
4489
5245
  async assertScope(tenantId, scopeId) {
4490
5246
  const rec = await this.cp.getScopeRecord(tenantId, scopeId);
4491
5247
  if (!rec)
4492
- throw substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
5248
+ throw unknownScopeForTenant(tenantId, scopeId);
4493
5249
  }
4494
5250
  /**
4495
5251
  * The same K-3 cross-check, for a read that then opens the scope's STORAGE — every
@@ -4504,7 +5260,7 @@ export class CloudflareScopeHost {
4504
5260
  async scopeRecordForRead(tenantId, scopeId) {
4505
5261
  const row = await this.cp.getScopeRecord(tenantId, scopeId);
4506
5262
  if (!row)
4507
- throw substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
5263
+ throw unknownScopeForTenant(tenantId, scopeId);
4508
5264
  if (row.status === 'reaped') {
4509
5265
  throw new Error(`scope ${scopeId} is reaped — its storage is gone and cannot be read`);
4510
5266
  }
@@ -4785,7 +5541,7 @@ export class CloudflareScopeHost {
4785
5541
  * its claims before the OFF's join lands. The joined row then holds the module the operator just
4786
5542
  * turned ON, until the next surviving move of that switch releases it (another ON does).
4787
5543
  */
4788
- async switchInScope(scopeId, moduleId, to, at) {
5544
+ async switchInScope(scopeId, moduleId, to, at, tenantHeld = false) {
4789
5545
  let scopeClaims;
4790
5546
  let readError;
4791
5547
  try {
@@ -4794,7 +5550,7 @@ export class CloudflareScopeHost {
4794
5550
  catch (err) {
4795
5551
  readError = err;
4796
5552
  }
4797
- const { instance, ...outcome } = await this.scopeStub(scopeId).switchSystemSchedules(moduleId, scopeId, to, at);
5553
+ const { instance, ...outcome } = await this.scopeStub(scopeId).switchSystemSchedules(moduleId, scopeId, to, at, tenantHeld);
4798
5554
  if (!scopeClaims)
4799
5555
  throw readError;
4800
5556
  const now = Date.now();
@@ -4997,16 +5753,16 @@ export class CloudflareScopeHost {
4997
5753
  * suspended or archiving scope could still be addressed and handed a 101 — a
4998
5754
  * scope that refuses every ordinary read while quietly holding an open socket.
4999
5755
  *
5000
- * Costs a CP-less vertical nothing: `validateScopeAccess` is a **no-op** on the
5001
- * null control plane (the router already gated lifecycle and tenancy from the
5002
- * shared directory), so this is the hosted-vertical shape unchanged.
5756
+ * On a CP-less vertical, `validateScopeAccess` is a no-op on the null control
5757
+ * plane, and `assertLive` reads the lifecycle the platform delivered to the
5758
+ * scope's own storage instead (#1713).
5003
5759
  *
5004
5760
  * `migrateAndRecord`, not the DO's own `ensureMigrations`: the DO migrates
5005
5761
  * itself when the socket opens either way, but only this reports the applied
5006
5762
  * count to the directory — so a scope whose first contact after a deploy is a
5007
5763
  * subscription does not go dark in the migration fleet view.
5008
5764
  */
5009
- await this.validateScopeAccess(tenantId, scopeId);
5765
+ await this.assertLive(tenantId, scopeId);
5010
5766
  await this.migrateAndRecord(scopeId);
5011
5767
  // Asserted, not carried through from the client: the principal is the
5012
5768
  // vertical's own resolution of its session, and the tenant and scope are the
@@ -5094,7 +5850,7 @@ export class CloudflareScopeHost {
5094
5850
  // good. Read first, so a pair the directory does not hold writes nothing.
5095
5851
  const scope = await this.cp.getScopeRecord(tenantId, scopeId);
5096
5852
  if (!scope)
5097
- throw substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
5853
+ throw unknownScopeForTenant(tenantId, scopeId);
5098
5854
  const { roles, tuples, entitlements, identities } = await this.tenantProjection(tenantId);
5099
5855
  const connectionKeys = await this.connectionKeyRows(tenantId, scope.vertical);
5100
5856
  await this.scopeStub(scopeId).applyProjection(tenantId, roles, tuples, entitlements, undefined, identities, connectionKeys);
@@ -5272,7 +6028,7 @@ export class CloudflareScopeHost {
5272
6028
  // #1742: the recorded-off modules, switched off after the seat in the same unit, on the
5273
6029
  // scope this call provisions — the one the seat's tuples name.
5274
6030
  input.switchedOff
5275
- ? { scopeId: input.scopeId, moduleIds: input.switchedOff, at: new Date().toISOString() }
6031
+ ? { scopeId: input.scopeId, moduleIds: input.switchedOff, at: new Date().toISOString(), tenantHeld: input.tenantHeld }
5276
6032
  : undefined, services?.map((id) => `principal:${id}`));
5277
6033
  return input.switchedOff ? { switchedOff } : {};
5278
6034
  }
@@ -5369,14 +6125,19 @@ export class CloudflareScopeHost {
5369
6125
  // side before the call; what runs HERE is the half only this deployment can
5370
6126
  // enforce: the scope's own permission check against its delivered
5371
6127
  // `connection:<id>` tuple, in the scope's own DO. Fail closed — no grant, no
5372
- // effect — exactly as for any other caller.
6128
+ // effect — exactly as for any other caller. And the lifecycle delivered to the
6129
+ // scope (#1713): a held scope's invoke and bytes doors refuse, as every other door does.
5373
6130
  /** Invoke ONE operation in this deployment as a CONNECTION (#574). */
5374
6131
  async connectorInvokeLocal(connectionId, tenantId, scopeId, operation, input) {
6132
+ await this.assertLive(tenantId, scopeId); // #1713: a held scope runs no connector work either
5375
6133
  await this.migrateAndRecord(scopeId);
5376
6134
  return this.buildStub(tenantId, scopeId, undefined, connectionId).invoke(operation, input);
5377
6135
  }
5378
6136
  /** Land provider bytes in this deployment as a CONNECTION — the bytes leg (#574). */
5379
6137
  async connectorAttachmentUploadLocal(connectionId, tenantId, scopeId, upload) {
6138
+ // The lifecycle gate every attachments door takes (#1713), ahead of the bucket: a held
6139
+ // scope's provider bytes wait for the delivery's retry rather than land (#1995).
6140
+ await this.assertLive(tenantId, scopeId);
5380
6141
  await this.migrateAndRecord(scopeId);
5381
6142
  const store = await this.resolveAttachmentStore(tenantId);
5382
6143
  return this.buildAttachmentSurface({ connectionId }, tenantId, scopeId, store).upload(upload);
@@ -5389,6 +6150,8 @@ export class CloudflareScopeHost {
5389
6150
  * so a caller falls back rather than failing a dispatch over a missing file.
5390
6151
  */
5391
6152
  async connectorAttachmentOpenLocal(connectionId, tenantId, scopeId, attachmentId, eventId) {
6153
+ // Same gate as the upload leg: a held scope's bytes are not read out of its bucket.
6154
+ await this.assertLive(tenantId, scopeId);
5392
6155
  await this.migrateAndRecord(scopeId);
5393
6156
  const store = await this.resolveAttachmentStore(tenantId);
5394
6157
  return this.buildAttachmentSurface({ connectionId }, tenantId, scopeId, store,
@@ -5414,11 +6177,11 @@ export class CloudflareScopeHost {
5414
6177
  * admin log and writes the row once this returns. `held: false` is an answer, not a
5415
6178
  * throw — see `systemSwitchOutcome`.
5416
6179
  */
5417
- async systemSwitchLocal(scopeId, moduleId, to) {
6180
+ async systemSwitchLocal(scopeId, moduleId, to, opts) {
5418
6181
  // Parsed on the way out: this is the wire answer the platform reads, and the DO's
5419
6182
  // plain strings become the published shape here rather than on trust.
5420
6183
  // #1819: `switchInScope` releases a rewind's hold on the module once the switch lands.
5421
- return systemSwitchOutcome.parse(await this.switchInScope(scopeId, moduleId, to, new Date().toISOString()));
6184
+ return systemSwitchOutcome.parse(await this.switchInScope(scopeId, moduleId, to, new Date().toISOString(), opts?.tenantHeld ?? false));
5422
6185
  }
5423
6186
  /**
5424
6187
  * The far end of the status read for a scope served HERE (#1674): `/internal/system-grants`
@@ -5519,7 +6282,7 @@ export class CloudflareScopeHost {
5519
6282
  }
5520
6283
  const rec = await this.cp.getScopeRecord(tenantId, scopeId);
5521
6284
  if (!rec)
5522
- throw substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
6285
+ throw unknownScopeForTenant(tenantId, scopeId);
5523
6286
  const source = await importCursorSourceOf((t, v) => this.admin.resolveVerticalInstance(t, v), { tenantId, scopeId, vertical: rec.vertical }, move.from);
5524
6287
  // Where the write lands is the peer switch's rule: a scope bound to a vertical is served by
5525
6288
  // that vertical's deployment, and its watermark lives there. Without a delegation that