@substrat-run/adapter-cloudflare 0.135.0 → 0.136.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/host.js CHANGED
@@ -1,7 +1,8 @@
1
+ import { env as ambientEnv } from 'cloudflare:workers';
1
2
  import { isRewindRefusal, REWIND_REFUSED } from './rewind-refusal.js';
2
- import { delegatedReadParams, fromWireFailure, exportReadInput, exportedBatch, importBatch, importResult, importState, importsOfManifestJson, exportsOfManifestJson, importCursorMove, importCursorMoveAt, importCursorMoved, accessLogEntry, adminLogEntry, opsFailureEntry, opsFailureFingerprint, issueEntry, sweepRunEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunsPayload, modelUsageEntry, attachmentRecord, createTenantInput, identityLink, identityPool, createOrgInput, promotionAcknowledgement, bindAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, registerVerticalInput, vertical as verticalSchema, verticalServingState, verticalChannel, verticalVersion, connection, capabilityExchange, capabilityGrant, principalId, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, systemSwitch, systemSwitchOutcome, systemScheduleEntry, systemSwitchRecord, entitlementGrant, entitlementGrantInput, capabilityFilter, instant, meterReading, subjectRef, createConnectionInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, org as orgSchema, orgMembership, resolvedIdentity, identityMembership, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, delegatedReadRecord, ownerTransferAudit, subjectShredReceipt, connectorDispatchKind, callsOfManifestJson, outboundOfManifestJson, substratError, redrainEventsInput, coverage, peerCoverage, peerGrantsEntry, peerSwitch, peerSwitchOutcome, verticalCaller, verticalResolution, verticalSlug, entityObjectRef, } from '@substrat-run/contracts';
3
+ import { delegatedReadParams, fromWireFailure, exportReadInput, exportedBatch, importBatch, importResult, importState, importsOfManifestJson, exportsOfManifestJson, importCursorMove, importCursorMoveAt, importCursorMoved, accessLogEntry, adminLogEntry, opsFailureEntry, opsFailureFingerprint, issueEntry, sweepRunEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunsPayload, modelUsageEntry, attachmentRecord, createTenantInput, identityLink, identityPool, createOrgInput, promotionAcknowledgement, bindAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, registerVerticalInput, vertical as verticalSchema, verticalServingState, verticalChannel, verticalVersion, connection, capabilityExchange, capabilityGrant, principalId, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, systemSwitch, systemSwitchOutcome, systemScheduleEntry, systemSwitchRecord, entitlementGrant, entitlementGrantInput, capabilityFilter, instant, meterReading, subjectRef, createConnectionInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, org as orgSchema, orgMembership, resolvedIdentity, identityMembership, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, delegatedReadRecord, ownerTransferAudit, copyMarkAudit, subjectShredReceipt, connectorDispatchKind, callsOfManifestJson, outboundOfManifestJson, substratError, redrainEventsInput, coverage, peerCoverage, peerGrantsEntry, peerSwitch, peerSwitchOutcome, verticalCaller, verticalResolution, verticalSlug, entityObjectRef, ATTACHMENT_BLOB_BINDING, blobStoreBindingName, lifecycleDelivery, scopeLifecycle, } from '@substrat-run/contracts';
3
4
  import { normalizeHostname, toRouteTarget } from './route-resolver.js';
4
- import { attachmentBlobKey, attachmentSha256, entitlementDenial, requiredEntitlementFor, foldMeterReading, parseValidationRecords, resolveScopeRecord, ulid, capabilityTokenHash, checkBecomeInput, plausibleSessionToken, foldModelUsage, backoffAt, resolveRetryPolicy, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, openSealed, ImpersonationRefused, assertSessionUsable, impersonationRowValues, mapImpersonationRow, newImpersonationSession, jobRunOf, runDueJobRuns, startJobRun, attachmentTextJob, assertAttachmentExtractors, assertJobRegistrable, isAttachmentTextRun, searchLimit, searchMatchExpression, globalFetch, assertRedrainWindow, platformRequestOf, undrainedEventsOf, inUnitMovesToAudit, staleCarryRevertRow, staleCarryReverts, withRecorded, systemSwitchedOffMessage, tenantSystemSwitchedOffMessage, CrossVerticalRegistry, importCursorSourceOf, exportBreaksOf, exportBreakRefusal, bindExportBreaksOf, bindExportBreakRefusal, collectPeers, peerSeats, connectorCallRecord, noopConnectorCallRecorder, recordConnectorCall, settleConnectionUse, splitManifestMigrations, unknownRoleError, assertRowLimit, assertRowOffset, } from '@substrat-run/kernel';
5
+ import { attachmentBlobKey, attachmentSha256, entitlementDenial, requiredEntitlementFor, foldMeterReading, parseValidationRecords, resolveScopeRecord, ulid, KEPT_COPY_REFUSAL, capabilityTokenHash, checkBecomeInput, plausibleSessionToken, foldModelUsage, backoffAt, resolveRetryPolicy, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, openSealed, ImpersonationRefused, assertSessionUsable, impersonationRowValues, mapImpersonationRow, newImpersonationSession, jobRunOf, runDueJobRuns, startJobRun, attachmentTextJob, assertAttachmentExtractors, assertJobRegistrable, isAttachmentTextRun, searchLimit, searchMatchExpression, globalFetch, assertRedrainWindow, platformRequestOf, undrainedEventsOf, inUnitMovesToAudit, staleCarryRevertRow, staleCarryReverts, withRecorded, systemSwitchedOffMessage, tenantSystemSwitchedOffMessage, CrossVerticalRegistry, importCursorSourceOf, exportBreaksOf, exportBreakRefusal, bindExportBreaksOf, bindExportBreakRefusal, collectPeers, peerSeats, connectorCallRecord, noopConnectorCallRecorder, recordConnectorCall, settleConnectionUse, splitManifestMigrations, unknownRoleError, assertRowLimit, assertRowOffset, INERT_SCOPE_REASON, lifecycleReceipt, lifecycleRefusal, isPrimaryScope, isPrimaryScopeRow, asyncInvocationId, asyncLinePass, } from '@substrat-run/kernel';
5
6
  import { attributedHost } from '@substrat-run/kernel';
6
7
  import { isOrangeToOrange, isUpgradeRequest, LIVE_MODE_HEADER, LIVE_PRINCIPAL_HEADER, LIVE_SCOPE_HEADER, LIVE_SUBSCRIBE_PATH, LIVE_TENANT_HEADER, } from './live-reads.js';
7
8
  import { tenantStoreDatabaseName } from './d1.js';
@@ -30,12 +31,29 @@ function unwrapCapabilityReply(reply) {
30
31
  throw fromWireFailure(reply.failure);
31
32
  return reply.value;
32
33
  }
34
+ /**
35
+ * How far ahead of the clock a held lifecycle epoch may be and still be learned past (#1713). An
36
+ * epoch is the time a directory restore minted it, so a legitimate one is behind its minter's
37
+ * clock; the skew allows for clocks that disagree, and bounds what a forged answer can move.
38
+ */
39
+ export const LIFECYCLE_EPOCH_SKEW_MS = 24 * 60 * 60 * 1000;
40
+ /**
41
+ * Refuse to mark a scope a copy unless the request classifies it non-primary (#2005). The
42
+ * classification is the platform directory's; the vertical applies the same `isPrimaryScope` to
43
+ * it, so a primary can be marked only by a request that misstates its lineage, never by default.
44
+ */
45
+ function assertCopyLineage(lineage) {
46
+ if (isPrimaryScope(lineage)) {
47
+ throw substratError('conflict', 'mark-copy refused: the directory classifies this scope as primary, not as a copy');
48
+ }
49
+ }
33
50
  /**
34
51
  * A control-plane stand-in for a CP-less vertical (scope-local-permissions.md Phase 3).
35
52
  * The hot path a served scope actually touches becomes trust-the-upstream:
36
- * - `scopeAccessRefusal` / `setMigrationState` → no-op: the router already gated the
37
- * scope's lifecycle + tenancy from the shared directory, so the vertical trusts the
38
- * asserted node rather than re-reading a directory it does not have.
53
+ * - `scopeAccessRefusal` / `setMigrationState` → no-op: the router gates the scope's
54
+ * lifecycle + tenancy from the shared directory, and the vertical trusts the asserted
55
+ * node rather than re-reading a directory it does not have. Its own half of the
56
+ * lifecycle gate reads the scope's storage instead (`assertLive`, #1713).
39
57
  * - `tenantHoldsEntitlement` → true: the SKU was enforced on the shared control plane
40
58
  * at provision (before `provisionInstance`), so a scope that EXISTS here was granted
41
59
  * it upstream — a single-vertical deployment holds its own entitlements by construction.
@@ -106,6 +124,14 @@ export const SWITCH_HOLD_EXTRA_WAITS = 3;
106
124
  * clock, against the releasing host's clock: minutes, not seconds.
107
125
  */
108
126
  export const SWITCH_HOLD_PENDING_MAX_MS = 5 * 60_000;
127
+ /**
128
+ * The attachment bucket a host resolves when it was handed no resolver (#1995): the
129
+ * platform's per-tenant binding, read off this script's own env, named by the one shared
130
+ * encoding (`blobStoreBindingName`) for the tenant the caller passed in.
131
+ */
132
+ function ambientAttachmentBucket(tenantId) {
133
+ return ambientEnv[blobStoreBindingName(ATTACHMENT_BLOB_BINDING, tenantId)] ?? null;
134
+ }
109
135
  export class CloudflareScopeHost {
110
136
  admin;
111
137
  scopeNs;
@@ -145,7 +171,7 @@ export class CloudflareScopeHost {
145
171
  tenantStores;
146
172
  /** The live R2 client for per-tenant blob stores (#473); undefined ⇒ refuse loudly. */
147
173
  blobStores;
148
- /** Worker-side attachment-bucket resolver (#473); undefined ⇒ attachments() refuses. */
174
+ /** Worker-side attachment-bucket resolver (#473); the ambient per-tenant binding unless overridden (#1995). */
149
175
  attachmentBuckets;
150
176
  /** The parsers attachment text is extracted with (K-43); the host's own, never imported here. */
151
177
  attachmentExtractors;
@@ -180,6 +206,8 @@ export class CloudflareScopeHost {
180
206
  eventDrainDelegation;
181
207
  /** #1666: the schedule kill switch's reach into the deployment serving a scope. */
182
208
  systemSwitchDelegation;
209
+ /** #1713: the lifecycle's reach into the deployment serving a scope. */
210
+ lifecycleDelegation;
183
211
  /** #1706: the peer kill switch's reach into the deployment serving a scope. */
184
212
  peerSwitchDelegation;
185
213
  /** #1705 PR 3: the replay lever's reach into the deployment serving a consumer scope. */
@@ -211,7 +239,7 @@ export class CloudflareScopeHost {
211
239
  this.secretBox = options.secretBox ?? unconfiguredSecretBox;
212
240
  this.tenantStores = options.tenantStores;
213
241
  this.blobStores = options.blobStores;
214
- this.attachmentBuckets = options.attachmentBuckets;
242
+ this.attachmentBuckets = options.attachmentBuckets ?? ambientAttachmentBucket;
215
243
  assertAttachmentExtractors(options.attachmentExtractors ?? []);
216
244
  this.attachmentExtractors = options.attachmentExtractors ?? [];
217
245
  this.fetchImpl = options.fetch ?? globalFetch;
@@ -225,6 +253,7 @@ export class CloudflareScopeHost {
225
253
  this.connectorDelegation = options.connectorDelegation;
226
254
  this.eventDrainDelegation = options.eventDrainDelegation;
227
255
  this.systemSwitchDelegation = options.systemSwitchDelegation;
256
+ this.lifecycleDelegation = options.lifecycleDelegation;
228
257
  this.peerSwitchDelegation = options.peerSwitchDelegation;
229
258
  this.importCursorDelegation = options.importCursorDelegation;
230
259
  this.admin = this.buildAdmin();
@@ -353,21 +382,78 @@ export class CloudflareScopeHost {
353
382
  };
354
383
  if (this.executors.size === 0)
355
384
  return report;
385
+ // #1901: one line per attempt, capped per pass. Written on the coordinator, where the
386
+ // attempt runs — the DO only journals it.
387
+ const lines = asyncLinePass();
388
+ try {
389
+ await this.drainExecutorsPass(tenantId, scopeId, invocationId, report, lines);
390
+ }
391
+ finally {
392
+ lines.end();
393
+ }
394
+ return report;
395
+ }
396
+ async drainExecutorsPass(tenantId, scopeId, invocationId, report, lines) {
356
397
  const stub = this.scopeStub(scopeId);
357
- for (const [id, executor] of this.executors) {
398
+ // #2005: a non-primary scope (a fork, a snapshot, a preview of either kind) causes no
399
+ // outbound effects, so its deliveries are journaled terminal with the reason and no
400
+ // handler runs. Asked of the directory where there is one. A CP-less host has none, so it
401
+ // asks the scope's own storage whether it is classified a copy (`_substrat_copy_origin`'s
402
+ // `is_copy`, #2009, set on the directory's word) — a preview and a snapshot reach a hosted
403
+ // vertical as restores the platform marks.
404
+ // Asked on the first due event, once per pass: most passes have none, and the directory
405
+ // is one global object.
406
+ let inert;
407
+ const isInert = () => (inert ??= this.cpLess
408
+ ? stub.isCopy()
409
+ : this.cp.getScopeRecord(tenantId, scopeId).then((row) => !isPrimaryScopeRow(row)));
410
+ // #1713: a CP-less scope its lifecycle holds attempts nothing and journals nothing, so every
411
+ // due delivery stays due for the first pass after it is live again. Asked on the first due
412
+ // event, like `isInert`. Every caller passed `assertLive` already, so this is for the scope
413
+ // suspended between that gate and the drain: a request in flight, whose write commits while
414
+ // its effect waits. A host with a directory answers false without a read.
415
+ let held;
416
+ const isHeld = () => (held ??= this.lifecycleHeld(scopeId));
417
+ drain: for (const [id, executor] of this.executors) {
358
418
  const deliveryId = `executor:${id}`;
359
419
  const { events, undecodable } = await stub.pendingExecutorDeliveries(deliveryId, executor.eventType);
360
420
  // #1636: an event the DO could not decode is dead-lettered for this executor at once,
361
421
  // and its handler never sees it. Terminal on the FIRST failure, unlike a handler's:
362
422
  // the decode is pure, so a retry cannot succeed. The rows behind it are delivered
363
423
  // below — the decode used to throw the whole list, on every pass.
424
+ if ((undecodable.length > 0 || events.length > 0) && (await isHeld())) {
425
+ report.lifecycleHeld = true;
426
+ break drain;
427
+ }
428
+ // #1901: an attempt's line. The attempt number is the one the journal is about to record.
429
+ const unitOf = (eventId, attempt) => ({
430
+ kind: 'consumer',
431
+ tenantId,
432
+ scopeId,
433
+ invocationId: asyncInvocationId(invocationId),
434
+ operation: deliveryId,
435
+ eventType: executor.eventType,
436
+ eventId,
437
+ attempt,
438
+ startedAt: Date.now(),
439
+ });
364
440
  for (const bad of undecodable) {
365
441
  report.attempted += 1;
366
- await stub.recordExecutorAttempt(bad.eventId, deliveryId, bad.error, null, invocationId);
442
+ const attempt = await stub.recordExecutorAttempt(bad.eventId, deliveryId, bad.error, null, invocationId);
367
443
  report.deadLettered += 1;
444
+ lines.write({ ...unitOf(bad.eventId, attempt), outcome: 'dead-lettered' });
368
445
  }
369
446
  for (const event of events) {
370
447
  report.attempted += 1;
448
+ const startedAt = Date.now();
449
+ if (await isInert()) {
450
+ // No next attempt, like an undecodable row: a scope does not become primary.
451
+ const attempt = await stub.recordExecutorAttempt(event.id, deliveryId, INERT_SCOPE_REASON, null, invocationId);
452
+ report.inert = (report.inert ?? 0) + 1;
453
+ // Its own outcome, never `delivered`: no handler ran (#2005).
454
+ lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'inert' });
455
+ continue;
456
+ }
371
457
  this.causedBy = event.id;
372
458
  try {
373
459
  if (executor.kind === 'connector' && this.cpLess) {
@@ -381,16 +467,20 @@ export class CloudflareScopeHost {
381
467
  // the at-least-once residue, as it already must in-process.
382
468
  await stub.routeExecutorEventToPlatform(event.id, deliveryId, connectorDispatchKind(executor.provider), JSON.stringify({ executorId: id, event }), JSON.stringify({ system: 'connector-dispatch' }), invocationId);
383
469
  report.routedToPlatform += 1;
470
+ // Handed on, not run here: the platform's drain owns its attempts from now on.
471
+ lines.write({ ...unitOf(event.id, 1), startedAt, outcome: 'routed' });
384
472
  }
385
473
  else if (executor.kind === 'connector') {
386
474
  await executor.handler(await this.connectorContext(tenantId, scopeId, executor.timeoutMs, event.id), event);
387
- await stub.recordExecutorAttempt(event.id, deliveryId, null, null, invocationId);
475
+ const attempt = await stub.recordExecutorAttempt(event.id, deliveryId, null, null, invocationId);
388
476
  report.delivered += 1;
477
+ lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'delivered' });
389
478
  }
390
479
  else {
391
480
  await executor.handler(this.admin, event);
392
- await stub.recordExecutorAttempt(event.id, deliveryId, null, null, invocationId);
481
+ const attempt = await stub.recordExecutorAttempt(event.id, deliveryId, null, null, invocationId);
393
482
  report.delivered += 1;
483
+ lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'delivered' });
394
484
  }
395
485
  }
396
486
  catch (err) {
@@ -405,18 +495,23 @@ export class CloudflareScopeHost {
405
495
  report.deadLettered += 1;
406
496
  else
407
497
  report.retrying += 1;
498
+ lines.write({
499
+ ...unitOf(event.id, attempts),
500
+ startedAt,
501
+ outcome: exhausted ? 'dead-lettered' : 'retrying',
502
+ error: err,
503
+ });
408
504
  }
409
505
  finally {
410
506
  this.causedBy = null;
411
507
  }
412
508
  }
413
509
  }
414
- return report;
415
510
  }
416
511
  async drainDue(tenantId, scopeId) {
417
512
  // Same lifecycle gate `getScope` applies (K-3): a suspended or archived scope
418
513
  // does not get its effects driven either.
419
- await this.validateScopeAccess(tenantId, scopeId);
514
+ await this.assertLive(tenantId, scopeId);
420
515
  await this.migrateAndRecord(scopeId);
421
516
  // #1525: null, and honestly so — a sweep is not a call. An attempt this pass makes
422
517
  // records no invocation, which is what distinguishes it from the first attempt the
@@ -474,14 +569,14 @@ export class CloudflareScopeHost {
474
569
  this.attachmentExtractors);
475
570
  }
476
571
  async startJobRun(tenantId, scopeId, input) {
477
- await this.validateScopeAccess(tenantId, scopeId);
572
+ await this.assertLive(tenantId, scopeId);
478
573
  await this.migrateAndRecord(scopeId);
479
574
  return jobRunOf(await startJobRun(this.jobStore(scopeId), input, ulid, () => new Date().toISOString()));
480
575
  }
481
576
  async runDueJobs(tenantId, scopeId, options) {
482
577
  // Same lifecycle gate as `drainDue`: a suspended scope's runs wait rather than
483
578
  // advance, and an archived one's never move again.
484
- await this.validateScopeAccess(tenantId, scopeId);
579
+ await this.assertLive(tenantId, scopeId);
485
580
  await this.migrateAndRecord(scopeId);
486
581
  // #1575: the kernel's extraction job is this host's own, bound to this scope — no
487
582
  // deployment registers it, and none can shadow it.
@@ -506,7 +601,7 @@ export class CloudflareScopeHost {
506
601
  // same context build as the in-process path, so the handler cannot tell which host
507
602
  // ran it. On a CP-less host `connectorContext` throws from the null control plane:
508
603
  // fail closed, exactly the hole routing exists to avoid.
509
- await this.validateScopeAccess(tenantId, scopeId);
604
+ await this.assertLive(tenantId, scopeId);
510
605
  this.causedBy = event.id;
511
606
  try {
512
607
  await handler(await this.connectorContext(tenantId, scopeId, options?.timeoutMs ?? 30_000, event.id), event);
@@ -653,7 +748,9 @@ export class CloudflareScopeHost {
653
748
  */
654
749
  async snapshotScopeLocal(sourceScopeId, destScopeId) {
655
750
  const tables = await this.scopeStub(sourceScopeId).exportDump();
656
- await this.scopeStub(destScopeId).importDump(tables, destScopeId, { sourceScopeId, exact: true });
751
+ // #2009: a snapshot is a fork by construction (the control plane's row names `forkedFrom`, so
752
+ // the directory never calls it primary), and a load classifies nothing by itself: marked here.
753
+ await this.scopeStub(destScopeId).importDump(tables, destScopeId, { sourceScopeId, exact: true, markCopy: true });
657
754
  return { tables: tables.length };
658
755
  }
659
756
  /**
@@ -666,14 +763,131 @@ export class CloudflareScopeHost {
666
763
  /** #1742: the directory's recorded-off modules, switched off on THIS scope in the replay's
667
764
  * own event — a dump from before a switch was pulled brings its grants back live. #1869:
668
765
  * `sourceScopeId`, the scope the dump was captured from, narrows the grant re-point to it;
669
- * `exact` says the platform exported the dump itself, so the re-point never falls back. */
766
+ * `exact` says the platform exported the dump itself, so the re-point never falls back.
767
+ * #1722: `loadStamp`, the stamp a carry leaves on the copy it lands, for a fenced wipe later,
768
+ * and `expect`, the marker the carry read here: the load is refused if the store moved since.
769
+ * #2005: `markCopy` — the directory's classification of the scope, sent when it is not
770
+ * primary; the restore then marks it a copy. Refused if it classifies a primary. */
670
771
  opts) {
671
- const switchedOff = await this.scopeStub(scopeId).importDump(tables, scopeId, {
772
+ if (opts?.markCopy)
773
+ assertCopyLineage(opts.markCopy);
774
+ const load = {
672
775
  switchOff: opts?.switchedOff ? { moduleIds: opts.switchedOff, at: new Date().toISOString() } : undefined,
673
776
  sourceScopeId: opts?.sourceScopeId,
674
777
  exact: opts?.exact,
778
+ loadStamp: opts?.loadStamp,
779
+ markCopy: opts?.markCopy !== undefined,
780
+ };
781
+ // Both refusals come back as values (a throw over the RPC carries only its message) and are
782
+ // thrown here, in the vertical's own isolate, so its route answers 409 or 412, not a fault.
783
+ const out = await this.scopeStub(scopeId).importDumpChecked(tables, scopeId, {
784
+ ...load,
785
+ ...(opts?.expect ? { expect: opts.expect } : {}),
786
+ });
787
+ if (out.refused === false) {
788
+ return { tables: tables.length, ...(opts?.switchedOff ? { switchedOff: out.switchedOff } : {}) };
789
+ }
790
+ if (out.refused === 'kept')
791
+ throw substratError('conflict', KEPT_COPY_REFUSAL);
792
+ // A retry of a load that already committed (its answer was lost on the way back) is
793
+ // refused by the marker that load itself moved. The store holding THIS request's stamp
794
+ // says so: no other load writes it. Answered as applied, without `switchedOff`, so the
795
+ // caller's re-assert after the bind covers the OFF positions.
796
+ const now = await this.scopeStub(scopeId).loadMarker();
797
+ if (!opts?.loadStamp || now.loadStamp !== opts.loadStamp) {
798
+ throw substratError('precondition_failed', 'scope store changed since the carry read it; nothing was loaded (#1722)');
799
+ }
800
+ return { tables: tables.length };
801
+ }
802
+ /** The kept-copy marker of this scope's store in THIS deployment (#1722), or null. */
803
+ async keptCopyLocal(scopeId) {
804
+ return this.scopeStub(scopeId).keptCopy();
805
+ }
806
+ /**
807
+ * Discard the kept copy of a scope in THIS deployment (#1722), behind the vertical's
808
+ * `/internal/kept-copy/discard`: the staff resolution, only at the write revision the operator
809
+ * acted on. Answers whether it was discarded, or why not.
810
+ */
811
+ async discardKeptCopyLocal(scopeId, revision, carriedAway,
812
+ /** #2005: the directory's classification, sent when the scope is not primary. */
813
+ markCopy,
814
+ /** #1722 (Codex #2008 r13): the load stamp read with `revision`; a replaced store is refused. */
815
+ loadStamp) {
816
+ if (markCopy)
817
+ assertCopyLineage(markCopy);
818
+ return this.scopeStub(scopeId).discardKeptCopy(scopeId, revision, carriedAway, markCopy !== undefined, ...(loadStamp !== undefined ? [loadStamp] : []));
819
+ }
820
+ /**
821
+ * What a carry's restore into this scope's store expects to find unchanged (#1722), behind
822
+ * the vertical's `/internal/load-marker`. Read before the carry checks the binding again; the
823
+ * restore then carries it as `expect`, and is refused if the store was loaded or written since.
824
+ */
825
+ async loadMarkerLocal(scopeId) {
826
+ return this.scopeStub(scopeId).loadMarker();
827
+ }
828
+ /**
829
+ * Wipe the copy a carry left in THIS deployment (#1722), behind the vertical's
830
+ * `/internal/wipe-carried`: only if nothing was loaded into the scope's DO since the stamp
831
+ * the carry read (`expectLoadStamp`, null for a store no load has stamped). Non-terminal,
832
+ * so a later bind back to this version can restore into it. False when refused.
833
+ */
834
+ async wipeCarriedLocal(scopeId, expectLoadStamp, carriedAway, opts = {}) {
835
+ if (opts.markCopy)
836
+ assertCopyLineage(opts.markCopy);
837
+ return this.scopeStub(scopeId).wipeCarried(scopeId, expectLoadStamp, carriedAway, {
838
+ ...(opts.expectRevision !== undefined ? { expectRevision: opts.expectRevision } : {}),
839
+ ...(opts.protectIfChanged !== undefined ? { protectIfChanged: opts.protectIfChanged } : {}),
840
+ markCopy: opts.markCopy !== undefined,
675
841
  });
676
- return { tables: tables.length, ...(opts?.switchedOff ? { switchedOff } : {}) };
842
+ }
843
+ /** Release a kept copy that is the live store after all (#1722), at the revision read. */
844
+ async releaseKeptCopyLocal(scopeId, revision,
845
+ /** #2005: the directory's classification, sent when the scope is not primary. */
846
+ markCopy,
847
+ /** #1722 (Codex #2008 r13): the load stamp read with `revision`; a replaced store is refused. */
848
+ loadStamp) {
849
+ if (markCopy)
850
+ assertCopyLineage(markCopy);
851
+ return this.scopeStub(scopeId).releaseKeptCopy(revision, markCopy !== undefined, ...(loadStamp !== undefined ? [loadStamp] : []));
852
+ }
853
+ /**
854
+ * Store the lifecycle the platform delivered for one scope in THIS deployment (#1713), behind
855
+ * the vertical's `/internal/lifecycle`. Kept unless the scope already holds a newer one, so a
856
+ * push that arrives late cannot undo a later transition. `applied` is what the platform's heal
857
+ * sweep records as delivered; `changed` says the gate's answer moved.
858
+ */
859
+ async setLifecycleLocal(scopeId, next) {
860
+ return lifecycleDelivery.parse(await this.scopeStub(scopeId).setLifecycle(scopeLifecycle.parse(next)));
861
+ }
862
+ /**
863
+ * Mark one scope in THIS deployment a copy (#2005), behind the vertical's `/internal/mark-copy`:
864
+ * the repair of a copy that predates the marker, which a CP-less coordinator reads for primacy.
865
+ * The control plane decides which scopes (its directory says they are not primary) and audits.
866
+ */
867
+ async markCopyLocal(scopeId, lineage) {
868
+ assertCopyLineage(lineage);
869
+ return { marked: await this.scopeStub(scopeId).markCopy() };
870
+ }
871
+ /**
872
+ * Clear a mistaken copy classification from one scope in THIS deployment (#2005), behind the
873
+ * vertical's `/internal/clear-copy-mark`: staff's correction for a scope the directory says IS
874
+ * primary. Refused for a scope classified a copy. A load's copied-events mark is never touched
875
+ * (#2009), so a primary restored from another scope's backup runs its own effects again and
876
+ * still never runs the work it copied.
877
+ */
878
+ async clearCopyMarkLocal(scopeId, lineage,
879
+ /** #1722 (Codex #2008 r12–r13): the store as the caller means it, its load stamp and write
880
+ * revision, compared in the clear's own transaction; refused (412) if either moved. The
881
+ * platform's reconcile of a carry's destination sends it; staff send none. */
882
+ expect) {
883
+ if (!isPrimaryScope(lineage)) {
884
+ throw substratError('conflict', 'clear-copy-mark refused: the directory classifies this scope as a copy (a preview or a fork)');
885
+ }
886
+ const outcome = await this.scopeStub(scopeId).clearCopyMark(...(expect ? [expect] : []));
887
+ if (outcome === 'changed') {
888
+ throw substratError('precondition_failed', `clear-copy-mark refused: scope ${scopeId}'s store changed since its revision was read`);
889
+ }
890
+ return { cleared: outcome === 'cleared' };
677
891
  }
678
892
  /**
679
893
  * Re-apply the vertical's OWN role definitions to one scope, CP-lessly — the
@@ -710,6 +924,13 @@ export class CloudflareScopeHost {
710
924
  async exportScopeLocal(scopeId) {
711
925
  return this.scopeStub(scopeId).exportDump();
712
926
  }
927
+ /**
928
+ * `exportScopeLocal` with the store's load stamp, read in the same DO call (#1722): what a
929
+ * carry's fenced wipe of the copy it leaves here expects. Behind the vertical's `/internal/export`.
930
+ */
931
+ async exportScopeStampedLocal(scopeId) {
932
+ return this.scopeStub(scopeId).exportDumpStamped();
933
+ }
713
934
  /** Facet this host's own scope's outbox (#1239) — the vertical-host read. */
714
935
  async facetEventsLocal(scopeId, input) {
715
936
  return this.scopeStub(scopeId).facetEvents(input);
@@ -1065,7 +1286,7 @@ export class CloudflareScopeHost {
1065
1286
  // gates and metadata facts live in the ScopeDO (per-scope serialization, spine event
1066
1287
  // in the same transaction); bytes go straight to the per-tenant R2 bucket through the
1067
1288
  // binding the vertical's worker resolved — never through the DO.
1068
- await this.validateScopeAccess(tenantId, scopeId);
1289
+ await this.assertLive(tenantId, scopeId);
1069
1290
  await this.migrateAndRecord(scopeId);
1070
1291
  const store = await this.resolveAttachmentStore(tenantId);
1071
1292
  return this.buildAttachmentSurface({ principal }, tenantId, scopeId, store);
@@ -1138,17 +1359,14 @@ export class CloudflareScopeHost {
1138
1359
  const store = await this.resolveAttachmentStore(conn.tenant_id);
1139
1360
  return this.buildAttachmentSurface({ connectionId }, conn.tenant_id, scopeId, store, forEvent);
1140
1361
  }
1141
- /** Resolve the per-tenant R2 blob store, or fail closed exactly as `attachments` did. */
1362
+ /** Resolve the per-tenant R2 blob store, or fail closed. */
1142
1363
  async resolveAttachmentStore(tenantId) {
1143
- if (!this.attachmentBuckets) {
1144
- throw new Error(`attachments are not configured on this host (#473): pass ` +
1145
- `CloudflareScopeHostOptions.attachmentBuckets — (tenantId) => ` +
1146
- `env[blobStoreBindingName('<BINDING>', tenantId)] for the vertical's declared blob store`);
1147
- }
1148
1364
  const bucket = await this.attachmentBuckets(tenantId);
1149
1365
  if (!bucket) {
1150
- throw new Error(`no attachment bucket resolved for tenant ${tenantId} (#473) — is the per-tenant blob ` +
1151
- `store provisioned and its r2_bucket binding attached to the serving script?`);
1366
+ throw new Error(`no attachment bucket resolved for tenant ${tenantId} (#473, #1995) — this script has no ` +
1367
+ `${blobStoreBindingName(ATTACHMENT_BLOB_BINDING, tenantId)} binding. A vertical whose modules declare ` +
1368
+ `attachmentTargets gets one per installed tenant once it is pushed with a CLI that declares the ` +
1369
+ `store (re-push); off the platform, pass CloudflareScopeHostOptions.attachmentBuckets.`);
1152
1370
  }
1153
1371
  return r2TenantBlobStore(bucket);
1154
1372
  }
@@ -1381,7 +1599,7 @@ export class CloudflareScopeHost {
1381
1599
  async getScope(principal, tenantId, scopeId, options) {
1382
1600
  // Lifecycle gates (control-plane.md §4.1/§4.2), the K-3 fail-closed path,
1383
1601
  // evaluated durably in the ControlPlaneDO. A throw propagates here.
1384
- await this.validateScopeAccess(tenantId, scopeId);
1602
+ await this.assertLive(tenantId, scopeId);
1385
1603
  await this.migrateAndRecord(scopeId);
1386
1604
  return this.buildStub(tenantId, scopeId, principal, undefined, undefined, options);
1387
1605
  }
@@ -1456,7 +1674,7 @@ export class CloudflareScopeHost {
1456
1674
  const record = await this.resolveImpersonation(session, tenantId, scopeId);
1457
1675
  // The same lifecycle gate the principal door applies: a suspended tenant
1458
1676
  // refuses a support session exactly as it refuses a user.
1459
- await this.validateScopeAccess(tenantId, scopeId);
1677
+ await this.assertLive(tenantId, scopeId);
1460
1678
  await this.migrateAndRecord(scopeId);
1461
1679
  return this.buildStub(tenantId, scopeId, record.principal, undefined, undefined, options, record.id);
1462
1680
  }
@@ -1470,7 +1688,7 @@ export class CloudflareScopeHost {
1470
1688
  if (!this.moduleIds.has(moduleId)) {
1471
1689
  throw new Error(`module not registered on this host: ${moduleId}`);
1472
1690
  }
1473
- await this.validateScopeAccess(tenantId, scopeId);
1691
+ await this.assertLive(tenantId, scopeId);
1474
1692
  await this.migrateAndRecord(scopeId);
1475
1693
  return this.buildStub(tenantId, scopeId, undefined, undefined, moduleId);
1476
1694
  }
@@ -1503,7 +1721,7 @@ export class CloudflareScopeHost {
1503
1721
  * capability row lives. The answer is re-parsed on this side of the RPC.
1504
1722
  */
1505
1723
  async exchangeCapability(tenantId, scopeId, secret, options) {
1506
- await this.validateScopeAccess(tenantId, scopeId);
1724
+ await this.assertLive(tenantId, scopeId);
1507
1725
  await this.migrateAndRecord(scopeId);
1508
1726
  const outcome = await this.scopeStub(scopeId).exchangeCapability(secret, tenantId, scopeId, options?.mode);
1509
1727
  return capabilityExchange.nullable().parse(outcome);
@@ -1517,7 +1735,7 @@ export class CloudflareScopeHost {
1517
1735
  if (!plausibleSessionToken(sessionToken)) {
1518
1736
  throw substratError('unauthenticated', 'not a capability session token');
1519
1737
  }
1520
- await this.validateScopeAccess(tenantId, scopeId);
1738
+ await this.assertLive(tenantId, scopeId);
1521
1739
  await this.migrateAndRecord(scopeId);
1522
1740
  const hash = await capabilityTokenHash(sessionToken);
1523
1741
  return this.buildStub(tenantId, scopeId, undefined, undefined, undefined, options, undefined, hash);
@@ -1593,7 +1811,7 @@ export class CloudflareScopeHost {
1593
1811
  }
1594
1812
  this.assertServedHere(record, scopeId, verb);
1595
1813
  }
1596
- await this.validateScopeAccess(tenantId, scopeId);
1814
+ await this.assertLive(tenantId, scopeId);
1597
1815
  await this.migrateAndRecord(scopeId);
1598
1816
  }
1599
1817
  /**
@@ -1606,7 +1824,7 @@ export class CloudflareScopeHost {
1606
1824
  if (!plausibleSessionToken(sessionToken)) {
1607
1825
  throw substratError('unauthenticated', 'not a capability session token');
1608
1826
  }
1609
- await this.validateScopeAccess(tenantId, scopeId);
1827
+ await this.assertLive(tenantId, scopeId);
1610
1828
  await this.migrateAndRecord(scopeId);
1611
1829
  const store = await this.resolveAttachmentStore(tenantId);
1612
1830
  const capabilitySession = await capabilityTokenHash(sessionToken);
@@ -1703,6 +1921,10 @@ export class CloudflareScopeHost {
1703
1921
  if (!rec || rec.status !== 'active')
1704
1922
  return report;
1705
1923
  }
1924
+ else if (await this.lifecycleHeld(scopeId)) {
1925
+ // #1713: a held scope emits nothing, so judging it stale would only raise a false alarm.
1926
+ return report;
1927
+ }
1706
1928
  const stub = this.scopeStub(scopeId);
1707
1929
  const windows = new Map();
1708
1930
  for (const specs of this.moduleFreshness.values()) {
@@ -1741,14 +1963,23 @@ export class CloudflareScopeHost {
1741
1963
  return report;
1742
1964
  // Only run on a live scope of this tenant; a scope archived between the sweep's
1743
1965
  // enumeration and here simply has nothing due. A CP-less host has no directory
1744
- // to ask (#461) — it already trusts the router-asserted (tenant, scope) for the
1745
- // whole request path, and lifecycle for its scopes lives wherever provisioning
1746
- // does, so the grant check below is the only gate it can and does enforce.
1966
+ // to ask (#461), so it reads the lifecycle the platform delivered to the scope (#1713).
1747
1967
  if (!this.cpLess) {
1748
1968
  const rec = await this.cp.getScopeRecord(tenantId, scopeId);
1749
1969
  if (!rec || rec.status !== 'active')
1750
1970
  return report;
1751
1971
  }
1972
+ else if (await this.lifecycleHeld(scopeId)) {
1973
+ // #1713: the lifecycle the platform delivered holds the scope. Every schedule is
1974
+ // `skipped` and no cadence row moves, as under the kill switch, so a schedule that
1975
+ // came due meanwhile fires once on the first pass after the scope is live again.
1976
+ for (const schedule of schedules) {
1977
+ report.skipped += 1;
1978
+ report.runs.push({ operation: schedule.operation, outcome: 'skipped' });
1979
+ }
1980
+ report.lifecycleHeld = true;
1981
+ return report;
1982
+ }
1752
1983
  const stub = this.scopeStub(scopeId);
1753
1984
  // The grant IS the switch (#383), and the kill switch is its lever (#1666): the
1754
1985
  // kernel's `systemScheduleState`, the one predicate both adapters run. A scope that
@@ -1772,6 +2003,7 @@ export class CloudflareScopeHost {
1772
2003
  return report;
1773
2004
  }
1774
2005
  const now = Date.now();
2006
+ const lines = asyncLinePass();
1775
2007
  for (const schedule of schedules) {
1776
2008
  const last = await stub.scheduleLastRun(schedule.operation);
1777
2009
  const lastRun = last ? Date.parse(last) : null;
@@ -1786,24 +2018,43 @@ export class CloudflareScopeHost {
1786
2018
  // produces either way carries it, and so does the row below.
1787
2019
  const invocationId = ulid();
1788
2020
  let status = 'ok';
2021
+ // #1901: the run's line, under the call's own id — the one its `ctx.log` lines carry.
2022
+ const startedAt = Date.now();
2023
+ let emitted;
2024
+ let failure;
1789
2025
  try {
1790
2026
  const scope = await this.getSystemScope(moduleId, tenantId, scopeId);
1791
- await scope.invoke(schedule.operation, schedule.input, { invocationId });
2027
+ await scope.invoke(schedule.operation, schedule.input, { invocationId, onEmitted: (r) => (emitted = r) });
1792
2028
  report.fired += 1;
1793
2029
  }
1794
2030
  catch (err) {
1795
2031
  status = 'failed';
2032
+ failure = { error: err };
1796
2033
  report.failed += 1;
1797
2034
  report.errors.push({
1798
2035
  operation: schedule.operation,
1799
2036
  error: err instanceof Error ? err.message : String(err),
1800
2037
  });
1801
2038
  }
2039
+ lines.write({
2040
+ kind: 'schedule',
2041
+ tenantId,
2042
+ scopeId,
2043
+ invocationId,
2044
+ operation: schedule.operation,
2045
+ startedAt,
2046
+ outcome: status,
2047
+ ...failure,
2048
+ dueAt: lastRun === null ? null : new Date(dueAt).toISOString(),
2049
+ latenessMs: lastRun === null ? null : now - dueAt,
2050
+ ...(emitted ? { emitted } : {}),
2051
+ });
1802
2052
  // #1288: 'schedule', whatever this operation happens to be called — including
1803
2053
  // `freshness:<something>`, which is exactly the row the evaluator no longer eats.
1804
2054
  await stub.recordScheduleRun(schedule.operation, new Date(now).toISOString(), status, 'schedule', invocationId);
1805
2055
  report.runs.push({ operation: schedule.operation, outcome: status === 'ok' ? 'ok' : 'failed' });
1806
2056
  }
2057
+ lines.end();
1807
2058
  return report;
1808
2059
  }
1809
2060
  /** The stub body, shared by the principal, connection, system and impersonation doors. */
@@ -2125,6 +2376,8 @@ export class CloudflareScopeHost {
2125
2376
  // returns it with the previous status, so the trail cannot disagree with
2126
2377
  // the directory about which deployment the action touched.
2127
2378
  await this.recordAdmin(actor, action, { tenantId, scopeId, vertical: before.vertical }, { status: before.status }, { status: to, ...afterExtra });
2379
+ // #1713: after the directory moved and the move was audited, never before or instead.
2380
+ await this.deliverLifecycles(actor, { scopeId });
2128
2381
  };
2129
2382
  const writeGrant = async (subject, permission, node, entity, expiresAt) => {
2130
2383
  if (entity) {
@@ -3126,7 +3379,7 @@ export class CloudflareScopeHost {
3126
3379
  if (channel === 'prod' && !incoming.deployment_ref) {
3127
3380
  const owning = await this.cp.readVertical(verticalSlug);
3128
3381
  if (owning && owning.owner_tenant !== null && !owning.listed) {
3129
- const bound = (await this.cp.listScopes({ tenantId: owning.owner_tenant, vertical: verticalSlug, status: ['active'] })).filter((s) => !s.forked_from && s.kind !== 'preview');
3382
+ const bound = (await this.cp.listScopes({ tenantId: owning.owner_tenant, vertical: verticalSlug, status: ['active'] })).filter(isPrimaryScopeRow);
3130
3383
  for (const s of bound) {
3131
3384
  if (s.vertical_version_id === versionId)
3132
3385
  continue;
@@ -3284,6 +3537,12 @@ export class CloudflareScopeHost {
3284
3537
  await this.cp.setScopeServingRef(scopeId, servingRef);
3285
3538
  await this.recordAdmin(actor, 'setScopeServingRef', { tenantId, scopeId }, { servingRef: scope.serving_ref ?? null }, { servingRef, ...(ack.exportBreak ? { acknowledged: ack } : {}) });
3286
3539
  },
3540
+ recordKeptCopyResolution: async (actor, tenantId, scopeId, r) => {
3541
+ const scope = await this.cp.getScopeRecord(tenantId, scopeId);
3542
+ if (!scope)
3543
+ throw substratError('not_found', `unknown scope ${scopeId} in tenant ${tenantId}`);
3544
+ await this.recordAdmin(actor, 'resolveKeptCopy', { tenantId, scopeId }, { script: r.script, keptAt: r.keptAt, revision: r.revisionBefore }, { action: r.action, liveScript: r.liveScript, revision: r.revisionAfter });
3545
+ },
3287
3546
  setScopeExpiresAt: async (actor, tenantId, scopeId, expiresAt) => {
3288
3547
  const scope = await this.cp.getScopeRecord(tenantId, scopeId);
3289
3548
  if (!scope)
@@ -3388,6 +3647,8 @@ export class CloudflareScopeHost {
3388
3647
  setTenantStatus: async (actor, tenantId, status) => {
3389
3648
  const before = await this.cp.setTenantStatus(tenantId, status);
3390
3649
  await this.recordAdmin(actor, 'setTenantStatus', { tenantId }, { status: before }, { status });
3650
+ // #1713: every hosted scope under the tenant holds or resumes its work by this.
3651
+ await this.deliverLifecycles(actor, { tenantId });
3391
3652
  },
3392
3653
  setTenantName: async (actor, tenantId, name) => {
3393
3654
  const before = await this.cp.setTenantName(tenantId, name);
@@ -4260,6 +4521,11 @@ export class CloudflareScopeHost {
4260
4521
  const { tenantId, scopeId, ...after } = ownerTransferAudit.parse(entry);
4261
4522
  await this.recordAdmin(actor, 'transferOwner', { tenantId, scopeId }, null, after);
4262
4523
  },
4524
+ /** #2005: one change to a scope's copy marker, written around the vertical's own change. */
4525
+ recordCopyMark: async (actor, entry) => {
4526
+ const { tenantId, scopeId, action, ...after } = copyMarkAudit.parse(entry);
4527
+ await this.recordAdmin(actor, action === 'mark' ? 'markScopeCopy' : 'clearScopeCopyMark', { tenantId, scopeId }, null, after);
4528
+ },
4263
4529
  accessLog: async (actor, filter) => {
4264
4530
  const rows = await this.cp.accessLog({
4265
4531
  actor: filter?.actor,
@@ -4481,6 +4747,184 @@ export class CloudflareScopeHost {
4481
4747
  return;
4482
4748
  throw refusal.code ? substratError(refusal.code, refusal.message) : new Error(refusal.message);
4483
4749
  }
4750
+ /**
4751
+ * THE lifecycle gate (#1713) at every door that runs a scope's work: a request's stubs and
4752
+ * attachments, a capability or impersonation session, a peer call or delivery, a subscription,
4753
+ * and the entry points no request asked for — the retry driver, the job runner, a schedule's
4754
+ * system stub, a routed connector delivery.
4755
+ *
4756
+ * `validateScopeAccess` first, which is the directory's gate where there is a directory. A
4757
+ * CP-less host has none, so it then reads the lifecycle the platform delivered to the scope's
4758
+ * own storage (`/internal/lifecycle`), judged by the kernel's `lifecycleRefusal` in the
4759
+ * directory's own words. The router refuses a suspended scope's requests first (#1730); this
4760
+ * is the deployment's own half, and the only one a timer or a retry ever meets.
4761
+ *
4762
+ * The admin and diagnostic reads (dead letters, job runs, platform requests, delivered
4763
+ * grants) keep `validateScopeAccess` alone: an operator still reads a suspended scope.
4764
+ */
4765
+ async assertLive(tenantId, scopeId) {
4766
+ await this.validateScopeAccess(tenantId, scopeId);
4767
+ const refusal = await this.cpLessRefusal(scopeId, { tenantId, scopeId });
4768
+ if (refusal)
4769
+ throw new Error(refusal);
4770
+ }
4771
+ /** The delivered lifecycle's refusal on a CP-less host (#1713); always null with a directory. */
4772
+ async cpLessRefusal(scopeId, ids) {
4773
+ if (!this.cpLess)
4774
+ return null;
4775
+ return lifecycleRefusal(await this.scopeStub(scopeId).lifecycle(), ids);
4776
+ }
4777
+ /**
4778
+ * Deliver the directory's lifecycle to the deployments serving the matching scopes (#1713):
4779
+ * one scope after its transition, a tenant's scopes after the tenant's, or the heal sweep's
4780
+ * drift. Each delivery carries the directory's revisions, read in the same query as the
4781
+ * statuses, so a late one cannot undo a later transition: the deployment keeps only a delivery
4782
+ * strictly newer than the one it holds.
4783
+ *
4784
+ * Never throws. A transition is the operator's lever in an incident, and the router refuses
4785
+ * the scope's requests as soon as the directory moves (#1730), so a deployment that cannot be
4786
+ * reached must not refuse the lever. Each failure is an ops-failure row, no receipt is written,
4787
+ * and the heal sweep delivers again.
4788
+ */
4789
+ async deliverLifecycles(actor, filter) {
4790
+ const report = { attempted: 0, delivered: 0, failed: 0 };
4791
+ const delegation = this.lifecycleDelegation;
4792
+ if (!delegation || this.cpLess)
4793
+ return report;
4794
+ let targets;
4795
+ try {
4796
+ targets = await this.cp.lifecycleTargets(filter);
4797
+ }
4798
+ catch (err) {
4799
+ console.error('substrat: could not read the scopes to deliver a lifecycle to (#1713)', err);
4800
+ return report;
4801
+ }
4802
+ for (const first of targets) {
4803
+ const tenantId = first.tenant_id;
4804
+ const scopeId = first.scope_id;
4805
+ let t = first;
4806
+ let lifecycle = this.lifecycleOfTarget(t);
4807
+ if (!lifecycle) {
4808
+ // A status this code does not know (a newer directory): delivering a guess could lift a hold.
4809
+ report.failed += 1;
4810
+ console.error(`substrat: scope ${scopeId} has a lifecycle this code cannot read (#1713)`);
4811
+ continue;
4812
+ }
4813
+ // A live scope whose deployment already acknowledged exactly this has nothing to receive,
4814
+ // and neither does one with no receipt in a directory never restored: a deployment holding
4815
+ // no lifecycle runs live. That keeps an activation, and every transition before a
4816
+ // deployment carries the route, from posting a delivery that changes nothing. After a
4817
+ // restore (epoch > 0) no receipt says nothing about what the deployment holds, so it is
4818
+ // delivered. A HOLD is always delivered.
4819
+ if (lifecycleRefusal(lifecycle) === null && (t.delivered === null ? t.epoch === 0 : t.delivered === lifecycleReceipt(lifecycle))) {
4820
+ continue;
4821
+ }
4822
+ report.attempted += 1;
4823
+ try {
4824
+ let answer = await delegation.deliver({ tenantId, scopeId, lifecycle });
4825
+ // SINGLE AUTHORITY (the invariant this rests on): an environment has exactly one directory,
4826
+ // the singleton `CONTROL_PLANE.idFromName('control-plane')`. A "fresh directory" restore is
4827
+ // that same object restored after its storage was lost, never a second live writer, and
4828
+ // the raise below always re-reads the CURRENT store and delivers what it says. Two control
4829
+ // planes healing one dispatch namespace at once is a split brain that corrupts far more
4830
+ // than lifecycle, and is out of scope here; fencing it in lifecycle alone would guarantee
4831
+ // nothing.
4832
+ //
4833
+ // The scope refused us as OLDER while holding a revision this directory's history did not
4834
+ // write: an epoch ahead of ours, or ours with other counters. Only another history of the
4835
+ // directory delivers those — the one a fresh-directory restore replaced, whose epoch a
4836
+ // clock running behind (or a restore in the same millisecond) cannot outrank. A scope only
4837
+ // ever holds epochs a directory minted, so this directory learns past it: its epoch is
4838
+ // raised above the one held (monotonic, one statement), and the scope is delivered again.
4839
+ const held = answer.lifecycle.revision;
4840
+ const foreign = !answer.applied &&
4841
+ held !== undefined &&
4842
+ (held.epoch > lifecycle.revision.epoch ||
4843
+ (held.epoch === lifecycle.revision.epoch && lifecycleReceipt(answer.lifecycle) !== lifecycleReceipt(lifecycle)));
4844
+ // Bounded, because the deployment answering is the vertical's own code: only a directory
4845
+ // that has been restored can meet another history (epoch > 0), and a legitimate epoch is
4846
+ // a mint time, so one further ahead than the skew is forged or broken. Either way nothing
4847
+ // is raised; the refusal is an ops failure for an operator to look at.
4848
+ if (foreign && (lifecycle.revision.epoch === 0 || held.epoch > Date.now() + LIFECYCLE_EPOCH_SKEW_MS)) {
4849
+ report.failed += 1;
4850
+ await this.admin
4851
+ .recordOpsFailure({
4852
+ actor,
4853
+ operation: 'scope.lifecycle',
4854
+ stage: 'foreign-epoch',
4855
+ tenantId,
4856
+ scopeId,
4857
+ message: `the deployment refused this directory's lifecycle while holding epoch ${held.epoch} ` +
4858
+ `(this directory is at ${lifecycle.revision.epoch}); not raised past it — ` +
4859
+ (lifecycle.revision.epoch === 0
4860
+ ? 'this directory has never been restored, so no other history should have written it'
4861
+ : 'it is further ahead of the clock than any directory mints'),
4862
+ })
4863
+ .catch((e) => console.error('substrat: could not record a foreign lifecycle epoch (#1713)', e));
4864
+ continue;
4865
+ }
4866
+ if (foreign) {
4867
+ await this.cp.raiseLifecycleEpoch(held.epoch + 1);
4868
+ const [again] = await this.cp.lifecycleTargets({ scopeId });
4869
+ const next = again ? this.lifecycleOfTarget(again) : null;
4870
+ if (again && next) {
4871
+ t = again;
4872
+ lifecycle = next;
4873
+ answer = await delegation.deliver({ tenantId, scopeId, lifecycle });
4874
+ }
4875
+ }
4876
+ // The receipt is what the scope HOLDS: a newer delivery it kept instead is the truth.
4877
+ await this.cp.recordLifecycleReceipt(scopeId, lifecycleReceipt(answer.lifecycle), new Date().toISOString());
4878
+ report.delivered += 1;
4879
+ }
4880
+ catch (err) {
4881
+ report.failed += 1;
4882
+ try {
4883
+ await this.admin.recordOpsFailure({
4884
+ actor,
4885
+ operation: 'scope.lifecycle',
4886
+ stage: 'deliver',
4887
+ tenantId,
4888
+ scopeId,
4889
+ message: `the lifecycle (scope ${t.scope_status}, tenant ${t.tenant_status}) did not reach the ` +
4890
+ `deployment serving this scope; the heal sweep will deliver it again: ${err instanceof Error ? err.message : String(err)}`,
4891
+ });
4892
+ }
4893
+ catch (recordErr) {
4894
+ console.error('substrat: could not record a lifecycle delivery failure (#1713)', recordErr);
4895
+ }
4896
+ }
4897
+ }
4898
+ return report;
4899
+ }
4900
+ /** One directory row as the lifecycle a delivery carries (#1713), or null for a status this code does not know. */
4901
+ lifecycleOfTarget(t) {
4902
+ const parsed = scopeLifecycle.safeParse({
4903
+ scope: t.scope_status,
4904
+ tenant: t.tenant_status,
4905
+ at: new Date().toISOString(),
4906
+ revision: { epoch: t.epoch, scope: t.scope_rev, tenant: t.tenant_rev },
4907
+ });
4908
+ return parsed.success ? parsed.data : null;
4909
+ }
4910
+ /**
4911
+ * The heal sweep (#1713): deliver the lifecycle to every hosted scope whose deployment's
4912
+ * acknowledged state differs from the directory, and again to every scope held now — which is
4913
+ * what puts a hold back on a store a carry or a restore landed without it. Bounded per pass
4914
+ * (`limit`), drifted scopes first. Run from the shared control plane's cron.
4915
+ */
4916
+ async healLifecycles(actor, opts) {
4917
+ return this.deliverLifecycles(actor, { drift: true, limit: opts?.limit ?? 200 });
4918
+ }
4919
+ /**
4920
+ * Whether a CP-less host holds this scope's work (#1713): `assertLive`'s predicate, answered
4921
+ * rather than thrown, for the entry points that defer instead of refusing — a sweep pass, a
4922
+ * schedule run, an executor drain. Always false on a host with a directory, whose entry
4923
+ * points ask the directory themselves.
4924
+ */
4925
+ async lifecycleHeld(scopeId) {
4926
+ return (await this.cpLessRefusal(scopeId)) !== null;
4927
+ }
4484
4928
  /**
4485
4929
  * K-3's cross-check on its own: the (tenant, scope) pair must exist and agree before a
4486
4930
  * subject-key operation touches anything. Without it a caller could reach another
@@ -4997,16 +5441,16 @@ export class CloudflareScopeHost {
4997
5441
  * suspended or archiving scope could still be addressed and handed a 101 — a
4998
5442
  * scope that refuses every ordinary read while quietly holding an open socket.
4999
5443
  *
5000
- * Costs a CP-less vertical nothing: `validateScopeAccess` is a **no-op** on the
5001
- * null control plane (the router already gated lifecycle and tenancy from the
5002
- * shared directory), so this is the hosted-vertical shape unchanged.
5444
+ * On a CP-less vertical, `validateScopeAccess` is a no-op on the null control
5445
+ * plane, and `assertLive` reads the lifecycle the platform delivered to the
5446
+ * scope's own storage instead (#1713).
5003
5447
  *
5004
5448
  * `migrateAndRecord`, not the DO's own `ensureMigrations`: the DO migrates
5005
5449
  * itself when the socket opens either way, but only this reports the applied
5006
5450
  * count to the directory — so a scope whose first contact after a deploy is a
5007
5451
  * subscription does not go dark in the migration fleet view.
5008
5452
  */
5009
- await this.validateScopeAccess(tenantId, scopeId);
5453
+ await this.assertLive(tenantId, scopeId);
5010
5454
  await this.migrateAndRecord(scopeId);
5011
5455
  // Asserted, not carried through from the client: the principal is the
5012
5456
  // vertical's own resolution of its session, and the tenant and scope are the
@@ -5369,14 +5813,19 @@ export class CloudflareScopeHost {
5369
5813
  // side before the call; what runs HERE is the half only this deployment can
5370
5814
  // enforce: the scope's own permission check against its delivered
5371
5815
  // `connection:<id>` tuple, in the scope's own DO. Fail closed — no grant, no
5372
- // effect — exactly as for any other caller.
5816
+ // effect — exactly as for any other caller. And the lifecycle delivered to the
5817
+ // scope (#1713): a held scope's invoke and bytes doors refuse, as every other door does.
5373
5818
  /** Invoke ONE operation in this deployment as a CONNECTION (#574). */
5374
5819
  async connectorInvokeLocal(connectionId, tenantId, scopeId, operation, input) {
5820
+ await this.assertLive(tenantId, scopeId); // #1713: a held scope runs no connector work either
5375
5821
  await this.migrateAndRecord(scopeId);
5376
5822
  return this.buildStub(tenantId, scopeId, undefined, connectionId).invoke(operation, input);
5377
5823
  }
5378
5824
  /** Land provider bytes in this deployment as a CONNECTION — the bytes leg (#574). */
5379
5825
  async connectorAttachmentUploadLocal(connectionId, tenantId, scopeId, upload) {
5826
+ // The lifecycle gate every attachments door takes (#1713), ahead of the bucket: a held
5827
+ // scope's provider bytes wait for the delivery's retry rather than land (#1995).
5828
+ await this.assertLive(tenantId, scopeId);
5380
5829
  await this.migrateAndRecord(scopeId);
5381
5830
  const store = await this.resolveAttachmentStore(tenantId);
5382
5831
  return this.buildAttachmentSurface({ connectionId }, tenantId, scopeId, store).upload(upload);
@@ -5389,6 +5838,8 @@ export class CloudflareScopeHost {
5389
5838
  * so a caller falls back rather than failing a dispatch over a missing file.
5390
5839
  */
5391
5840
  async connectorAttachmentOpenLocal(connectionId, tenantId, scopeId, attachmentId, eventId) {
5841
+ // Same gate as the upload leg: a held scope's bytes are not read out of its bucket.
5842
+ await this.assertLive(tenantId, scopeId);
5392
5843
  await this.migrateAndRecord(scopeId);
5393
5844
  const store = await this.resolveAttachmentStore(tenantId);
5394
5845
  return this.buildAttachmentSurface({ connectionId }, tenantId, scopeId, store,