@substrat-run/adapter-cloudflare 0.134.0 → 0.136.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/host.js CHANGED
@@ -1,7 +1,8 @@
1
+ import { env as ambientEnv } from 'cloudflare:workers';
1
2
  import { isRewindRefusal, REWIND_REFUSED } from './rewind-refusal.js';
2
- import { delegatedReadParams, fromWireFailure, exportReadInput, exportedBatch, importBatch, importResult, importState, importsOfManifestJson, exportsOfManifestJson, importCursorMove, importCursorMoveAt, importCursorMoved, accessLogEntry, adminLogEntry, opsFailureEntry, opsFailureFingerprint, issueEntry, sweepRunEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunsPayload, modelUsageEntry, attachmentRecord, createTenantInput, identityLink, identityPool, createOrgInput, promotionAcknowledgement, bindAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, registerVerticalInput, vertical as verticalSchema, verticalServingState, verticalChannel, verticalVersion, connection, capabilityExchange, capabilityGrant, principalId, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, systemSwitch, systemSwitchOutcome, systemScheduleEntry, systemSwitchRecord, entitlementGrant, entitlementGrantInput, instant, meterReading, subjectRef, createConnectionInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, org as orgSchema, orgMembership, resolvedIdentity, identityMembership, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, delegatedReadRecord, ownerTransferAudit, subjectShredReceipt, connectorDispatchKind, callsOfManifestJson, outboundOfManifestJson, substratError, redrainEventsInput, coverage, peerCoverage, peerGrantsEntry, peerSwitch, peerSwitchOutcome, verticalCaller, verticalResolution, verticalSlug, entityObjectRef, } from '@substrat-run/contracts';
3
+ import { delegatedReadParams, fromWireFailure, exportReadInput, exportedBatch, importBatch, importResult, importState, importsOfManifestJson, exportsOfManifestJson, importCursorMove, importCursorMoveAt, importCursorMoved, accessLogEntry, adminLogEntry, opsFailureEntry, opsFailureFingerprint, issueEntry, sweepRunEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunsPayload, modelUsageEntry, attachmentRecord, createTenantInput, identityLink, identityPool, createOrgInput, promotionAcknowledgement, bindAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, registerVerticalInput, vertical as verticalSchema, verticalServingState, verticalChannel, verticalVersion, connection, capabilityExchange, capabilityGrant, principalId, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, systemSwitch, systemSwitchOutcome, systemScheduleEntry, systemSwitchRecord, entitlementGrant, entitlementGrantInput, capabilityFilter, instant, meterReading, subjectRef, createConnectionInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, org as orgSchema, orgMembership, resolvedIdentity, identityMembership, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, delegatedReadRecord, ownerTransferAudit, copyMarkAudit, subjectShredReceipt, connectorDispatchKind, callsOfManifestJson, outboundOfManifestJson, substratError, redrainEventsInput, coverage, peerCoverage, peerGrantsEntry, peerSwitch, peerSwitchOutcome, verticalCaller, verticalResolution, verticalSlug, entityObjectRef, ATTACHMENT_BLOB_BINDING, blobStoreBindingName, lifecycleDelivery, scopeLifecycle, } from '@substrat-run/contracts';
3
4
  import { normalizeHostname, toRouteTarget } from './route-resolver.js';
4
- import { attachmentBlobKey, attachmentSha256, entitlementDenial, foldMeterReading, parseValidationRecords, resolveScopeRecord, ulid, capabilityTokenHash, checkBecomeInput, plausibleSessionToken, foldModelUsage, backoffAt, resolveRetryPolicy, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, openSealed, ImpersonationRefused, assertSessionUsable, impersonationRowValues, mapImpersonationRow, newImpersonationSession, jobRunOf, runDueJobRuns, startJobRun, attachmentTextJob, assertAttachmentExtractors, assertJobRegistrable, isAttachmentTextRun, searchLimit, searchMatchExpression, globalFetch, assertRedrainWindow, platformRequestOf, undrainedEventsOf, inUnitMovesToAudit, staleCarryRevertRow, staleCarryReverts, withRecorded, systemSwitchedOffMessage, tenantSystemSwitchedOffMessage, CrossVerticalRegistry, importCursorSourceOf, exportBreaksOf, exportBreakRefusal, bindExportBreaksOf, bindExportBreakRefusal, collectPeers, peerSeats, connectorCallRecord, noopConnectorCallRecorder, recordConnectorCall, settleConnectionUse, splitManifestMigrations, unknownRoleError, assertRowLimit, assertRowOffset, } from '@substrat-run/kernel';
5
+ import { attachmentBlobKey, attachmentSha256, entitlementDenial, requiredEntitlementFor, foldMeterReading, parseValidationRecords, resolveScopeRecord, ulid, KEPT_COPY_REFUSAL, capabilityTokenHash, checkBecomeInput, plausibleSessionToken, foldModelUsage, backoffAt, resolveRetryPolicy, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, openSealed, ImpersonationRefused, assertSessionUsable, impersonationRowValues, mapImpersonationRow, newImpersonationSession, jobRunOf, runDueJobRuns, startJobRun, attachmentTextJob, assertAttachmentExtractors, assertJobRegistrable, isAttachmentTextRun, searchLimit, searchMatchExpression, globalFetch, assertRedrainWindow, platformRequestOf, undrainedEventsOf, inUnitMovesToAudit, staleCarryRevertRow, staleCarryReverts, withRecorded, systemSwitchedOffMessage, tenantSystemSwitchedOffMessage, CrossVerticalRegistry, importCursorSourceOf, exportBreaksOf, exportBreakRefusal, bindExportBreaksOf, bindExportBreakRefusal, collectPeers, peerSeats, connectorCallRecord, noopConnectorCallRecorder, recordConnectorCall, settleConnectionUse, splitManifestMigrations, unknownRoleError, assertRowLimit, assertRowOffset, INERT_SCOPE_REASON, lifecycleReceipt, lifecycleRefusal, isPrimaryScope, isPrimaryScopeRow, asyncInvocationId, asyncLinePass, } from '@substrat-run/kernel';
5
6
  import { attributedHost } from '@substrat-run/kernel';
6
7
  import { isOrangeToOrange, isUpgradeRequest, LIVE_MODE_HEADER, LIVE_PRINCIPAL_HEADER, LIVE_SCOPE_HEADER, LIVE_SUBSCRIBE_PATH, LIVE_TENANT_HEADER, } from './live-reads.js';
7
8
  import { tenantStoreDatabaseName } from './d1.js';
@@ -30,12 +31,29 @@ function unwrapCapabilityReply(reply) {
30
31
  throw fromWireFailure(reply.failure);
31
32
  return reply.value;
32
33
  }
34
+ /**
35
+ * How far ahead of the clock a held lifecycle epoch may be and still be learned past (#1713). An
36
+ * epoch is the time a directory restore minted it, so a legitimate one is behind its minter's
37
+ * clock; the skew allows for clocks that disagree, and bounds what a forged answer can move.
38
+ */
39
+ export const LIFECYCLE_EPOCH_SKEW_MS = 24 * 60 * 60 * 1000;
40
+ /**
41
+ * Refuse to mark a scope a copy unless the request classifies it non-primary (#2005). The
42
+ * classification is the platform directory's; the vertical applies the same `isPrimaryScope` to
43
+ * it, so a primary can be marked only by a request that misstates its lineage, never by default.
44
+ */
45
+ function assertCopyLineage(lineage) {
46
+ if (isPrimaryScope(lineage)) {
47
+ throw substratError('conflict', 'mark-copy refused: the directory classifies this scope as primary, not as a copy');
48
+ }
49
+ }
33
50
  /**
34
51
  * A control-plane stand-in for a CP-less vertical (scope-local-permissions.md Phase 3).
35
52
  * The hot path a served scope actually touches becomes trust-the-upstream:
36
- * - `scopeAccessRefusal` / `setMigrationState` → no-op: the router already gated the
37
- * scope's lifecycle + tenancy from the shared directory, so the vertical trusts the
38
- * asserted node rather than re-reading a directory it does not have.
53
+ * - `scopeAccessRefusal` / `setMigrationState` → no-op: the router gates the scope's
54
+ * lifecycle + tenancy from the shared directory, and the vertical trusts the asserted
55
+ * node rather than re-reading a directory it does not have. Its own half of the
56
+ * lifecycle gate reads the scope's storage instead (`assertLive`, #1713).
39
57
  * - `tenantHoldsEntitlement` → true: the SKU was enforced on the shared control plane
40
58
  * at provision (before `provisionInstance`), so a scope that EXISTS here was granted
41
59
  * it upstream — a single-vertical deployment holds its own entitlements by construction.
@@ -106,6 +124,14 @@ export const SWITCH_HOLD_EXTRA_WAITS = 3;
106
124
  * clock, against the releasing host's clock: minutes, not seconds.
107
125
  */
108
126
  export const SWITCH_HOLD_PENDING_MAX_MS = 5 * 60_000;
127
+ /**
128
+ * The attachment bucket a host resolves when it was handed no resolver (#1995): the
129
+ * platform's per-tenant binding, read off this script's own env, named by the one shared
130
+ * encoding (`blobStoreBindingName`) for the tenant the caller passed in.
131
+ */
132
+ function ambientAttachmentBucket(tenantId) {
133
+ return ambientEnv[blobStoreBindingName(ATTACHMENT_BLOB_BINDING, tenantId)] ?? null;
134
+ }
109
135
  export class CloudflareScopeHost {
110
136
  admin;
111
137
  scopeNs;
@@ -145,7 +171,7 @@ export class CloudflareScopeHost {
145
171
  tenantStores;
146
172
  /** The live R2 client for per-tenant blob stores (#473); undefined ⇒ refuse loudly. */
147
173
  blobStores;
148
- /** Worker-side attachment-bucket resolver (#473); undefined ⇒ attachments() refuses. */
174
+ /** Worker-side attachment-bucket resolver (#473); the ambient per-tenant binding unless overridden (#1995). */
149
175
  attachmentBuckets;
150
176
  /** The parsers attachment text is extracted with (K-43); the host's own, never imported here. */
151
177
  attachmentExtractors;
@@ -171,6 +197,8 @@ export class CloudflareScopeHost {
171
197
  */
172
198
  onBehalfOf = null;
173
199
  withdrawn = new Map(); // operation → module
200
+ /** Operation → who binds it (module, SKU flag, its declared schedules) — the §4.3 gate's
201
+ * input, resolved per invoke by the kernel's `requiredEntitlementFor` (#1654). */
174
202
  operationEntitlement = new Map();
175
203
  /** #574: remote connector write-back for scopes served by another deployment. */
176
204
  connectorDelegation;
@@ -178,6 +206,8 @@ export class CloudflareScopeHost {
178
206
  eventDrainDelegation;
179
207
  /** #1666: the schedule kill switch's reach into the deployment serving a scope. */
180
208
  systemSwitchDelegation;
209
+ /** #1713: the lifecycle's reach into the deployment serving a scope. */
210
+ lifecycleDelegation;
181
211
  /** #1706: the peer kill switch's reach into the deployment serving a scope. */
182
212
  peerSwitchDelegation;
183
213
  /** #1705 PR 3: the replay lever's reach into the deployment serving a consumer scope. */
@@ -209,7 +239,7 @@ export class CloudflareScopeHost {
209
239
  this.secretBox = options.secretBox ?? unconfiguredSecretBox;
210
240
  this.tenantStores = options.tenantStores;
211
241
  this.blobStores = options.blobStores;
212
- this.attachmentBuckets = options.attachmentBuckets;
242
+ this.attachmentBuckets = options.attachmentBuckets ?? ambientAttachmentBucket;
213
243
  assertAttachmentExtractors(options.attachmentExtractors ?? []);
214
244
  this.attachmentExtractors = options.attachmentExtractors ?? [];
215
245
  this.fetchImpl = options.fetch ?? globalFetch;
@@ -223,6 +253,7 @@ export class CloudflareScopeHost {
223
253
  this.connectorDelegation = options.connectorDelegation;
224
254
  this.eventDrainDelegation = options.eventDrainDelegation;
225
255
  this.systemSwitchDelegation = options.systemSwitchDelegation;
256
+ this.lifecycleDelegation = options.lifecycleDelegation;
226
257
  this.peerSwitchDelegation = options.peerSwitchDelegation;
227
258
  this.importCursorDelegation = options.importCursorDelegation;
228
259
  this.admin = this.buildAdmin();
@@ -351,21 +382,78 @@ export class CloudflareScopeHost {
351
382
  };
352
383
  if (this.executors.size === 0)
353
384
  return report;
385
+ // #1901: one line per attempt, capped per pass. Written on the coordinator, where the
386
+ // attempt runs — the DO only journals it.
387
+ const lines = asyncLinePass();
388
+ try {
389
+ await this.drainExecutorsPass(tenantId, scopeId, invocationId, report, lines);
390
+ }
391
+ finally {
392
+ lines.end();
393
+ }
394
+ return report;
395
+ }
396
+ async drainExecutorsPass(tenantId, scopeId, invocationId, report, lines) {
354
397
  const stub = this.scopeStub(scopeId);
355
- for (const [id, executor] of this.executors) {
398
+ // #2005: a non-primary scope (a fork, a snapshot, a preview of either kind) causes no
399
+ // outbound effects, so its deliveries are journaled terminal with the reason and no
400
+ // handler runs. Asked of the directory where there is one. A CP-less host has none, so it
401
+ // asks the scope's own storage whether it is classified a copy (`_substrat_copy_origin`'s
402
+ // `is_copy`, #2009, set on the directory's word) — a preview and a snapshot reach a hosted
403
+ // vertical as restores the platform marks.
404
+ // Asked on the first due event, once per pass: most passes have none, and the directory
405
+ // is one global object.
406
+ let inert;
407
+ const isInert = () => (inert ??= this.cpLess
408
+ ? stub.isCopy()
409
+ : this.cp.getScopeRecord(tenantId, scopeId).then((row) => !isPrimaryScopeRow(row)));
410
+ // #1713: a CP-less scope its lifecycle holds attempts nothing and journals nothing, so every
411
+ // due delivery stays due for the first pass after it is live again. Asked on the first due
412
+ // event, like `isInert`. Every caller passed `assertLive` already, so this is for the scope
413
+ // suspended between that gate and the drain: a request in flight, whose write commits while
414
+ // its effect waits. A host with a directory answers false without a read.
415
+ let held;
416
+ const isHeld = () => (held ??= this.lifecycleHeld(scopeId));
417
+ drain: for (const [id, executor] of this.executors) {
356
418
  const deliveryId = `executor:${id}`;
357
419
  const { events, undecodable } = await stub.pendingExecutorDeliveries(deliveryId, executor.eventType);
358
420
  // #1636: an event the DO could not decode is dead-lettered for this executor at once,
359
421
  // and its handler never sees it. Terminal on the FIRST failure, unlike a handler's:
360
422
  // the decode is pure, so a retry cannot succeed. The rows behind it are delivered
361
423
  // below — the decode used to throw the whole list, on every pass.
424
+ if ((undecodable.length > 0 || events.length > 0) && (await isHeld())) {
425
+ report.lifecycleHeld = true;
426
+ break drain;
427
+ }
428
+ // #1901: an attempt's line. The attempt number is the one the journal is about to record.
429
+ const unitOf = (eventId, attempt) => ({
430
+ kind: 'consumer',
431
+ tenantId,
432
+ scopeId,
433
+ invocationId: asyncInvocationId(invocationId),
434
+ operation: deliveryId,
435
+ eventType: executor.eventType,
436
+ eventId,
437
+ attempt,
438
+ startedAt: Date.now(),
439
+ });
362
440
  for (const bad of undecodable) {
363
441
  report.attempted += 1;
364
- await stub.recordExecutorAttempt(bad.eventId, deliveryId, bad.error, null, invocationId);
442
+ const attempt = await stub.recordExecutorAttempt(bad.eventId, deliveryId, bad.error, null, invocationId);
365
443
  report.deadLettered += 1;
444
+ lines.write({ ...unitOf(bad.eventId, attempt), outcome: 'dead-lettered' });
366
445
  }
367
446
  for (const event of events) {
368
447
  report.attempted += 1;
448
+ const startedAt = Date.now();
449
+ if (await isInert()) {
450
+ // No next attempt, like an undecodable row: a scope does not become primary.
451
+ const attempt = await stub.recordExecutorAttempt(event.id, deliveryId, INERT_SCOPE_REASON, null, invocationId);
452
+ report.inert = (report.inert ?? 0) + 1;
453
+ // Its own outcome, never `delivered`: no handler ran (#2005).
454
+ lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'inert' });
455
+ continue;
456
+ }
369
457
  this.causedBy = event.id;
370
458
  try {
371
459
  if (executor.kind === 'connector' && this.cpLess) {
@@ -379,16 +467,20 @@ export class CloudflareScopeHost {
379
467
  // the at-least-once residue, as it already must in-process.
380
468
  await stub.routeExecutorEventToPlatform(event.id, deliveryId, connectorDispatchKind(executor.provider), JSON.stringify({ executorId: id, event }), JSON.stringify({ system: 'connector-dispatch' }), invocationId);
381
469
  report.routedToPlatform += 1;
470
+ // Handed on, not run here: the platform's drain owns its attempts from now on.
471
+ lines.write({ ...unitOf(event.id, 1), startedAt, outcome: 'routed' });
382
472
  }
383
473
  else if (executor.kind === 'connector') {
384
474
  await executor.handler(await this.connectorContext(tenantId, scopeId, executor.timeoutMs, event.id), event);
385
- await stub.recordExecutorAttempt(event.id, deliveryId, null, null, invocationId);
475
+ const attempt = await stub.recordExecutorAttempt(event.id, deliveryId, null, null, invocationId);
386
476
  report.delivered += 1;
477
+ lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'delivered' });
387
478
  }
388
479
  else {
389
480
  await executor.handler(this.admin, event);
390
- await stub.recordExecutorAttempt(event.id, deliveryId, null, null, invocationId);
481
+ const attempt = await stub.recordExecutorAttempt(event.id, deliveryId, null, null, invocationId);
391
482
  report.delivered += 1;
483
+ lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'delivered' });
392
484
  }
393
485
  }
394
486
  catch (err) {
@@ -403,18 +495,23 @@ export class CloudflareScopeHost {
403
495
  report.deadLettered += 1;
404
496
  else
405
497
  report.retrying += 1;
498
+ lines.write({
499
+ ...unitOf(event.id, attempts),
500
+ startedAt,
501
+ outcome: exhausted ? 'dead-lettered' : 'retrying',
502
+ error: err,
503
+ });
406
504
  }
407
505
  finally {
408
506
  this.causedBy = null;
409
507
  }
410
508
  }
411
509
  }
412
- return report;
413
510
  }
414
511
  async drainDue(tenantId, scopeId) {
415
512
  // Same lifecycle gate `getScope` applies (K-3): a suspended or archived scope
416
513
  // does not get its effects driven either.
417
- await this.validateScopeAccess(tenantId, scopeId);
514
+ await this.assertLive(tenantId, scopeId);
418
515
  await this.migrateAndRecord(scopeId);
419
516
  // #1525: null, and honestly so — a sweep is not a call. An attempt this pass makes
420
517
  // records no invocation, which is what distinguishes it from the first attempt the
@@ -472,14 +569,14 @@ export class CloudflareScopeHost {
472
569
  this.attachmentExtractors);
473
570
  }
474
571
  async startJobRun(tenantId, scopeId, input) {
475
- await this.validateScopeAccess(tenantId, scopeId);
572
+ await this.assertLive(tenantId, scopeId);
476
573
  await this.migrateAndRecord(scopeId);
477
574
  return jobRunOf(await startJobRun(this.jobStore(scopeId), input, ulid, () => new Date().toISOString()));
478
575
  }
479
576
  async runDueJobs(tenantId, scopeId, options) {
480
577
  // Same lifecycle gate as `drainDue`: a suspended scope's runs wait rather than
481
578
  // advance, and an archived one's never move again.
482
- await this.validateScopeAccess(tenantId, scopeId);
579
+ await this.assertLive(tenantId, scopeId);
483
580
  await this.migrateAndRecord(scopeId);
484
581
  // #1575: the kernel's extraction job is this host's own, bound to this scope — no
485
582
  // deployment registers it, and none can shadow it.
@@ -504,7 +601,7 @@ export class CloudflareScopeHost {
504
601
  // same context build as the in-process path, so the handler cannot tell which host
505
602
  // ran it. On a CP-less host `connectorContext` throws from the null control plane:
506
603
  // fail closed, exactly the hole routing exists to avoid.
507
- await this.validateScopeAccess(tenantId, scopeId);
604
+ await this.assertLive(tenantId, scopeId);
508
605
  this.causedBy = event.id;
509
606
  try {
510
607
  await handler(await this.connectorContext(tenantId, scopeId, options?.timeoutMs ?? 30_000, event.id), event);
@@ -631,6 +728,15 @@ export class CloudflareScopeHost {
631
728
  async summarizeDenialsLocal(scopeId, filter) {
632
729
  return this.scopeStub(scopeId).summarizeDenials(filter);
633
730
  }
731
+ /**
732
+ * The operator's capability read's CP-less path (#1686) — the denial log's trust line: the
733
+ * directory rows live in the scope's own DO, in the vertical's deployment, reachable only
734
+ * through its platform-gated `/internal/capabilities`; the K-3 check and the K-24 entry are
735
+ * the control plane's, made before it calls. Records only — the DO's query selects no hash.
736
+ */
737
+ async listCapabilitiesLocal(scopeId, filter) {
738
+ return this.scopeStub(scopeId).listCapabilities(filter);
739
+ }
634
740
  /**
635
741
  * Copy one scope's data into a fresh scope DO, entirely within THIS deployment —
636
742
  * the data half of an orchestrated snapshot (preview-and-snapshots.md §9). Like the
@@ -642,7 +748,9 @@ export class CloudflareScopeHost {
642
748
  */
643
749
  async snapshotScopeLocal(sourceScopeId, destScopeId) {
644
750
  const tables = await this.scopeStub(sourceScopeId).exportDump();
645
- await this.scopeStub(destScopeId).importDump(tables, destScopeId, { sourceScopeId, exact: true });
751
+ // #2009: a snapshot is a fork by construction (the control plane's row names `forkedFrom`, so
752
+ // the directory never calls it primary), and a load classifies nothing by itself: marked here.
753
+ await this.scopeStub(destScopeId).importDump(tables, destScopeId, { sourceScopeId, exact: true, markCopy: true });
646
754
  return { tables: tables.length };
647
755
  }
648
756
  /**
@@ -655,14 +763,131 @@ export class CloudflareScopeHost {
655
763
  /** #1742: the directory's recorded-off modules, switched off on THIS scope in the replay's
656
764
  * own event — a dump from before a switch was pulled brings its grants back live. #1869:
657
765
  * `sourceScopeId`, the scope the dump was captured from, narrows the grant re-point to it;
658
- * `exact` says the platform exported the dump itself, so the re-point never falls back. */
766
+ * `exact` says the platform exported the dump itself, so the re-point never falls back.
767
+ * #1722: `loadStamp`, the stamp a carry leaves on the copy it lands, for a fenced wipe later,
768
+ * and `expect`, the marker the carry read here: the load is refused if the store moved since.
769
+ * #2005: `markCopy` — the directory's classification of the scope, sent when it is not
770
+ * primary; the restore then marks it a copy. Refused if it classifies a primary. */
659
771
  opts) {
660
- const switchedOff = await this.scopeStub(scopeId).importDump(tables, scopeId, {
772
+ if (opts?.markCopy)
773
+ assertCopyLineage(opts.markCopy);
774
+ const load = {
661
775
  switchOff: opts?.switchedOff ? { moduleIds: opts.switchedOff, at: new Date().toISOString() } : undefined,
662
776
  sourceScopeId: opts?.sourceScopeId,
663
777
  exact: opts?.exact,
778
+ loadStamp: opts?.loadStamp,
779
+ markCopy: opts?.markCopy !== undefined,
780
+ };
781
+ // Both refusals come back as values (a throw over the RPC carries only its message) and are
782
+ // thrown here, in the vertical's own isolate, so its route answers 409 or 412, not a fault.
783
+ const out = await this.scopeStub(scopeId).importDumpChecked(tables, scopeId, {
784
+ ...load,
785
+ ...(opts?.expect ? { expect: opts.expect } : {}),
664
786
  });
665
- return { tables: tables.length, ...(opts?.switchedOff ? { switchedOff } : {}) };
787
+ if (out.refused === false) {
788
+ return { tables: tables.length, ...(opts?.switchedOff ? { switchedOff: out.switchedOff } : {}) };
789
+ }
790
+ if (out.refused === 'kept')
791
+ throw substratError('conflict', KEPT_COPY_REFUSAL);
792
+ // A retry of a load that already committed (its answer was lost on the way back) is
793
+ // refused by the marker that load itself moved. The store holding THIS request's stamp
794
+ // says so: no other load writes it. Answered as applied, without `switchedOff`, so the
795
+ // caller's re-assert after the bind covers the OFF positions.
796
+ const now = await this.scopeStub(scopeId).loadMarker();
797
+ if (!opts?.loadStamp || now.loadStamp !== opts.loadStamp) {
798
+ throw substratError('precondition_failed', 'scope store changed since the carry read it; nothing was loaded (#1722)');
799
+ }
800
+ return { tables: tables.length };
801
+ }
802
+ /** The kept-copy marker of this scope's store in THIS deployment (#1722), or null. */
803
+ async keptCopyLocal(scopeId) {
804
+ return this.scopeStub(scopeId).keptCopy();
805
+ }
806
+ /**
807
+ * Discard the kept copy of a scope in THIS deployment (#1722), behind the vertical's
808
+ * `/internal/kept-copy/discard`: the staff resolution, only at the write revision the operator
809
+ * acted on. Answers whether it was discarded, or why not.
810
+ */
811
+ async discardKeptCopyLocal(scopeId, revision, carriedAway,
812
+ /** #2005: the directory's classification, sent when the scope is not primary. */
813
+ markCopy,
814
+ /** #1722 (Codex #2008 r13): the load stamp read with `revision`; a replaced store is refused. */
815
+ loadStamp) {
816
+ if (markCopy)
817
+ assertCopyLineage(markCopy);
818
+ return this.scopeStub(scopeId).discardKeptCopy(scopeId, revision, carriedAway, markCopy !== undefined, ...(loadStamp !== undefined ? [loadStamp] : []));
819
+ }
820
+ /**
821
+ * What a carry's restore into this scope's store expects to find unchanged (#1722), behind
822
+ * the vertical's `/internal/load-marker`. Read before the carry checks the binding again; the
823
+ * restore then carries it as `expect`, and is refused if the store was loaded or written since.
824
+ */
825
+ async loadMarkerLocal(scopeId) {
826
+ return this.scopeStub(scopeId).loadMarker();
827
+ }
828
+ /**
829
+ * Wipe the copy a carry left in THIS deployment (#1722), behind the vertical's
830
+ * `/internal/wipe-carried`: only if nothing was loaded into the scope's DO since the stamp
831
+ * the carry read (`expectLoadStamp`, null for a store no load has stamped). Non-terminal,
832
+ * so a later bind back to this version can restore into it. False when refused.
833
+ */
834
+ async wipeCarriedLocal(scopeId, expectLoadStamp, carriedAway, opts = {}) {
835
+ if (opts.markCopy)
836
+ assertCopyLineage(opts.markCopy);
837
+ return this.scopeStub(scopeId).wipeCarried(scopeId, expectLoadStamp, carriedAway, {
838
+ ...(opts.expectRevision !== undefined ? { expectRevision: opts.expectRevision } : {}),
839
+ ...(opts.protectIfChanged !== undefined ? { protectIfChanged: opts.protectIfChanged } : {}),
840
+ markCopy: opts.markCopy !== undefined,
841
+ });
842
+ }
843
+ /** Release a kept copy that is the live store after all (#1722), at the revision read. */
844
+ async releaseKeptCopyLocal(scopeId, revision,
845
+ /** #2005: the directory's classification, sent when the scope is not primary. */
846
+ markCopy,
847
+ /** #1722 (Codex #2008 r13): the load stamp read with `revision`; a replaced store is refused. */
848
+ loadStamp) {
849
+ if (markCopy)
850
+ assertCopyLineage(markCopy);
851
+ return this.scopeStub(scopeId).releaseKeptCopy(revision, markCopy !== undefined, ...(loadStamp !== undefined ? [loadStamp] : []));
852
+ }
853
+ /**
854
+ * Store the lifecycle the platform delivered for one scope in THIS deployment (#1713), behind
855
+ * the vertical's `/internal/lifecycle`. Kept unless the scope already holds a newer one, so a
856
+ * push that arrives late cannot undo a later transition. `applied` is what the platform's heal
857
+ * sweep records as delivered; `changed` says the gate's answer moved.
858
+ */
859
+ async setLifecycleLocal(scopeId, next) {
860
+ return lifecycleDelivery.parse(await this.scopeStub(scopeId).setLifecycle(scopeLifecycle.parse(next)));
861
+ }
862
+ /**
863
+ * Mark one scope in THIS deployment a copy (#2005), behind the vertical's `/internal/mark-copy`:
864
+ * the repair of a copy that predates the marker, which a CP-less coordinator reads for primacy.
865
+ * The control plane decides which scopes (its directory says they are not primary) and audits.
866
+ */
867
+ async markCopyLocal(scopeId, lineage) {
868
+ assertCopyLineage(lineage);
869
+ return { marked: await this.scopeStub(scopeId).markCopy() };
870
+ }
871
+ /**
872
+ * Clear a mistaken copy classification from one scope in THIS deployment (#2005), behind the
873
+ * vertical's `/internal/clear-copy-mark`: staff's correction for a scope the directory says IS
874
+ * primary. Refused for a scope classified a copy. A load's copied-events mark is never touched
875
+ * (#2009), so a primary restored from another scope's backup runs its own effects again and
876
+ * still never runs the work it copied.
877
+ */
878
+ async clearCopyMarkLocal(scopeId, lineage,
879
+ /** #1722 (Codex #2008 r12–r13): the store as the caller means it, its load stamp and write
880
+ * revision, compared in the clear's own transaction; refused (412) if either moved. The
881
+ * platform's reconcile of a carry's destination sends it; staff send none. */
882
+ expect) {
883
+ if (!isPrimaryScope(lineage)) {
884
+ throw substratError('conflict', 'clear-copy-mark refused: the directory classifies this scope as a copy (a preview or a fork)');
885
+ }
886
+ const outcome = await this.scopeStub(scopeId).clearCopyMark(...(expect ? [expect] : []));
887
+ if (outcome === 'changed') {
888
+ throw substratError('precondition_failed', `clear-copy-mark refused: scope ${scopeId}'s store changed since its revision was read`);
889
+ }
890
+ return { cleared: outcome === 'cleared' };
666
891
  }
667
892
  /**
668
893
  * Re-apply the vertical's OWN role definitions to one scope, CP-lessly — the
@@ -699,6 +924,13 @@ export class CloudflareScopeHost {
699
924
  async exportScopeLocal(scopeId) {
700
925
  return this.scopeStub(scopeId).exportDump();
701
926
  }
927
+ /**
928
+ * `exportScopeLocal` with the store's load stamp, read in the same DO call (#1722): what a
929
+ * carry's fenced wipe of the copy it leaves here expects. Behind the vertical's `/internal/export`.
930
+ */
931
+ async exportScopeStampedLocal(scopeId) {
932
+ return this.scopeStub(scopeId).exportDumpStamped();
933
+ }
702
934
  /** Facet this host's own scope's outbox (#1239) — the vertical-host read. */
703
935
  async facetEventsLocal(scopeId, input) {
704
936
  return this.scopeStub(scopeId).facetEvents(input);
@@ -845,9 +1077,14 @@ export class CloudflareScopeHost {
845
1077
  throw new Error(`${manifest.id} declares operationInputs for unbound operation(s): ` +
846
1078
  `${unboundInputs.sort().join(', ')} — a schema on nothing reads as a parse that is not there`);
847
1079
  }
1080
+ const binding = {
1081
+ moduleId: manifest.id,
1082
+ entitlementKey: manifest.entitlementKey,
1083
+ scheduledOperations: new Set((manifest.schedules ?? []).map((sch) => sch.operation)),
1084
+ };
848
1085
  for (const name of Object.keys(registration.operations ?? {})) {
849
1086
  this.bindOperation(name);
850
- this.operationEntitlement.set(name, manifest.entitlementKey);
1087
+ this.operationEntitlement.set(name, binding);
851
1088
  }
852
1089
  }
853
1090
  defineOperation(name, _handler) {
@@ -1049,7 +1286,7 @@ export class CloudflareScopeHost {
1049
1286
  // gates and metadata facts live in the ScopeDO (per-scope serialization, spine event
1050
1287
  // in the same transaction); bytes go straight to the per-tenant R2 bucket through the
1051
1288
  // binding the vertical's worker resolved — never through the DO.
1052
- await this.validateScopeAccess(tenantId, scopeId);
1289
+ await this.assertLive(tenantId, scopeId);
1053
1290
  await this.migrateAndRecord(scopeId);
1054
1291
  const store = await this.resolveAttachmentStore(tenantId);
1055
1292
  return this.buildAttachmentSurface({ principal }, tenantId, scopeId, store);
@@ -1122,17 +1359,14 @@ export class CloudflareScopeHost {
1122
1359
  const store = await this.resolveAttachmentStore(conn.tenant_id);
1123
1360
  return this.buildAttachmentSurface({ connectionId }, conn.tenant_id, scopeId, store, forEvent);
1124
1361
  }
1125
- /** Resolve the per-tenant R2 blob store, or fail closed exactly as `attachments` did. */
1362
+ /** Resolve the per-tenant R2 blob store, or fail closed. */
1126
1363
  async resolveAttachmentStore(tenantId) {
1127
- if (!this.attachmentBuckets) {
1128
- throw new Error(`attachments are not configured on this host (#473): pass ` +
1129
- `CloudflareScopeHostOptions.attachmentBuckets — (tenantId) => ` +
1130
- `env[blobStoreBindingName('<BINDING>', tenantId)] for the vertical's declared blob store`);
1131
- }
1132
1364
  const bucket = await this.attachmentBuckets(tenantId);
1133
1365
  if (!bucket) {
1134
- throw new Error(`no attachment bucket resolved for tenant ${tenantId} (#473) — is the per-tenant blob ` +
1135
- `store provisioned and its r2_bucket binding attached to the serving script?`);
1366
+ throw new Error(`no attachment bucket resolved for tenant ${tenantId} (#473, #1995) — this script has no ` +
1367
+ `${blobStoreBindingName(ATTACHMENT_BLOB_BINDING, tenantId)} binding. A vertical whose modules declare ` +
1368
+ `attachmentTargets gets one per installed tenant once it is pushed with a CLI that declares the ` +
1369
+ `store (re-push); off the platform, pass CloudflareScopeHostOptions.attachmentBuckets.`);
1136
1370
  }
1137
1371
  return r2TenantBlobStore(bucket);
1138
1372
  }
@@ -1365,7 +1599,7 @@ export class CloudflareScopeHost {
1365
1599
  async getScope(principal, tenantId, scopeId, options) {
1366
1600
  // Lifecycle gates (control-plane.md §4.1/§4.2), the K-3 fail-closed path,
1367
1601
  // evaluated durably in the ControlPlaneDO. A throw propagates here.
1368
- await this.validateScopeAccess(tenantId, scopeId);
1602
+ await this.assertLive(tenantId, scopeId);
1369
1603
  await this.migrateAndRecord(scopeId);
1370
1604
  return this.buildStub(tenantId, scopeId, principal, undefined, undefined, options);
1371
1605
  }
@@ -1440,7 +1674,7 @@ export class CloudflareScopeHost {
1440
1674
  const record = await this.resolveImpersonation(session, tenantId, scopeId);
1441
1675
  // The same lifecycle gate the principal door applies: a suspended tenant
1442
1676
  // refuses a support session exactly as it refuses a user.
1443
- await this.validateScopeAccess(tenantId, scopeId);
1677
+ await this.assertLive(tenantId, scopeId);
1444
1678
  await this.migrateAndRecord(scopeId);
1445
1679
  return this.buildStub(tenantId, scopeId, record.principal, undefined, undefined, options, record.id);
1446
1680
  }
@@ -1454,7 +1688,7 @@ export class CloudflareScopeHost {
1454
1688
  if (!this.moduleIds.has(moduleId)) {
1455
1689
  throw new Error(`module not registered on this host: ${moduleId}`);
1456
1690
  }
1457
- await this.validateScopeAccess(tenantId, scopeId);
1691
+ await this.assertLive(tenantId, scopeId);
1458
1692
  await this.migrateAndRecord(scopeId);
1459
1693
  return this.buildStub(tenantId, scopeId, undefined, undefined, moduleId);
1460
1694
  }
@@ -1487,7 +1721,7 @@ export class CloudflareScopeHost {
1487
1721
  * capability row lives. The answer is re-parsed on this side of the RPC.
1488
1722
  */
1489
1723
  async exchangeCapability(tenantId, scopeId, secret, options) {
1490
- await this.validateScopeAccess(tenantId, scopeId);
1724
+ await this.assertLive(tenantId, scopeId);
1491
1725
  await this.migrateAndRecord(scopeId);
1492
1726
  const outcome = await this.scopeStub(scopeId).exchangeCapability(secret, tenantId, scopeId, options?.mode);
1493
1727
  return capabilityExchange.nullable().parse(outcome);
@@ -1501,7 +1735,7 @@ export class CloudflareScopeHost {
1501
1735
  if (!plausibleSessionToken(sessionToken)) {
1502
1736
  throw substratError('unauthenticated', 'not a capability session token');
1503
1737
  }
1504
- await this.validateScopeAccess(tenantId, scopeId);
1738
+ await this.assertLive(tenantId, scopeId);
1505
1739
  await this.migrateAndRecord(scopeId);
1506
1740
  const hash = await capabilityTokenHash(sessionToken);
1507
1741
  return this.buildStub(tenantId, scopeId, undefined, undefined, undefined, options, undefined, hash);
@@ -1577,7 +1811,7 @@ export class CloudflareScopeHost {
1577
1811
  }
1578
1812
  this.assertServedHere(record, scopeId, verb);
1579
1813
  }
1580
- await this.validateScopeAccess(tenantId, scopeId);
1814
+ await this.assertLive(tenantId, scopeId);
1581
1815
  await this.migrateAndRecord(scopeId);
1582
1816
  }
1583
1817
  /**
@@ -1590,7 +1824,7 @@ export class CloudflareScopeHost {
1590
1824
  if (!plausibleSessionToken(sessionToken)) {
1591
1825
  throw substratError('unauthenticated', 'not a capability session token');
1592
1826
  }
1593
- await this.validateScopeAccess(tenantId, scopeId);
1827
+ await this.assertLive(tenantId, scopeId);
1594
1828
  await this.migrateAndRecord(scopeId);
1595
1829
  const store = await this.resolveAttachmentStore(tenantId);
1596
1830
  const capabilitySession = await capabilityTokenHash(sessionToken);
@@ -1687,6 +1921,10 @@ export class CloudflareScopeHost {
1687
1921
  if (!rec || rec.status !== 'active')
1688
1922
  return report;
1689
1923
  }
1924
+ else if (await this.lifecycleHeld(scopeId)) {
1925
+ // #1713: a held scope emits nothing, so judging it stale would only raise a false alarm.
1926
+ return report;
1927
+ }
1690
1928
  const stub = this.scopeStub(scopeId);
1691
1929
  const windows = new Map();
1692
1930
  for (const specs of this.moduleFreshness.values()) {
@@ -1725,14 +1963,23 @@ export class CloudflareScopeHost {
1725
1963
  return report;
1726
1964
  // Only run on a live scope of this tenant; a scope archived between the sweep's
1727
1965
  // enumeration and here simply has nothing due. A CP-less host has no directory
1728
- // to ask (#461) — it already trusts the router-asserted (tenant, scope) for the
1729
- // whole request path, and lifecycle for its scopes lives wherever provisioning
1730
- // does, so the grant check below is the only gate it can and does enforce.
1966
+ // to ask (#461), so it reads the lifecycle the platform delivered to the scope (#1713).
1731
1967
  if (!this.cpLess) {
1732
1968
  const rec = await this.cp.getScopeRecord(tenantId, scopeId);
1733
1969
  if (!rec || rec.status !== 'active')
1734
1970
  return report;
1735
1971
  }
1972
+ else if (await this.lifecycleHeld(scopeId)) {
1973
+ // #1713: the lifecycle the platform delivered holds the scope. Every schedule is
1974
+ // `skipped` and no cadence row moves, as under the kill switch, so a schedule that
1975
+ // came due meanwhile fires once on the first pass after the scope is live again.
1976
+ for (const schedule of schedules) {
1977
+ report.skipped += 1;
1978
+ report.runs.push({ operation: schedule.operation, outcome: 'skipped' });
1979
+ }
1980
+ report.lifecycleHeld = true;
1981
+ return report;
1982
+ }
1736
1983
  const stub = this.scopeStub(scopeId);
1737
1984
  // The grant IS the switch (#383), and the kill switch is its lever (#1666): the
1738
1985
  // kernel's `systemScheduleState`, the one predicate both adapters run. A scope that
@@ -1756,6 +2003,7 @@ export class CloudflareScopeHost {
1756
2003
  return report;
1757
2004
  }
1758
2005
  const now = Date.now();
2006
+ const lines = asyncLinePass();
1759
2007
  for (const schedule of schedules) {
1760
2008
  const last = await stub.scheduleLastRun(schedule.operation);
1761
2009
  const lastRun = last ? Date.parse(last) : null;
@@ -1770,24 +2018,43 @@ export class CloudflareScopeHost {
1770
2018
  // produces either way carries it, and so does the row below.
1771
2019
  const invocationId = ulid();
1772
2020
  let status = 'ok';
2021
+ // #1901: the run's line, under the call's own id — the one its `ctx.log` lines carry.
2022
+ const startedAt = Date.now();
2023
+ let emitted;
2024
+ let failure;
1773
2025
  try {
1774
2026
  const scope = await this.getSystemScope(moduleId, tenantId, scopeId);
1775
- await scope.invoke(schedule.operation, schedule.input, { invocationId });
2027
+ await scope.invoke(schedule.operation, schedule.input, { invocationId, onEmitted: (r) => (emitted = r) });
1776
2028
  report.fired += 1;
1777
2029
  }
1778
2030
  catch (err) {
1779
2031
  status = 'failed';
2032
+ failure = { error: err };
1780
2033
  report.failed += 1;
1781
2034
  report.errors.push({
1782
2035
  operation: schedule.operation,
1783
2036
  error: err instanceof Error ? err.message : String(err),
1784
2037
  });
1785
2038
  }
2039
+ lines.write({
2040
+ kind: 'schedule',
2041
+ tenantId,
2042
+ scopeId,
2043
+ invocationId,
2044
+ operation: schedule.operation,
2045
+ startedAt,
2046
+ outcome: status,
2047
+ ...failure,
2048
+ dueAt: lastRun === null ? null : new Date(dueAt).toISOString(),
2049
+ latenessMs: lastRun === null ? null : now - dueAt,
2050
+ ...(emitted ? { emitted } : {}),
2051
+ });
1786
2052
  // #1288: 'schedule', whatever this operation happens to be called — including
1787
2053
  // `freshness:<something>`, which is exactly the row the evaluator no longer eats.
1788
2054
  await stub.recordScheduleRun(schedule.operation, new Date(now).toISOString(), status, 'schedule', invocationId);
1789
2055
  report.runs.push({ operation: schedule.operation, outcome: status === 'ok' ? 'ok' : 'failed' });
1790
2056
  }
2057
+ lines.end();
1791
2058
  return report;
1792
2059
  }
1793
2060
  /** The stub body, shared by the principal, connection, system and impersonation doors. */
@@ -1848,7 +2115,13 @@ export class CloudflareScopeHost {
1848
2115
  // (`cp.tenantHoldsEntitlement`); for a hosted/CP-less scope that call is a trusting
1849
2116
  // no-op, so the SAME `requiredKey` is passed to the DO, which fails closed against
1850
2117
  // its PROJECTED entitlements (#304). One or the other enforces, never neither.
1851
- const requiredKey = operationEntitlement.get(operation);
2118
+ //
2119
+ // EXCEPTION (#1654): a module's own declared schedule, through this system door,
2120
+ // demands no SKU — its `system:<moduleId>` grant is the switch, as it already is for
2121
+ // permissions (#383). `requiredEntitlementFor` holds the whole rule and why; it
2122
+ // answers `undefined` there, so neither the coordinator nor the DO asks. Every other
2123
+ // door passes no `systemModuleId` and is gated exactly as before.
2124
+ const requiredKey = requiredEntitlementFor(operation, operationEntitlement.get(operation), systemModuleId);
1852
2125
  if (requiredKey && !(await cp.tenantHoldsEntitlement(tenantId, requiredKey))) {
1853
2126
  // Required AND held (#691) — a second CP read, but only on the denial path.
1854
2127
  const now = new Date().toISOString();
@@ -2103,6 +2376,8 @@ export class CloudflareScopeHost {
2103
2376
  // returns it with the previous status, so the trail cannot disagree with
2104
2377
  // the directory about which deployment the action touched.
2105
2378
  await this.recordAdmin(actor, action, { tenantId, scopeId, vertical: before.vertical }, { status: before.status }, { status: to, ...afterExtra });
2379
+ // #1713: after the directory moved and the move was audited, never before or instead.
2380
+ await this.deliverLifecycles(actor, { scopeId });
2106
2381
  };
2107
2382
  const writeGrant = async (subject, permission, node, entity, expiresAt) => {
2108
2383
  if (entity) {
@@ -3032,7 +3307,7 @@ export class CloudflareScopeHost {
3032
3307
  if (!v)
3033
3308
  throw substratError('not_found', `unknown version ${versionId}`);
3034
3309
  if (v.admission === 'admitted') {
3035
- throw new Error(`version ${versionId} is already admitted — it may be bound`);
3310
+ throw substratError('conflict', `version ${versionId} is already admitted — it may be bound`);
3036
3311
  }
3037
3312
  if (v.admission === 'rejected')
3038
3313
  return;
@@ -3044,7 +3319,7 @@ export class CloudflareScopeHost {
3044
3319
  if (!incoming)
3045
3320
  throw substratError('not_found', `unknown version ${versionId}`);
3046
3321
  if (incoming.vertical_slug !== verticalSlug) {
3047
- throw new Error(`version ${versionId} belongs to '${incoming.vertical_slug}'`);
3322
+ throw substratError('conflict', `version ${versionId} belongs to '${incoming.vertical_slug}'`);
3048
3323
  }
3049
3324
  if (incoming.admission !== 'admitted') {
3050
3325
  throw substratError('conflict', `version ${versionId} is ${incoming.admission}, not admitted — it cannot be promoted`);
@@ -3104,7 +3379,7 @@ export class CloudflareScopeHost {
3104
3379
  if (channel === 'prod' && !incoming.deployment_ref) {
3105
3380
  const owning = await this.cp.readVertical(verticalSlug);
3106
3381
  if (owning && owning.owner_tenant !== null && !owning.listed) {
3107
- const bound = (await this.cp.listScopes({ tenantId: owning.owner_tenant, vertical: verticalSlug, status: ['active'] })).filter((s) => !s.forked_from && s.kind !== 'preview');
3382
+ const bound = (await this.cp.listScopes({ tenantId: owning.owner_tenant, vertical: verticalSlug, status: ['active'] })).filter(isPrimaryScopeRow);
3108
3383
  for (const s of bound) {
3109
3384
  if (s.vertical_version_id === versionId)
3110
3385
  continue;
@@ -3262,6 +3537,12 @@ export class CloudflareScopeHost {
3262
3537
  await this.cp.setScopeServingRef(scopeId, servingRef);
3263
3538
  await this.recordAdmin(actor, 'setScopeServingRef', { tenantId, scopeId }, { servingRef: scope.serving_ref ?? null }, { servingRef, ...(ack.exportBreak ? { acknowledged: ack } : {}) });
3264
3539
  },
3540
+ recordKeptCopyResolution: async (actor, tenantId, scopeId, r) => {
3541
+ const scope = await this.cp.getScopeRecord(tenantId, scopeId);
3542
+ if (!scope)
3543
+ throw substratError('not_found', `unknown scope ${scopeId} in tenant ${tenantId}`);
3544
+ await this.recordAdmin(actor, 'resolveKeptCopy', { tenantId, scopeId }, { script: r.script, keptAt: r.keptAt, revision: r.revisionBefore }, { action: r.action, liveScript: r.liveScript, revision: r.revisionAfter });
3545
+ },
3265
3546
  setScopeExpiresAt: async (actor, tenantId, scopeId, expiresAt) => {
3266
3547
  const scope = await this.cp.getScopeRecord(tenantId, scopeId);
3267
3548
  if (!scope)
@@ -3366,6 +3647,8 @@ export class CloudflareScopeHost {
3366
3647
  setTenantStatus: async (actor, tenantId, status) => {
3367
3648
  const before = await this.cp.setTenantStatus(tenantId, status);
3368
3649
  await this.recordAdmin(actor, 'setTenantStatus', { tenantId }, { status: before }, { status });
3650
+ // #1713: every hosted scope under the tenant holds or resumes its work by this.
3651
+ await this.deliverLifecycles(actor, { tenantId });
3369
3652
  },
3370
3653
  setTenantName: async (actor, tenantId, name) => {
3371
3654
  const before = await this.cp.setTenantName(tenantId, name);
@@ -3660,6 +3943,26 @@ export class CloudflareScopeHost {
3660
3943
  await this.recordAccess(actor, 'listDenials', { tenantId, scopeId }, filter ?? null, rows.length);
3661
3944
  return rows;
3662
3945
  },
3946
+ // #1686: the operator's capability read. K-3 on the directory before the DO, like the
3947
+ // denial log. On the shared control plane a scope a vertical's own deployment serves
3948
+ // holds its storage THERE, and this namespace holds a placeholder: the control-plane
3949
+ // API asks the vertical (`listCapabilitiesLocal`), so this branch is the co-located one.
3950
+ listCapabilities: async (actor, tenantId, scopeId, filter) => {
3951
+ // Parsed here too: a typed refusal thrown inside the DO arrives as a bare message.
3952
+ const parsed = capabilityFilter.parse(filter ?? {});
3953
+ const rec = await this.scopeRecordForRead(tenantId, scopeId);
3954
+ this.assertServedHere(rec, scopeId, 'listCapabilities');
3955
+ const page = await this.scopeStub(scopeId).listCapabilities(parsed);
3956
+ await this.recordAccess(actor, 'listCapabilities', { tenantId, scopeId }, filter ?? null, page.entries.length);
3957
+ return page;
3958
+ },
3959
+ listRefusals: async (actor, tenantId, scopeId, filter) => {
3960
+ // #1745: the denial log's discipline — K-3 on the directory before the DO is reached.
3961
+ await this.scopeRecordForRead(tenantId, scopeId);
3962
+ const rows = await this.scopeStub(scopeId).listRefusals(filter);
3963
+ await this.recordAccess(actor, 'listRefusals', { tenantId, scopeId }, filter ?? null, rows.length);
3964
+ return rows;
3965
+ },
3663
3966
  summarizeDenials: async (actor, tenantId, scopeId, filter) => {
3664
3967
  await this.scopeRecordForRead(tenantId, scopeId);
3665
3968
  const summary = await this.scopeStub(scopeId).summarizeDenials(filter);
@@ -4218,6 +4521,11 @@ export class CloudflareScopeHost {
4218
4521
  const { tenantId, scopeId, ...after } = ownerTransferAudit.parse(entry);
4219
4522
  await this.recordAdmin(actor, 'transferOwner', { tenantId, scopeId }, null, after);
4220
4523
  },
4524
+ /** #2005: one change to a scope's copy marker, written around the vertical's own change. */
4525
+ recordCopyMark: async (actor, entry) => {
4526
+ const { tenantId, scopeId, action, ...after } = copyMarkAudit.parse(entry);
4527
+ await this.recordAdmin(actor, action === 'mark' ? 'markScopeCopy' : 'clearScopeCopyMark', { tenantId, scopeId }, null, after);
4528
+ },
4221
4529
  accessLog: async (actor, filter) => {
4222
4530
  const rows = await this.cp.accessLog({
4223
4531
  actor: filter?.actor,
@@ -4439,6 +4747,184 @@ export class CloudflareScopeHost {
4439
4747
  return;
4440
4748
  throw refusal.code ? substratError(refusal.code, refusal.message) : new Error(refusal.message);
4441
4749
  }
4750
+ /**
4751
+ * THE lifecycle gate (#1713) at every door that runs a scope's work: a request's stubs and
4752
+ * attachments, a capability or impersonation session, a peer call or delivery, a subscription,
4753
+ * and the entry points no request asked for — the retry driver, the job runner, a schedule's
4754
+ * system stub, a routed connector delivery.
4755
+ *
4756
+ * `validateScopeAccess` first, which is the directory's gate where there is a directory. A
4757
+ * CP-less host has none, so it then reads the lifecycle the platform delivered to the scope's
4758
+ * own storage (`/internal/lifecycle`), judged by the kernel's `lifecycleRefusal` in the
4759
+ * directory's own words. The router refuses a suspended scope's requests first (#1730); this
4760
+ * is the deployment's own half, and the only one a timer or a retry ever meets.
4761
+ *
4762
+ * The admin and diagnostic reads (dead letters, job runs, platform requests, delivered
4763
+ * grants) keep `validateScopeAccess` alone: an operator still reads a suspended scope.
4764
+ */
4765
+ async assertLive(tenantId, scopeId) {
4766
+ await this.validateScopeAccess(tenantId, scopeId);
4767
+ const refusal = await this.cpLessRefusal(scopeId, { tenantId, scopeId });
4768
+ if (refusal)
4769
+ throw new Error(refusal);
4770
+ }
4771
+ /** The delivered lifecycle's refusal on a CP-less host (#1713); always null with a directory. */
4772
+ async cpLessRefusal(scopeId, ids) {
4773
+ if (!this.cpLess)
4774
+ return null;
4775
+ return lifecycleRefusal(await this.scopeStub(scopeId).lifecycle(), ids);
4776
+ }
4777
+ /**
4778
+ * Deliver the directory's lifecycle to the deployments serving the matching scopes (#1713):
4779
+ * one scope after its transition, a tenant's scopes after the tenant's, or the heal sweep's
4780
+ * drift. Each delivery carries the directory's revisions, read in the same query as the
4781
+ * statuses, so a late one cannot undo a later transition: the deployment keeps only a delivery
4782
+ * strictly newer than the one it holds.
4783
+ *
4784
+ * Never throws. A transition is the operator's lever in an incident, and the router refuses
4785
+ * the scope's requests as soon as the directory moves (#1730), so a deployment that cannot be
4786
+ * reached must not refuse the lever. Each failure is an ops-failure row, no receipt is written,
4787
+ * and the heal sweep delivers again.
4788
+ */
4789
+ async deliverLifecycles(actor, filter) {
4790
+ const report = { attempted: 0, delivered: 0, failed: 0 };
4791
+ const delegation = this.lifecycleDelegation;
4792
+ if (!delegation || this.cpLess)
4793
+ return report;
4794
+ let targets;
4795
+ try {
4796
+ targets = await this.cp.lifecycleTargets(filter);
4797
+ }
4798
+ catch (err) {
4799
+ console.error('substrat: could not read the scopes to deliver a lifecycle to (#1713)', err);
4800
+ return report;
4801
+ }
4802
+ for (const first of targets) {
4803
+ const tenantId = first.tenant_id;
4804
+ const scopeId = first.scope_id;
4805
+ let t = first;
4806
+ let lifecycle = this.lifecycleOfTarget(t);
4807
+ if (!lifecycle) {
4808
+ // A status this code does not know (a newer directory): delivering a guess could lift a hold.
4809
+ report.failed += 1;
4810
+ console.error(`substrat: scope ${scopeId} has a lifecycle this code cannot read (#1713)`);
4811
+ continue;
4812
+ }
4813
+ // A live scope whose deployment already acknowledged exactly this has nothing to receive,
4814
+ // and neither does one with no receipt in a directory never restored: a deployment holding
4815
+ // no lifecycle runs live. That keeps an activation, and every transition before a
4816
+ // deployment carries the route, from posting a delivery that changes nothing. After a
4817
+ // restore (epoch > 0) no receipt says nothing about what the deployment holds, so it is
4818
+ // delivered. A HOLD is always delivered.
4819
+ if (lifecycleRefusal(lifecycle) === null && (t.delivered === null ? t.epoch === 0 : t.delivered === lifecycleReceipt(lifecycle))) {
4820
+ continue;
4821
+ }
4822
+ report.attempted += 1;
4823
+ try {
4824
+ let answer = await delegation.deliver({ tenantId, scopeId, lifecycle });
4825
+ // SINGLE AUTHORITY (the invariant this rests on): an environment has exactly one directory,
4826
+ // the singleton `CONTROL_PLANE.idFromName('control-plane')`. A "fresh directory" restore is
4827
+ // that same object restored after its storage was lost, never a second live writer, and
4828
+ // the raise below always re-reads the CURRENT store and delivers what it says. Two control
4829
+ // planes healing one dispatch namespace at once is a split brain that corrupts far more
4830
+ // than lifecycle, and is out of scope here; fencing it in lifecycle alone would guarantee
4831
+ // nothing.
4832
+ //
4833
+ // The scope refused us as OLDER while holding a revision this directory's history did not
4834
+ // write: an epoch ahead of ours, or ours with other counters. Only another history of the
4835
+ // directory delivers those — the one a fresh-directory restore replaced, whose epoch a
4836
+ // clock running behind (or a restore in the same millisecond) cannot outrank. A scope only
4837
+ // ever holds epochs a directory minted, so this directory learns past it: its epoch is
4838
+ // raised above the one held (monotonic, one statement), and the scope is delivered again.
4839
+ const held = answer.lifecycle.revision;
4840
+ const foreign = !answer.applied &&
4841
+ held !== undefined &&
4842
+ (held.epoch > lifecycle.revision.epoch ||
4843
+ (held.epoch === lifecycle.revision.epoch && lifecycleReceipt(answer.lifecycle) !== lifecycleReceipt(lifecycle)));
4844
+ // Bounded, because the deployment answering is the vertical's own code: only a directory
4845
+ // that has been restored can meet another history (epoch > 0), and a legitimate epoch is
4846
+ // a mint time, so one further ahead than the skew is forged or broken. Either way nothing
4847
+ // is raised; the refusal is an ops failure for an operator to look at.
4848
+ if (foreign && (lifecycle.revision.epoch === 0 || held.epoch > Date.now() + LIFECYCLE_EPOCH_SKEW_MS)) {
4849
+ report.failed += 1;
4850
+ await this.admin
4851
+ .recordOpsFailure({
4852
+ actor,
4853
+ operation: 'scope.lifecycle',
4854
+ stage: 'foreign-epoch',
4855
+ tenantId,
4856
+ scopeId,
4857
+ message: `the deployment refused this directory's lifecycle while holding epoch ${held.epoch} ` +
4858
+ `(this directory is at ${lifecycle.revision.epoch}); not raised past it — ` +
4859
+ (lifecycle.revision.epoch === 0
4860
+ ? 'this directory has never been restored, so no other history should have written it'
4861
+ : 'it is further ahead of the clock than any directory mints'),
4862
+ })
4863
+ .catch((e) => console.error('substrat: could not record a foreign lifecycle epoch (#1713)', e));
4864
+ continue;
4865
+ }
4866
+ if (foreign) {
4867
+ await this.cp.raiseLifecycleEpoch(held.epoch + 1);
4868
+ const [again] = await this.cp.lifecycleTargets({ scopeId });
4869
+ const next = again ? this.lifecycleOfTarget(again) : null;
4870
+ if (again && next) {
4871
+ t = again;
4872
+ lifecycle = next;
4873
+ answer = await delegation.deliver({ tenantId, scopeId, lifecycle });
4874
+ }
4875
+ }
4876
+ // The receipt is what the scope HOLDS: a newer delivery it kept instead is the truth.
4877
+ await this.cp.recordLifecycleReceipt(scopeId, lifecycleReceipt(answer.lifecycle), new Date().toISOString());
4878
+ report.delivered += 1;
4879
+ }
4880
+ catch (err) {
4881
+ report.failed += 1;
4882
+ try {
4883
+ await this.admin.recordOpsFailure({
4884
+ actor,
4885
+ operation: 'scope.lifecycle',
4886
+ stage: 'deliver',
4887
+ tenantId,
4888
+ scopeId,
4889
+ message: `the lifecycle (scope ${t.scope_status}, tenant ${t.tenant_status}) did not reach the ` +
4890
+ `deployment serving this scope; the heal sweep will deliver it again: ${err instanceof Error ? err.message : String(err)}`,
4891
+ });
4892
+ }
4893
+ catch (recordErr) {
4894
+ console.error('substrat: could not record a lifecycle delivery failure (#1713)', recordErr);
4895
+ }
4896
+ }
4897
+ }
4898
+ return report;
4899
+ }
4900
+ /** One directory row as the lifecycle a delivery carries (#1713), or null for a status this code does not know. */
4901
+ lifecycleOfTarget(t) {
4902
+ const parsed = scopeLifecycle.safeParse({
4903
+ scope: t.scope_status,
4904
+ tenant: t.tenant_status,
4905
+ at: new Date().toISOString(),
4906
+ revision: { epoch: t.epoch, scope: t.scope_rev, tenant: t.tenant_rev },
4907
+ });
4908
+ return parsed.success ? parsed.data : null;
4909
+ }
4910
+ /**
4911
+ * The heal sweep (#1713): deliver the lifecycle to every hosted scope whose deployment's
4912
+ * acknowledged state differs from the directory, and again to every scope held now — which is
4913
+ * what puts a hold back on a store a carry or a restore landed without it. Bounded per pass
4914
+ * (`limit`), drifted scopes first. Run from the shared control plane's cron.
4915
+ */
4916
+ async healLifecycles(actor, opts) {
4917
+ return this.deliverLifecycles(actor, { drift: true, limit: opts?.limit ?? 200 });
4918
+ }
4919
+ /**
4920
+ * Whether a CP-less host holds this scope's work (#1713): `assertLive`'s predicate, answered
4921
+ * rather than thrown, for the entry points that defer instead of refusing — a sweep pass, a
4922
+ * schedule run, an executor drain. Always false on a host with a directory, whose entry
4923
+ * points ask the directory themselves.
4924
+ */
4925
+ async lifecycleHeld(scopeId) {
4926
+ return (await this.cpLessRefusal(scopeId)) !== null;
4927
+ }
4442
4928
  /**
4443
4929
  * K-3's cross-check on its own: the (tenant, scope) pair must exist and agree before a
4444
4930
  * subject-key operation touches anything. Without it a caller could reach another
@@ -4955,16 +5441,16 @@ export class CloudflareScopeHost {
4955
5441
  * suspended or archiving scope could still be addressed and handed a 101 — a
4956
5442
  * scope that refuses every ordinary read while quietly holding an open socket.
4957
5443
  *
4958
- * Costs a CP-less vertical nothing: `validateScopeAccess` is a **no-op** on the
4959
- * null control plane (the router already gated lifecycle and tenancy from the
4960
- * shared directory), so this is the hosted-vertical shape unchanged.
5444
+ * On a CP-less vertical, `validateScopeAccess` is a no-op on the null control
5445
+ * plane, and `assertLive` reads the lifecycle the platform delivered to the
5446
+ * scope's own storage instead (#1713).
4961
5447
  *
4962
5448
  * `migrateAndRecord`, not the DO's own `ensureMigrations`: the DO migrates
4963
5449
  * itself when the socket opens either way, but only this reports the applied
4964
5450
  * count to the directory — so a scope whose first contact after a deploy is a
4965
5451
  * subscription does not go dark in the migration fleet view.
4966
5452
  */
4967
- await this.validateScopeAccess(tenantId, scopeId);
5453
+ await this.assertLive(tenantId, scopeId);
4968
5454
  await this.migrateAndRecord(scopeId);
4969
5455
  // Asserted, not carried through from the client: the principal is the
4970
5456
  // vertical's own resolution of its session, and the tenant and scope are the
@@ -5327,14 +5813,19 @@ export class CloudflareScopeHost {
5327
5813
  // side before the call; what runs HERE is the half only this deployment can
5328
5814
  // enforce: the scope's own permission check against its delivered
5329
5815
  // `connection:<id>` tuple, in the scope's own DO. Fail closed — no grant, no
5330
- // effect — exactly as for any other caller.
5816
+ // effect — exactly as for any other caller. And the lifecycle delivered to the
5817
+ // scope (#1713): a held scope's invoke and bytes doors refuse, as every other door does.
5331
5818
  /** Invoke ONE operation in this deployment as a CONNECTION (#574). */
5332
5819
  async connectorInvokeLocal(connectionId, tenantId, scopeId, operation, input) {
5820
+ await this.assertLive(tenantId, scopeId); // #1713: a held scope runs no connector work either
5333
5821
  await this.migrateAndRecord(scopeId);
5334
5822
  return this.buildStub(tenantId, scopeId, undefined, connectionId).invoke(operation, input);
5335
5823
  }
5336
5824
  /** Land provider bytes in this deployment as a CONNECTION — the bytes leg (#574). */
5337
5825
  async connectorAttachmentUploadLocal(connectionId, tenantId, scopeId, upload) {
5826
+ // The lifecycle gate every attachments door takes (#1713), ahead of the bucket: a held
5827
+ // scope's provider bytes wait for the delivery's retry rather than land (#1995).
5828
+ await this.assertLive(tenantId, scopeId);
5338
5829
  await this.migrateAndRecord(scopeId);
5339
5830
  const store = await this.resolveAttachmentStore(tenantId);
5340
5831
  return this.buildAttachmentSurface({ connectionId }, tenantId, scopeId, store).upload(upload);
@@ -5347,6 +5838,8 @@ export class CloudflareScopeHost {
5347
5838
  * so a caller falls back rather than failing a dispatch over a missing file.
5348
5839
  */
5349
5840
  async connectorAttachmentOpenLocal(connectionId, tenantId, scopeId, attachmentId, eventId) {
5841
+ // Same gate as the upload leg: a held scope's bytes are not read out of its bucket.
5842
+ await this.assertLive(tenantId, scopeId);
5350
5843
  await this.migrateAndRecord(scopeId);
5351
5844
  const store = await this.resolveAttachmentStore(tenantId);
5352
5845
  return this.buildAttachmentSurface({ connectionId }, tenantId, scopeId, store,