@substrat-run/adapter-cloudflare 0.136.0 → 0.138.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/host.js CHANGED
@@ -1,12 +1,14 @@
1
1
  import { env as ambientEnv } from 'cloudflare:workers';
2
2
  import { isRewindRefusal, REWIND_REFUSED } from './rewind-refusal.js';
3
- import { delegatedReadParams, fromWireFailure, exportReadInput, exportedBatch, importBatch, importResult, importState, importsOfManifestJson, exportsOfManifestJson, importCursorMove, importCursorMoveAt, importCursorMoved, accessLogEntry, adminLogEntry, opsFailureEntry, opsFailureFingerprint, issueEntry, sweepRunEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunsPayload, modelUsageEntry, attachmentRecord, createTenantInput, identityLink, identityPool, createOrgInput, promotionAcknowledgement, bindAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, registerVerticalInput, vertical as verticalSchema, verticalServingState, verticalChannel, verticalVersion, connection, capabilityExchange, capabilityGrant, principalId, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, systemSwitch, systemSwitchOutcome, systemScheduleEntry, systemSwitchRecord, entitlementGrant, entitlementGrantInput, capabilityFilter, instant, meterReading, subjectRef, createConnectionInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, org as orgSchema, orgMembership, resolvedIdentity, identityMembership, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, delegatedReadRecord, ownerTransferAudit, copyMarkAudit, subjectShredReceipt, connectorDispatchKind, callsOfManifestJson, outboundOfManifestJson, substratError, redrainEventsInput, coverage, peerCoverage, peerGrantsEntry, peerSwitch, peerSwitchOutcome, verticalCaller, verticalResolution, verticalSlug, entityObjectRef, ATTACHMENT_BLOB_BINDING, blobStoreBindingName, lifecycleDelivery, scopeLifecycle, } from '@substrat-run/contracts';
3
+ import { isSystemDoorMoved, SYSTEM_DOOR_MOVED, SYSTEM_DOOR_REGATES } from './system-door.js';
4
+ import { delegatedReadParams, operationSeriesCount, fromWireFailure, exportReadInput, exportedBatch, importBatch, importResult, importState, importsOfManifestJson, exportsOfManifestJson, importCursorMove, importCursorMoveAt, importCursorMoved, accessLogEntry, adminLogEntry, opsFailureEntry, opsFailureFingerprint, issueEntry, sweepRunEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunsPayload, modelUsageEntry, attachmentRecord, createTenantInput, identityLink, identityPool, createOrgInput, promotionAcknowledgement, bindAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, registerVerticalInput, vertical as verticalSchema, verticalServingState, verticalChannel, verticalVersion, connection, capabilityExchange, mintedCapability, capabilityGrant, principalId, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, systemSwitch, systemSwitchOutcome, systemScheduleEntry, systemSwitchRecord, entitlementGrant, entitlementGrantInput, capabilityFilter, instant, meterReading, subjectRef, createConnectionInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, org as orgSchema, orgMembership, resolvedIdentity, identityMembership, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, delegatedReadRecord, ownerTransferAudit, copyMarkAudit, subjectShredReceipt, connectorDispatchKind, callsOfManifestJson, outboundOfManifestJson, SCOPE_GATE_REASONS, substratError, redrainEventsInput, coverage, peerCoverage, peerGrantsEntry, peerSwitch, peerSwitchOutcome, verticalCaller, verticalResolution, verticalSlug, entityObjectRef, ATTACHMENT_BLOB_BINDING, blobStoreBindingName, lifecycleDelivery, scopeLifecycle, } from '@substrat-run/contracts';
4
5
  import { normalizeHostname, toRouteTarget } from './route-resolver.js';
5
- import { attachmentBlobKey, attachmentSha256, entitlementDenial, requiredEntitlementFor, foldMeterReading, parseValidationRecords, resolveScopeRecord, ulid, KEPT_COPY_REFUSAL, capabilityTokenHash, checkBecomeInput, plausibleSessionToken, foldModelUsage, backoffAt, resolveRetryPolicy, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, openSealed, ImpersonationRefused, assertSessionUsable, impersonationRowValues, mapImpersonationRow, newImpersonationSession, jobRunOf, runDueJobRuns, startJobRun, attachmentTextJob, assertAttachmentExtractors, assertJobRegistrable, isAttachmentTextRun, searchLimit, searchMatchExpression, globalFetch, assertRedrainWindow, platformRequestOf, undrainedEventsOf, inUnitMovesToAudit, staleCarryRevertRow, staleCarryReverts, withRecorded, systemSwitchedOffMessage, tenantSystemSwitchedOffMessage, CrossVerticalRegistry, importCursorSourceOf, exportBreaksOf, exportBreakRefusal, bindExportBreaksOf, bindExportBreakRefusal, collectPeers, peerSeats, connectorCallRecord, noopConnectorCallRecorder, recordConnectorCall, settleConnectionUse, splitManifestMigrations, unknownRoleError, assertRowLimit, assertRowOffset, INERT_SCOPE_REASON, lifecycleReceipt, lifecycleRefusal, isPrimaryScope, isPrimaryScopeRow, asyncInvocationId, asyncLinePass, } from '@substrat-run/kernel';
6
+ import { attachmentBlobKey, attachmentSha256, entitlementDenial, requiredEntitlementFor, foldMeterReading, parseValidationRecords, resolveScopeRecord, ulid, KEPT_COPY_REFUSAL, capabilityTokenHash, checkBecomeInput, plausibleSessionToken, foldModelUsage, backoffAt, resolveRetryPolicy, executorOutcomeOf, isDeliveryRefusal, refusalJournalText, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, openSealed, ImpersonationRefused, assertSessionUsable, impersonationRowValues, mapImpersonationRow, newImpersonationSession, jobRunOf, assertLeaseMs, runDueJobRuns, SYSTEM_DOOR_WAIT, startJobRun, attachmentTextJob, assertAttachmentExtractors, assertJobRegistrable, isAttachmentTextRun, searchLimit, searchMatchExpression, globalFetch, assertRedrainWindow, platformRequestOf, undrainedEventsOf, inUnitMovesToAudit, staleCarryRevertRow, staleCarryReverts, recordedOffFromWire, peerSubjectRef, SWITCH_KINDS, reassertActionOf, reassertEntry, reassertOffRow, reassertOnRow, switchActionOf, switchAuditSubject, switchNotFoundMessage, recordWriteSuperseded, switchSubjectOf, switchSupersededMessage, withRecorded, systemSwitchedOffMessage, tenantSystemSwitchedOffMessage, CrossVerticalRegistry, importCursorSourceOf, exportBreaksOf, exportBreakRefusal, bindExportBreaksOf, bindExportBreakRefusal, collectPeers, peerSeats, connectorCallRecord, noopConnectorCallRecorder, recordConnectorCall, settleConnectionUse, splitManifestMigrations, unknownRoleError, assertRowLimit, assertRowOffset, emptyImportResult, INERT_SCOPE_REASON, lifecycleReceipt, lifecycleRefusal, isPrimaryScope, isPrimaryScopeRow, asyncInvocationId, asyncLinePass, } from '@substrat-run/kernel';
6
7
  import { attributedHost } from '@substrat-run/kernel';
7
- import { isOrangeToOrange, isUpgradeRequest, LIVE_MODE_HEADER, LIVE_PRINCIPAL_HEADER, LIVE_SCOPE_HEADER, LIVE_SUBSCRIBE_PATH, LIVE_TENANT_HEADER, } from './live-reads.js';
8
+ import { isOrangeToOrange, isUpgradeRequest, LIVE_MODE_HEADER, LIVE_PRINCIPAL_HEADER, LIVE_SCOPE_HEADER, LIVE_SUBSCRIBE_PATH, LIVE_TENANT_HEADER, LIVE_WITHIN_HEADER, encodeLiveWithin, liveWithinOf, } from './live-reads.js';
8
9
  import { tenantStoreDatabaseName } from './d1.js';
9
10
  import { blobStoreBucketName, r2TenantBlobStore } from './r2.js';
11
+ import { unwrapReply } from './do-reply.js';
10
12
  /** DO row → contract shape. Never reads the secrets table — that is the split. */
11
13
  const toConnection = (r) => connection.parse({
12
14
  id: r.id,
@@ -25,18 +27,34 @@ const toConnection = (r) => connection.parse({
25
27
  createdAt: r.created_at,
26
28
  revokedAt: r.revoked_at,
27
29
  });
28
- /** Rethrow a capability attachment verb's failure, rebuilt with its code; else its value. */
29
- function unwrapCapabilityReply(reply) {
30
- if (reply.failure)
31
- throw fromWireFailure(reply.failure);
32
- return reply.value;
33
- }
34
30
  /**
35
31
  * How far ahead of the clock a held lifecycle epoch may be and still be learned past (#1713). An
36
32
  * epoch is the time a directory restore minted it, so a legitimate one is behind its minter's
37
33
  * clock; the skew allows for clocks that disagree, and bounds what a forged answer can move.
38
34
  */
39
35
  export const LIFECYCLE_EPOCH_SKEW_MS = 24 * 60 * 60 * 1000;
36
+ /**
37
+ * #2016: how many served scopes one heal pass asks for their tenant record, beyond the drift and the
38
+ * holds. Rotated least-recently-asked first, so a deployment that cannot answer yet (a build from
39
+ * before the record) costs at most this many deliveries a pass and starves nothing.
40
+ */
41
+ export const TENANT_UNRECORDED_PER_PASS = 50;
42
+ /**
43
+ * K-3's refusal of a (tenant, scope) pair the host does not hold — the same answer for a scope that
44
+ * does not exist and for one of another tenant (#1718), in one wording.
45
+ */
46
+ const unknownScopeForTenant = (tenantId, scopeId) => substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
47
+ /**
48
+ * #2016: the line a CP-less host writes each time it admits a pair to a scope with no tenant record
49
+ * and no role rows — nothing to hold the pair against. Ids only.
50
+ */
51
+ export const tenantUnrecordedLine = (tenantId, scopeId) => ({
52
+ substrat: 'tenant-unrecorded',
53
+ level: 'warn',
54
+ tenantId,
55
+ scopeId,
56
+ message: 'admitted to a scope whose tenant is not recorded; a reconcile or lifecycle delivery records it',
57
+ });
40
58
  /**
41
59
  * Refuse to mark a scope a copy unless the request classifies it non-primary (#2005). The
42
60
  * classification is the platform directory's; the vertical applies the same `isPrimaryScope` to
@@ -51,9 +69,10 @@ function assertCopyLineage(lineage) {
51
69
  * A control-plane stand-in for a CP-less vertical (scope-local-permissions.md Phase 3).
52
70
  * The hot path a served scope actually touches becomes trust-the-upstream:
53
71
  * - `scopeAccessRefusal` / `setMigrationState` → no-op: the router gates the scope's
54
- * lifecycle + tenancy from the shared directory, and the vertical trusts the asserted
55
- * node rather than re-reading a directory it does not have. Its own half of the
56
- * lifecycle gate reads the scope's storage instead (`assertLive`, #1713).
72
+ * lifecycle + tenancy from the shared directory, and the vertical does not re-read a
73
+ * directory it does not have. Its own half of both gates reads the scope's storage
74
+ * instead (`validateScopeAccess`): the tenant the scope was provisioned for (#2016) and the lifecycle
75
+ * the platform delivered (#1713).
57
76
  * - `tenantHoldsEntitlement` → true: the SKU was enforced on the shared control plane
58
77
  * at provision (before `provisionInstance`), so a scope that EXISTS here was granted
59
78
  * it upstream — a single-vertical deployment holds its own entitlements by construction.
@@ -68,22 +87,31 @@ function nullControlPlane() {
68
87
  recordAdmin: noop,
69
88
  recordAccess: noop,
70
89
  tenantHoldsEntitlement: async () => true,
71
- // #1674: the schedule switch's record is a DIRECTORY store, and a CP-less host has no
90
+ // #1674, #2029: the switches' record is a DIRECTORY store, and a CP-less host has no
72
91
  // directory — the shared control plane that delegates here keeps it. Same posture as
73
92
  // `recordAdmin`: nothing to write, nothing recorded to read back.
74
- recordSystemSwitchedOff: noop,
75
- recordSystemSwitchedOn: async () => null,
76
- restoreSystemSwitchRecord: noop,
93
+ recordSwitchedOff: async () => null,
94
+ recordSwitchedOn: async () => null,
95
+ restoreSwitchRecord: noop,
77
96
  listSystemSwitches: async () => [],
78
- systemSwitchRecordsOf: async () => [],
79
- switchedOffModulesOf: async () => [],
97
+ switchRecordsOf: async () => [],
98
+ switchRecordStatesOf: async () => [],
99
+ switchesOwedOf: async () => [],
100
+ clearSwitchOwed: noop,
101
+ // #1823: no tenant tuples here either — the platform reads them and sends `tenantHeld`.
102
+ tenantHeldOf: async () => [],
103
+ };
104
+ const unavailable = (method) => new Error(`control plane unavailable: '${method}' — this host is scope-local / CP-less ` +
105
+ `(docs/architecture/scope-local-permissions.md, Phase 3)`);
106
+ // A directory write through `reply` (#113) is named for the write it carries, not for `reply`.
107
+ passthrough.reply = async (method) => {
108
+ throw unavailable(String(method));
80
109
  };
81
110
  return new Proxy({}, {
82
111
  get: (_t, prop) => typeof prop === 'string' && prop in passthrough
83
112
  ? passthrough[prop]
84
113
  : async () => {
85
- throw new Error(`control plane unavailable: '${String(prop)}' — this host is scope-local / CP-less ` +
86
- `(docs/architecture/scope-local-permissions.md, Phase 3)`);
114
+ throw unavailable(String(prop));
87
115
  },
88
116
  });
89
117
  }
@@ -124,6 +152,35 @@ export const SWITCH_HOLD_EXTRA_WAITS = 3;
124
152
  * clock, against the releasing host's clock: minutes, not seconds.
125
153
  */
126
154
  export const SWITCH_HOLD_PENDING_MAX_MS = 5 * 60_000;
155
+ /**
156
+ * A move the deployment serving the scope applied, or may have, without attesting what the
157
+ * platform needs from it (#2045 Codex r3): the switch fence (`fenced`), or for a tenant-held
158
+ * module's OFF the evaluator that denies a tenant-level grant (`deniesTenantGrants`, #1823).
159
+ *
160
+ * The preflight (`switchTarget`'s `attestFence`) tells a deployment built before the fence apart
161
+ * before anything is recorded or moved. On such a deployment only an ON is refused: the race the
162
+ * fence closes is an older OFF landing after a newer ON, and with every ON refused there is no
163
+ * newer ON to land after, so an OFF goes through unfenced (#2045 follow-up). A kill switch that
164
+ * cannot be pulled until every vertical is re-pushed is the outage the fence must never cause.
165
+ * An unfenced OFF keeps its write-ahead owed mark, so the first re-assert after the vertical is
166
+ * redeployed moves the scope again under the record's fence.
167
+ *
168
+ * This error fires when a deployment that attested the fence answers a move without it — a
169
+ * rollback to an old build between the preflight and the move — or when an ON reaches a
170
+ * deployment without the fence. It is NOT compensated with an opposite move: an unfenced
171
+ * compensation can undo a newer call's switch (Codex r3, finding 2). The call's record and its
172
+ * write-ahead owed mark stay, and the call is refused.
173
+ */
174
+ class UnattestedSwitch extends Error {
175
+ }
176
+ /** A fenced move answered without `fenced`: the deployment predates the fence. */
177
+ const unfencedMessage = (scopeId) => `the deployment serving scope ${scopeId} predates the switch fence (#2045): it cannot refuse an older ` +
178
+ `switch call's OFF landing after this ON, so no switch is turned on there. Switching off still works. ` +
179
+ `Redeploy the vertical, then retry.`;
180
+ /** A tenant-held module's OFF answered without `deniesTenantGrants` (#1823). */
181
+ const undeniedMessage = (scopeId, moduleId) => `the deployment serving scope ${scopeId} predates the kill switch's tenant-grant denial (#1823): ` +
182
+ `module '${moduleId}' holds a tenant-level grant that its OFF would leave authorizing there. ` +
183
+ `Redeploy the vertical, then retry.`;
127
184
  /**
128
185
  * The attachment bucket a host resolves when it was handed no resolver (#1995): the
129
186
  * platform's per-tenant binding, read off this script's own env, named by the one shared
@@ -132,6 +189,49 @@ export const SWITCH_HOLD_PENDING_MAX_MS = 5 * 60_000;
132
189
  function ambientAttachmentBucket(tenantId) {
133
190
  return ambientEnv[blobStoreBindingName(ATTACHMENT_BLOB_BINDING, tenantId)] ?? null;
134
191
  }
192
+ /**
193
+ * #2029: scope instances whose door has read the rewind hold and found NO claim on the scope at
194
+ * all, so their doors skip the hold read for the rest of the instance's life. Isolate-wide, because
195
+ * a host lives for one request: without it, every peer call would read the deployment's one hold
196
+ * object — an extra hop, on a single object, on the hot path of every scope never rewound.
197
+ *
198
+ * Safe because claims are only ever added for storage a LATER instance serves. A rewind writes its
199
+ * claim before it arms, and arming restarts the scope, so its rewound storage is always served by a
200
+ * new instance with a new id, which reads the hold afresh (`SWITCH_HOLD_SETTLE_MS` puts every row of
201
+ * the claim before that instance existed). A join (`switchHoldJoin`) goes into a rewind's claim too,
202
+ * so into that same later instance. An instance that read no claim therefore never gains one; claims
203
+ * only leave it, by a switch move. Only a GOOD read is remembered: a read that failed open is not.
204
+ * Bounded: past `CLEAR_SCOPE_INSTANCES_MAX` it is emptied, which costs a re-read, nothing else.
205
+ */
206
+ const CLEAR_SCOPE_INSTANCES = new Set();
207
+ const CLEAR_SCOPE_INSTANCES_MAX = 10_000;
208
+ const rememberClearInstance = (key) => {
209
+ if (CLEAR_SCOPE_INSTANCES.size >= CLEAR_SCOPE_INSTANCES_MAX)
210
+ CLEAR_SCOPE_INSTANCES.clear();
211
+ CLEAR_SCOPE_INSTANCES.add(key);
212
+ };
213
+ /**
214
+ * #2045: a carry's fences, by tuple subject (`RecordedOffCarry.fences`): for each recorded-off
215
+ * subject, the operation id of the call the record holds.
216
+ */
217
+ const switchFencesCarried = (kinds) => {
218
+ const out = {};
219
+ for (const [kind, { keys, fences }] of kinds) {
220
+ for (const key of keys) {
221
+ const fence = fences.get(key);
222
+ if (fence !== undefined)
223
+ out[switchSubjectOf(kind, key)] = fence;
224
+ }
225
+ }
226
+ return out;
227
+ };
228
+ /** #2029: how a door's refusals and a re-assert's name each kind of subject. */
229
+ const DOOR_WORDS = {
230
+ system: { what: (key) => `module '${key}'`, door: 'system door', switchName: 'schedule', switched: 'schedules' },
231
+ peer: { what: (key) => `peer vertical '${key}'`, door: 'peer door', switchName: 'peer', switched: 'peers' },
232
+ };
233
+ /** #2029: the rewind hold's key for one subject — a module's id, bare as #1819 stored it, or `vertical:<slug>`. */
234
+ const holdKeyOf = (kind, key) => (kind === 'system' ? key : peerSubjectRef(key));
135
235
  export class CloudflareScopeHost {
136
236
  admin;
137
237
  scopeNs;
@@ -349,6 +449,22 @@ export class CloudflareScopeHost {
349
449
  },
350
450
  };
351
451
  }
452
+ /**
453
+ * #1184: the scope reads an executor's handler may make. Plain RPCs to the ScopeDO: the
454
+ * handler runs here on the coordinator, after the DO's own task has returned.
455
+ */
456
+ executorScope(tenantId, scopeId) {
457
+ const stub = this.scopeStub(scopeId);
458
+ return {
459
+ history: async (entity, page) => stub.entityHistory({ entityType: entity.entityType, entityId: entity.entityId, limit: page?.limit, cursor: page?.cursor }),
460
+ covers: async (principal, roleKey, level) => {
461
+ const bound = await stub.canAssignFor(tenantId, scopeId, principalId.parse(principal), roleKey, level === 'tenant');
462
+ if (!bound)
463
+ throw unknownRoleError(roleKey);
464
+ return coverage.parse(bound);
465
+ },
466
+ };
467
+ }
352
468
  /**
353
469
  * Drain this scope's outbox into the registered executors (K-22 §4.2).
354
470
  *
@@ -372,7 +488,9 @@ export class CloudflareScopeHost {
372
488
  * own `invocationId` field reads null — an ambient read there would record "no
373
489
  * call" for every attempt an operation's own tail made.
374
490
  */
375
- invocationId) {
491
+ invocationId,
492
+ /** #1184: what each attempt did, for the emitting call's `onExecutorOutcomes`. */
493
+ outcomes) {
376
494
  const report = {
377
495
  attempted: 0,
378
496
  delivered: 0,
@@ -386,27 +504,22 @@ export class CloudflareScopeHost {
386
504
  // attempt runs — the DO only journals it.
387
505
  const lines = asyncLinePass();
388
506
  try {
389
- await this.drainExecutorsPass(tenantId, scopeId, invocationId, report, lines);
507
+ await this.drainExecutorsPass(tenantId, scopeId, invocationId, report, lines, outcomes);
390
508
  }
391
509
  finally {
392
510
  lines.end();
393
511
  }
394
512
  return report;
395
513
  }
396
- async drainExecutorsPass(tenantId, scopeId, invocationId, report, lines) {
514
+ async drainExecutorsPass(tenantId, scopeId, invocationId, report, lines, outcomes) {
397
515
  const stub = this.scopeStub(scopeId);
516
+ const scope = this.executorScope(tenantId, scopeId);
398
517
  // #2005: a non-primary scope (a fork, a snapshot, a preview of either kind) causes no
399
518
  // outbound effects, so its deliveries are journaled terminal with the reason and no
400
- // handler runs. Asked of the directory where there is one. A CP-less host has none, so it
401
- // asks the scope's own storage whether it is classified a copy (`_substrat_copy_origin`'s
402
- // `is_copy`, #2009, set on the directory's word) — a preview and a snapshot reach a hosted
403
- // vertical as restores the platform marks.
404
- // Asked on the first due event, once per pass: most passes have none, and the directory
405
- // is one global object.
519
+ // handler runs (`isInertScope`). Asked on the first due event, once per pass: most passes
520
+ // have none, and the directory is one global object.
406
521
  let inert;
407
- const isInert = () => (inert ??= this.cpLess
408
- ? stub.isCopy()
409
- : this.cp.getScopeRecord(tenantId, scopeId).then((row) => !isPrimaryScopeRow(row)));
522
+ const isInert = () => (inert ??= this.isInertScope(tenantId, scopeId));
410
523
  // #1713: a CP-less scope its lifecycle holds attempts nothing and journals nothing, so every
411
524
  // due delivery stays due for the first pass after it is live again. Asked on the first due
412
525
  // event, like `isInert`. Every caller passed `assertLive` already, so this is for the scope
@@ -437,6 +550,9 @@ export class CloudflareScopeHost {
437
550
  attempt,
438
551
  startedAt: Date.now(),
439
552
  });
553
+ const outcomeOf = (event, outcome, error) => {
554
+ outcomes?.push(executorOutcomeOf(id, event, outcome, error));
555
+ };
440
556
  for (const bad of undecodable) {
441
557
  report.attempted += 1;
442
558
  const attempt = await stub.recordExecutorAttempt(bad.eventId, deliveryId, bad.error, null, invocationId);
@@ -450,6 +566,7 @@ export class CloudflareScopeHost {
450
566
  // No next attempt, like an undecodable row: a scope does not become primary.
451
567
  const attempt = await stub.recordExecutorAttempt(event.id, deliveryId, INERT_SCOPE_REASON, null, invocationId);
452
568
  report.inert = (report.inert ?? 0) + 1;
569
+ outcomeOf(event, 'inert', INERT_SCOPE_REASON);
453
570
  // Its own outcome, never `delivered`: no handler ran (#2005).
454
571
  lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'inert' });
455
572
  continue;
@@ -467,6 +584,7 @@ export class CloudflareScopeHost {
467
584
  // the at-least-once residue, as it already must in-process.
468
585
  await stub.routeExecutorEventToPlatform(event.id, deliveryId, connectorDispatchKind(executor.provider), JSON.stringify({ executorId: id, event }), JSON.stringify({ system: 'connector-dispatch' }), invocationId);
469
586
  report.routedToPlatform += 1;
587
+ outcomeOf(event, 'routed');
470
588
  // Handed on, not run here: the platform's drain owns its attempts from now on.
471
589
  lines.write({ ...unitOf(event.id, 1), startedAt, outcome: 'routed' });
472
590
  }
@@ -474,13 +592,26 @@ export class CloudflareScopeHost {
474
592
  await executor.handler(await this.connectorContext(tenantId, scopeId, executor.timeoutMs, event.id), event);
475
593
  const attempt = await stub.recordExecutorAttempt(event.id, deliveryId, null, null, invocationId);
476
594
  report.delivered += 1;
595
+ outcomeOf(event, 'delivered');
477
596
  lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'delivered' });
478
597
  }
479
598
  else {
480
- await executor.handler(this.admin, event);
481
- const attempt = await stub.recordExecutorAttempt(event.id, deliveryId, null, null, invocationId);
482
- report.delivered += 1;
483
- lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'delivered' });
599
+ const result = await executor.handler(this.admin, event, scope);
600
+ // #1184: a refusal is the handler's own terminal decision — journaled with its
601
+ // reason, no next attempt, listed by `executorDeadLetters` beside an exhausted one.
602
+ const refused = isDeliveryRefusal(result) ? result : null;
603
+ const text = refused && refusalJournalText(refused);
604
+ const attempt = await stub.recordExecutorAttempt(event.id, deliveryId, text, null, invocationId);
605
+ if (refused) {
606
+ report.deadLettered += 1;
607
+ outcomeOf(event, 'refused', refused.reason);
608
+ lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'dead-lettered', error: text });
609
+ }
610
+ else {
611
+ report.delivered += 1;
612
+ outcomeOf(event, 'delivered');
613
+ lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'delivered' });
614
+ }
484
615
  }
485
616
  }
486
617
  catch (err) {
@@ -495,6 +626,7 @@ export class CloudflareScopeHost {
495
626
  report.deadLettered += 1;
496
627
  else
497
628
  report.retrying += 1;
629
+ outcomeOf(event, exhausted ? 'dead-lettered' : 'retrying', err);
498
630
  lines.write({
499
631
  ...unitOf(event.id, attempts),
500
632
  startedAt,
@@ -518,13 +650,14 @@ export class CloudflareScopeHost {
518
650
  // emitting operation's own tail made.
519
651
  return this.drainExecutors(tenantId, scopeId, null);
520
652
  }
521
- registerJob(moduleId, name, handler, retry) {
653
+ registerJob(moduleId, name, handler, retry, options) {
522
654
  // #1575: the kernel's own jobs are dispatched before this registry is read.
523
655
  assertJobRegistrable(moduleId, name);
656
+ assertLeaseMs(options?.leaseMs);
524
657
  const key = `${moduleId}/${name}`;
525
658
  if (this.jobs.has(key))
526
659
  throw new Error(`job '${key}' is already registered`);
527
- this.jobs.set(key, { handler, retry });
660
+ this.jobs.set(key, { handler, retry, leaseMs: options?.leaseMs });
528
661
  }
529
662
  /**
530
663
  * D-14's DURABLE driver: the run store is RPC to the scope DO, and nothing else.
@@ -542,13 +675,15 @@ export class CloudflareScopeHost {
542
675
  const stub = this.scopeStub(scopeId);
543
676
  return {
544
677
  startOrJoin: (key, row) => stub.jobRunStartOrJoin(key.moduleId, key.job, key.instance, row),
545
- get: (id) => stub.jobRunById(id),
546
- due: (now, limit, afterId) => stub.jobRunsDue(now, limit, afterId),
678
+ dueKeys: (now, max) => stub.jobRunsDueKeys(now, max),
679
+ claim: (id, owner, leaseMs) => stub.jobRunClaim(id, owner, leaseMs),
680
+ begin: (id, owner, marginMs) => stub.jobRunBegin(id, owner, marginMs),
681
+ miss: (id, owner, note) => stub.jobRunMiss(id, owner, note),
547
682
  list: (filter) => stub.jobRunList(filter),
548
- patch: (id, patch) => stub.jobRunPatch(id, patch),
549
- commitPass: (id, patch) => stub.jobCommitPass(id, patch),
550
- step: (runId, name) => stub.jobStepRow(runId, name),
551
- recordStep: (runId, name, result, attempts, lastError, at) => stub.jobStepRecord(runId, name, result, attempts, lastError, at),
683
+ patch: (id, patch, owner) => stub.jobRunPatch(id, patch, owner),
684
+ commitPass: (id, patch, owner) => stub.jobCommitPass(id, patch, owner),
685
+ beginStep: (runId, name, owner, leaseMs) => stub.jobStepBegin(runId, name, owner, leaseMs),
686
+ recordStep: (runId, name, result, attempts, lastError, at, owner, leaseMs) => stub.jobStepRecord(runId, name, result, attempts, lastError, at, owner, leaseMs),
552
687
  };
553
688
  }
554
689
  /**
@@ -585,7 +720,10 @@ export class CloudflareScopeHost {
585
720
  store: this.jobStore(scopeId),
586
721
  handlerFor: (run) => isAttachmentTextRun(run) ? attachmentText : this.jobs.get(`${run.module_id}/${run.job}`),
587
722
  now: () => new Date().toISOString(),
588
- openScope: (run) => this.getSystemScope(run.module_id, tenantId, scopeId),
723
+ // #1834: the door is opened FOR this pass, so its "not now" is tied to this pass alone.
724
+ openScope: async (run, pass) => this.buildStub(tenantId, scopeId, undefined, undefined, await this.openSystemDoor(run.module_id, tenantId, scopeId, undefined, pass)),
725
+ // #1834: only this host's own door refusals defer a pass, recognised by identity.
726
+ deferral: (err, pass) => this.takeDoorWait(err, pass),
589
727
  maxPasses: options?.maxPasses,
590
728
  limit: options?.limit,
591
729
  });
@@ -665,9 +803,9 @@ export class CloudflareScopeHost {
665
803
  const rec = await this.cp.getScopeRecord(tenantId, scopeId);
666
804
  // K-3: a scope under another tenant is indistinguishable from one that does not exist.
667
805
  if (!rec)
668
- throw substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
806
+ throw unknownScopeForTenant(tenantId, scopeId);
669
807
  if (rec.status !== 'active' && rec.status !== 'provisioning') {
670
- throw new Error(`scope not migratable (status: ${rec.status}): ${scopeId}`);
808
+ throw substratError('conflict', `scope not migratable (status: ${rec.status}): ${scopeId}`);
671
809
  }
672
810
  const stub = this.scopeStub(scopeId);
673
811
  try {
@@ -709,11 +847,11 @@ export class CloudflareScopeHost {
709
847
  return this.scopeStub(scopeId).introspectTables();
710
848
  }
711
849
  async introspectScopeTable(scopeId, input) {
712
- return this.scopeStub(scopeId).introspectTable(input.table, input.limit, input.offset);
850
+ return unwrapReply(await this.scopeStub(scopeId).introspectTableReply(input.table, input.limit, input.offset));
713
851
  }
714
852
  /** The SQL console's CP-less path (#219) — same trust line as the pair above. */
715
853
  async introspectScopeQuery(scopeId, input) {
716
- return this.scopeStub(scopeId).introspectQuery(input.sql);
854
+ return unwrapReply(await this.scopeStub(scopeId).introspectQueryReply(input.sql));
717
855
  }
718
856
  /**
719
857
  * The K-35 denial log's CP-less path (#867) — same trust line as the reads above.
@@ -737,6 +875,31 @@ export class CloudflareScopeHost {
737
875
  async listCapabilitiesLocal(scopeId, filter) {
738
876
  return this.scopeStub(scopeId).listCapabilities(filter);
739
877
  }
878
+ /**
879
+ * The platform's `become` mint on the CP-less path (#1686) — `HostAdmin.mintCapability`'s
880
+ * ScopeDO half, for the deployment that serves the scope. Its one caller is an owner claim
881
+ * link, minted on the platform's instruction behind the vertical's platform-gated
882
+ * `/internal/owner-claim`; the control plane made the K-3 check and keeps the admin log before
883
+ * it called. Not a module verb: a module mints through `ctx.capabilities`, which mints `act`
884
+ * capabilities only. Same lifecycle gate as the exchange, so a held scope mints nothing.
885
+ */
886
+ async mintCapabilityLocal(tenantId, scopeId, input, actor) {
887
+ // Checked on this side too, as `HostAdmin.mintCapability` does: a typed refusal thrown
888
+ // across the RPC arrives as a bare message.
889
+ checkBecomeInput(input, new Date().toISOString());
890
+ await this.assertLive(tenantId, scopeId);
891
+ await this.migrateAndRecord(scopeId);
892
+ return mintedCapability.parse(await this.scopeStub(scopeId).mintBecomeCapability(input, actor));
893
+ }
894
+ /**
895
+ * The platform's revoke on the CP-less path (#1686) — how a re-minted claim link retires the
896
+ * previous one. True when the scope held the capability (revoked now, or already); false when
897
+ * it holds no such capability. No lifecycle gate: a revoke only narrows, so a held scope still
898
+ * takes one. Idempotent.
899
+ */
900
+ async revokeCapabilityLocal(scopeId, capabilityId, actor) {
901
+ return (await this.scopeStub(scopeId).revokeCapabilityAsPlatform(capabilityId, actor)) !== null;
902
+ }
740
903
  /**
741
904
  * Copy one scope's data into a fresh scope DO, entirely within THIS deployment —
742
905
  * the data half of an orchestrated snapshot (preview-and-snapshots.md §9). Like the
@@ -745,12 +908,24 @@ export class CloudflareScopeHost {
745
908
  * activation, version bind) stays on the control plane's side. Because source and
746
909
  * destination sit in the same SCOPE namespace, no scope bytes ever leave the
747
910
  * deployment — the §9 property the trust line rests on.
911
+ *
912
+ * #2016: `tenantId`, the tenant the platform snapshots for (absent from an older platform). The
913
+ * source must not be foreign to it — a copy never reads another tenant's scope — and the fork
914
+ * records it as its own receipt, not the source's.
748
915
  */
749
- async snapshotScopeLocal(sourceScopeId, destScopeId) {
916
+ async snapshotScopeLocal(sourceScopeId, destScopeId, tenantId) {
917
+ if (tenantId !== undefined && (await this.scopeStub(sourceScopeId).admission(tenantId)).verdict === 'foreign') {
918
+ throw unknownScopeForTenant(tenantId, sourceScopeId);
919
+ }
750
920
  const tables = await this.scopeStub(sourceScopeId).exportDump();
751
921
  // #2009: a snapshot is a fork by construction (the control plane's row names `forkedFrom`, so
752
922
  // the directory never calls it primary), and a load classifies nothing by itself: marked here.
753
- await this.scopeStub(destScopeId).importDump(tables, destScopeId, { sourceScopeId, exact: true, markCopy: true });
923
+ await this.scopeStub(destScopeId).importDump(tables, destScopeId, {
924
+ sourceScopeId,
925
+ exact: true,
926
+ markCopy: true,
927
+ provisionedFor: tenantId,
928
+ });
754
929
  return { tables: tables.length };
755
930
  }
756
931
  /**
@@ -767,28 +942,34 @@ export class CloudflareScopeHost {
767
942
  * #1722: `loadStamp`, the stamp a carry leaves on the copy it lands, for a fenced wipe later,
768
943
  * and `expect`, the marker the carry read here: the load is refused if the store moved since.
769
944
  * #2005: `markCopy` — the directory's classification of the scope, sent when it is not
770
- * primary; the restore then marks it a copy. Refused if it classifies a primary. */
945
+ * primary; the restore then marks it a copy. Refused if it classifies a primary.
946
+ * #2016: `tenantId`, the tenant the platform restores for: recorded as the scope's receipt in
947
+ * the load's own transaction, and a store provisioned for another tenant refuses the load. */
771
948
  opts) {
772
949
  if (opts?.markCopy)
773
950
  assertCopyLineage(opts.markCopy);
951
+ const carry = opts ? recordedOffFromWire(opts) : undefined;
774
952
  const load = {
775
- switchOff: opts?.switchedOff ? { moduleIds: opts.switchedOff, at: new Date().toISOString() } : undefined,
953
+ switchOff: carry ? { at: new Date().toISOString(), ...carry } : undefined,
776
954
  sourceScopeId: opts?.sourceScopeId,
777
955
  exact: opts?.exact,
778
956
  loadStamp: opts?.loadStamp,
779
957
  markCopy: opts?.markCopy !== undefined,
958
+ provisionedFor: opts?.tenantId,
780
959
  };
781
- // Both refusals come back as values (a throw over the RPC carries only its message) and are
960
+ // The refusals come back as values (a throw over the RPC carries only its message) and are
782
961
  // thrown here, in the vertical's own isolate, so its route answers 409 or 412, not a fault.
783
962
  const out = await this.scopeStub(scopeId).importDumpChecked(tables, scopeId, {
784
963
  ...load,
785
964
  ...(opts?.expect ? { expect: opts.expect } : {}),
786
965
  });
787
966
  if (out.refused === false) {
788
- return { tables: tables.length, ...(opts?.switchedOff ? { switchedOff: out.switchedOff } : {}) };
967
+ return { tables: tables.length, ...(carry ? { switchedOff: out.switchedOff } : {}) };
789
968
  }
790
969
  if (out.refused === 'kept')
791
970
  throw substratError('conflict', KEPT_COPY_REFUSAL);
971
+ if (out.refused === 'tenant')
972
+ throw substratError('conflict', out.message);
792
973
  // A retry of a load that already committed (its answer was lost on the way back) is
793
974
  // refused by the marker that load itself moved. The store holding THIS request's stamp
794
975
  // says so: no other load writes it. Answered as applied, without `switchedOff`, so the
@@ -855,9 +1036,16 @@ export class CloudflareScopeHost {
855
1036
  * the vertical's `/internal/lifecycle`. Kept unless the scope already holds a newer one, so a
856
1037
  * push that arrives late cannot undo a later transition. `applied` is what the platform's heal
857
1038
  * sweep records as delivered; `changed` says the gate's answer moved.
1039
+ *
1040
+ * #2016: `tenantId`, the tenant the directory delivered it for (absent from an older platform).
1041
+ * A scope provisioned for another tenant refuses the delivery (`conflict`) and stores nothing;
1042
+ * a scope provisioned before the tenant receipt existed records it here, its role rows agreeing.
858
1043
  */
859
- async setLifecycleLocal(scopeId, next) {
860
- return lifecycleDelivery.parse(await this.scopeStub(scopeId).setLifecycle(scopeLifecycle.parse(next)));
1044
+ async setLifecycleLocal(scopeId, next, tenantId) {
1045
+ const out = await this.scopeStub(scopeId).setLifecycle(scopeLifecycle.parse(next), tenantId);
1046
+ if ('refused' in out)
1047
+ throw substratError('conflict', out.message);
1048
+ return lifecycleDelivery.parse(out);
861
1049
  }
862
1050
  /**
863
1051
  * Mark one scope in THIS deployment a copy (#2005), behind the vertical's `/internal/mark-copy`:
@@ -899,7 +1087,7 @@ export class CloudflareScopeHost {
899
1087
  * safe; scope-level tuples (the restored grants) are never touched.
900
1088
  */
901
1089
  async projectRolesLocal(tenantId, scopeId, roles) {
902
- await this.scopeStub(scopeId).applyProjection(tenantId, roles.map((r) => ({ role_key: r.key, permissions: JSON.stringify(r.permissions), source: r.source })), []);
1090
+ unwrapReply(await this.scopeStub(scopeId).applyProjectionReply(tenantId, roles.map((r) => ({ role_key: r.key, permissions: JSON.stringify(r.permissions), source: r.source })), []));
903
1091
  }
904
1092
  /**
905
1093
  * Wipe one scope DO's storage in THIS deployment — the reap half of an orchestrated
@@ -951,6 +1139,10 @@ export class CloudflareScopeHost {
951
1139
  async lifecycleFlowLocal(scopeId, input) {
952
1140
  return this.scopeStub(scopeId).lifecycleFlow(input);
953
1141
  }
1142
+ /** Business volumes per bucket (#1750) on this host's own scope — the vertical-host read. */
1143
+ async operationSeriesLocal(scopeId, input) {
1144
+ return this.scopeStub(scopeId).operationSeries(input);
1145
+ }
954
1146
  /** One event's causal chain (#1237) on this host's own scope — the vertical-host read. */
955
1147
  async eventCauseLocal(scopeId, input) {
956
1148
  return this.scopeStub(scopeId).eventCause(input);
@@ -1103,7 +1295,7 @@ export class CloudflareScopeHost {
1103
1295
  const record = resolveScopeRecord(input);
1104
1296
  // Fail-closed tenant gate throws out of the awaited cp call BEFORE migrate
1105
1297
  // or audit, so a rejected provision creates nothing and writes no audit row.
1106
- const created = await this.cp.provisionScope(input.tenantId, input.scopeId, record, new Date().toISOString());
1298
+ const created = await this.directory('provisionScope', input.tenantId, input.scopeId, record, new Date().toISOString());
1107
1299
  // Instantiate the scope DO and trigger its lazy migration.
1108
1300
  await this.migrateAndRecord(input.scopeId);
1109
1301
  // Scope-local permissions: a freshly-provisioned scope evaluates from its own
@@ -1144,10 +1336,9 @@ export class CloudflareScopeHost {
1144
1336
  // #1742: in the SAME unit as the seat, so no sweep can land between them; the re-assert
1145
1337
  // after it then finds them off, and audits what the unit moved.
1146
1338
  const ownStore = this.cpLess || record.vertical === null;
1147
- const recordedOff = ownStore ? await this.cp.switchedOffModulesOf(input.tenantId, input.scopeId) : [];
1148
- const switchedOff = await this.scopeStub(input.scopeId).seatTuples(seats, recordedOff.length
1149
- ? { scopeId: input.scopeId, moduleIds: recordedOff, at: new Date().toISOString() }
1150
- : undefined);
1339
+ const seatAt = new Date().toISOString();
1340
+ const carry = ownStore ? await this.recordedOffCarry(input.tenantId, input.scopeId, seatAt) : undefined;
1341
+ const switchedOff = await this.scopeStub(input.scopeId).seatTuples(seats, carry ? { scopeId: input.scopeId, at: seatAt, ...carry } : undefined);
1151
1342
  if (ownStore) {
1152
1343
  await this.admin.reassertSystemSwitches(actor, { tenantId: input.tenantId, scopeId: input.scopeId }, { appliedInUnit: switchedOff });
1153
1344
  }
@@ -1169,10 +1360,10 @@ export class CloudflareScopeHost {
1169
1360
  // inside the DO's ledger write, which is the serialization point that actually holds.
1170
1361
  const tenant = await this.cp.getTenant(input.tenantId);
1171
1362
  if (!tenant) {
1172
- throw new Error(`cannot provision tenant store under unknown tenant: ${input.tenantId}`);
1363
+ throw substratError('not_found', `cannot provision tenant store under unknown tenant: ${input.tenantId}`);
1173
1364
  }
1174
1365
  if (tenant.status !== 'active') {
1175
- throw new Error(`cannot provision tenant store under non-active tenant (status: ${tenant.status}): ${input.tenantId}`);
1366
+ throw substratError('conflict', `cannot provision tenant store under non-active tenant (status: ${tenant.status}): ${input.tenantId}`);
1176
1367
  }
1177
1368
  // Idempotent on (tenant, vertical, binding): a retried provision re-resolves the SAME
1178
1369
  // store rather than minting a second database (the K-31 ready-gate retries the whole
@@ -1187,7 +1378,7 @@ export class CloudflareScopeHost {
1187
1378
  // the source of truth, carrying the D1 database_id Cloudflare assigned as the ref.
1188
1379
  const name = await tenantStoreDatabaseName(input.tenantId, input.vertical, input.binding);
1189
1380
  const ref = await d1.create(name);
1190
- const stored = await this.cp.putTenantStore({
1381
+ const stored = await this.directory('putTenantStore', {
1191
1382
  tenantId: input.tenantId,
1192
1383
  vertical: input.vertical,
1193
1384
  binding: input.binding,
@@ -1238,17 +1429,17 @@ export class CloudflareScopeHost {
1238
1429
  const r2 = this.requireBlobStores(`provisionBlobStore(tenant=${input.tenantId} vertical=${input.vertical} binding=${input.binding})`);
1239
1430
  const tenant = await this.cp.getTenant(input.tenantId);
1240
1431
  if (!tenant) {
1241
- throw new Error(`cannot provision blob store under unknown tenant: ${input.tenantId}`);
1432
+ throw substratError('not_found', `cannot provision blob store under unknown tenant: ${input.tenantId}`);
1242
1433
  }
1243
1434
  if (tenant.status !== 'active') {
1244
- throw new Error(`cannot provision blob store under non-active tenant (status: ${tenant.status}): ${input.tenantId}`);
1435
+ throw substratError('conflict', `cannot provision blob store under non-active tenant (status: ${tenant.status}): ${input.tenantId}`);
1245
1436
  }
1246
1437
  const existing = await this.cp.getBlobStore(input.tenantId, input.vertical, input.binding);
1247
1438
  if (existing)
1248
1439
  return { binding: input.binding, kind: 'blob', ref: existing.ref };
1249
1440
  const name = await blobStoreBucketName(input.tenantId, input.vertical, input.binding);
1250
1441
  const ref = await r2.create(name);
1251
- const stored = await this.cp.putBlobStore({
1442
+ const stored = await this.directory('putBlobStore', {
1252
1443
  tenantId: input.tenantId,
1253
1444
  vertical: input.vertical,
1254
1445
  binding: input.binding,
@@ -1292,15 +1483,15 @@ export class CloudflareScopeHost {
1292
1483
  return this.buildAttachmentSurface({ principal }, tenantId, scopeId, store);
1293
1484
  }
1294
1485
  async getSystemAttachments(moduleId, tenantId, scopeId) {
1295
- // Reuse the system invoke door's module, tenant and lifecycle gate. The DO
1296
- // authorizes each open against the attachment target's read permission as
1486
+ // Reuse the system invoke door's module, tenant and lifecycle gate, and its rewind gate
1487
+ // (#1834). The DO authorizes each open against the attachment target's read permission as
1297
1488
  // system:<moduleId>; the worker reads bytes only after that gate succeeds.
1298
- await this.getSystemScope(moduleId, tenantId, scopeId);
1489
+ const door = await this.openSystemDoor(moduleId, tenantId, scopeId);
1299
1490
  const store = await this.resolveAttachmentStore(tenantId);
1300
1491
  const stub = this.scopeStub(scopeId);
1301
1492
  return {
1302
1493
  open: async (attachmentId) => {
1303
- const record = await stub.systemAttachmentAuthorize(attachmentId, moduleId, tenantId, scopeId);
1494
+ const record = await door.through((instance) => stub.systemAttachmentAuthorize(attachmentId, moduleId, tenantId, scopeId, instance));
1304
1495
  return record ? this.openAttachmentBytes(store, scopeId, record) : null;
1305
1496
  },
1306
1497
  };
@@ -1387,22 +1578,22 @@ export class CloudflareScopeHost {
1387
1578
  // the DO only to be refused and recorded — never to the blob store.
1388
1579
  const hash = subject.capabilitySession;
1389
1580
  return {
1390
- upload: async (input) => unwrapCapabilityReply(await stub.capabilityAttachmentRefuseWrite(hash, tenantId, scopeId, 'attachments.upload', {
1581
+ upload: async (input) => unwrapReply(await stub.capabilityAttachmentRefuseWrite(hash, tenantId, scopeId, 'attachments.upload', {
1391
1582
  entityType: input.entity.entityType,
1392
1583
  })),
1393
- list: async (entity) => unwrapCapabilityReply(await stub.capabilityAttachmentList(entity, hash, tenantId, scopeId)),
1584
+ list: async (entity) => unwrapReply(await stub.capabilityAttachmentList(entity, hash, tenantId, scopeId)),
1394
1585
  open: async (attachmentId) => {
1395
- const record = unwrapCapabilityReply(await stub.capabilityAttachmentOpen(attachmentId, hash, tenantId, scopeId));
1586
+ const record = unwrapReply(await stub.capabilityAttachmentOpen(attachmentId, hash, tenantId, scopeId));
1396
1587
  return record ? this.openAttachmentBytes(store, scopeId, record) : null;
1397
1588
  },
1398
- remove: async (attachmentId) => unwrapCapabilityReply(await stub.capabilityAttachmentRefuseWrite(hash, tenantId, scopeId, 'attachments.remove', {
1589
+ remove: async (attachmentId) => unwrapReply(await stub.capabilityAttachmentRefuseWrite(hash, tenantId, scopeId, 'attachments.remove', {
1399
1590
  attachmentId,
1400
1591
  })),
1401
1592
  search: async (term, options) => {
1402
1593
  // Judged here too, so a short term reaches the caller as `SearchTermTooShort`
1403
1594
  // rather than as a message that crossed the RPC boundary. The DO judges again.
1404
1595
  searchMatchExpression(term, 'prefix');
1405
- return unwrapCapabilityReply(await stub.capabilityAttachmentSearch(term, searchLimit(options?.limit), hash, tenantId, scopeId));
1596
+ return unwrapReply(await stub.capabilityAttachmentSearch(term, searchLimit(options?.limit), hash, tenantId, scopeId));
1406
1597
  },
1407
1598
  };
1408
1599
  }
@@ -1457,7 +1648,7 @@ export class CloudflareScopeHost {
1457
1648
  throw new Error('attachments.search is not available on a delivery-scoped surface');
1458
1649
  }
1459
1650
  searchMatchExpression(term, 'prefix');
1460
- return unwrapCapabilityReply(await stub.attachmentSearch(term, searchLimit(options?.limit), asPrincipalId, tenantId, scopeId, connectionId));
1651
+ return unwrapReply(await stub.attachmentSearch(term, searchLimit(options?.limit), asPrincipalId, tenantId, scopeId, connectionId));
1461
1652
  },
1462
1653
  };
1463
1654
  }
@@ -1505,16 +1696,51 @@ export class CloudflareScopeHost {
1505
1696
  // this is the scope's real store, as for the provision's seat above; a hosted scope is
1506
1697
  // restored in its deployment, which the platform carries the same list to.
1507
1698
  const ownStore = this.cpLess || existing.vertical === null;
1508
- const recordedOff = ownStore ? await this.cp.switchedOffModulesOf(tenantId, scopeId) : [];
1699
+ const replayAt = new Date().toISOString();
1700
+ const carry = ownStore ? await this.recordedOffCarry(tenantId, scopeId, replayAt) : undefined;
1509
1701
  const switchedOff = await this.scopeStub(scopeId).importDump(dump.tables, scopeId, {
1510
- switchOff: recordedOff.length ? { moduleIds: recordedOff, at: new Date().toISOString() } : undefined,
1702
+ switchOff: carry ? { at: replayAt, ...carry } : undefined,
1511
1703
  sourceScopeId: opts?.sourceScopeId ?? dump.scopeId,
1512
1704
  });
1513
1705
  await this.recordAdmin(actor, 'restoreScope', { tenantId, scopeId }, null, { sourceScopeId: dump.scopeId, tables: dump.tables.length, capturedAt: dump.capturedAt });
1514
- if (recordedOff.length) {
1706
+ if (carry) {
1515
1707
  await this.admin.reassertSystemSwitches(actor, { tenantId, scopeId }, { appliedInUnit: switchedOff });
1516
1708
  }
1517
1709
  }
1710
+ /**
1711
+ * #1742, #2029: what this host's own unit carries of the directory's record — the modules and
1712
+ * the peers recorded OFF on the scope, each with the ones held there only by a tenant-level
1713
+ * grant (#1823, #2030). Undefined when nothing is recorded off, so such a unit is unchanged.
1714
+ */
1715
+ async recordedOffCarry(tenantId, scopeId, at) {
1716
+ const [modules, peers] = await Promise.all([
1717
+ this.recordedOff('system', tenantId, scopeId, at),
1718
+ this.recordedOff('peer', tenantId, scopeId, at),
1719
+ ]);
1720
+ if (!modules.keys.length && !peers.keys.length)
1721
+ return undefined;
1722
+ return {
1723
+ moduleIds: modules.keys,
1724
+ tenantHeld: modules.tenantHeld,
1725
+ verticals: peers.keys,
1726
+ tenantHeldVerticals: peers.tenantHeld,
1727
+ fences: switchFencesCarried([
1728
+ ['system', modules],
1729
+ ['peer', peers],
1730
+ ]),
1731
+ };
1732
+ }
1733
+ /** The subjects of one kind the directory records OFF on the scope, and of those the tenant-held ones. */
1734
+ async recordedOff(kind, tenantId, scopeId, at) {
1735
+ // #2045: the OFF rows and their fences in one read, so every key carried has its own row's fence.
1736
+ const fences = new Map((await this.cp.switchRecordStatesOf(kind, tenantId, scopeId))
1737
+ .filter(([, row]) => row.position === 'off')
1738
+ .map(([key, row]) => [key, row.fence]));
1739
+ const keys = [...fences.keys()];
1740
+ if (!keys.length)
1741
+ return { keys, tenantHeld: [], fences };
1742
+ return { keys, tenantHeld: await this.cp.tenantHeldOf(kind, tenantId, keys, at), fences };
1743
+ }
1518
1744
  async snapshotScope(actor, tenantId, scopeId, opts) {
1519
1745
  const source = await this.admin.getScopeRecord(actor, tenantId, scopeId);
1520
1746
  if (!source)
@@ -1545,7 +1771,7 @@ export class CloudflareScopeHost {
1545
1771
  if (!rec)
1546
1772
  throw substratError('not_found', `unknown scope ${scopeId} in tenant ${tenantId}`);
1547
1773
  if (!rec.forkedFrom && rec.kind !== 'preview') {
1548
- throw new Error(`scope ${scopeId} is not a fork or preview — only previews may be deleted; ` +
1774
+ throw substratError('forbidden', `scope ${scopeId} is not a fork or preview — only previews may be deleted; ` +
1549
1775
  `archive a primary scope instead`);
1550
1776
  }
1551
1777
  // Storage BEFORE the directory row: a crash between the two leaves a visible row
@@ -1685,12 +1911,134 @@ export class CloudflareScopeHost {
1685
1911
  * against `system:<moduleId>` grants inside the stub.
1686
1912
  */
1687
1913
  async getSystemScope(moduleId, tenantId, scopeId) {
1914
+ return this.buildStub(tenantId, scopeId, undefined, undefined, await this.openSystemDoor(moduleId, tenantId, scopeId));
1915
+ }
1916
+ /**
1917
+ * #1834: the system door — the ONE place every entry acting as `system:<moduleId>` passes
1918
+ * through: a schedule's fire, a job run's `pass.scope()`, a module's attachment open. The
1919
+ * module must be registered here and the scope live, as before. Then the door GATES: the
1920
+ * scope's own state read for the module, and after it the rewind hold (#1819), in the order
1921
+ * `SWITCH_HOLD_SETTLE_MS` relies on. A module the hold keeps off is refused here, as a
1922
+ * schedule pass skips it.
1923
+ *
1924
+ * A gate read is about the instance that answered it, and a door can be held for as long as
1925
+ * a job pass runs. So every call through the door is PINNED to that instance: the DO refuses
1926
+ * one that lands on another (`SYSTEM_DOOR_MOVED`), since a PITR restore always restarts the
1927
+ * object, and the door gates again and retries. A restart within one call's round trip is
1928
+ * rare, so past `SYSTEM_DOOR_REGATES` it fails closed (`unavailable`). No clock is involved:
1929
+ * the gate and the call meet the same storage, or the call does not run.
1930
+ */
1931
+ async openSystemDoor(moduleId, tenantId, scopeId, gated,
1932
+ /** #1834: the job pass this door is opened for; its "not now" refusals are tied to it. */
1933
+ pass) {
1688
1934
  if (!this.moduleIds.has(moduleId)) {
1689
- throw new Error(`module not registered on this host: ${moduleId}`);
1935
+ throw substratError('not_found', `module not registered on this host: ${moduleId}`);
1690
1936
  }
1691
1937
  await this.assertLive(tenantId, scopeId);
1692
1938
  await this.migrateAndRecord(scopeId);
1693
- return this.buildStub(tenantId, scopeId, undefined, undefined, moduleId);
1939
+ return this.openDoor('system', moduleId, scopeId, gated, pass);
1940
+ }
1941
+ /**
1942
+ * #1834: one door, for either kind of switched subject (#2029). The gate is read here unless the
1943
+ * caller read it (`runDueSchedules` reports a hold it could not read on its pass report); a held
1944
+ * subject is refused, and a call landing on another instance than the gate's re-gates, at most
1945
+ * `SYSTEM_DOOR_REGATES` times, then fails closed.
1946
+ *
1947
+ * The peer door (`kind: 'peer'`) is the system door's for a peer vertical. A PITR rewind to before
1948
+ * a peer was switched off brings its grants back live and its OFF marker gone, so the rewind holds
1949
+ * it outside the scope (`rewindHolding`, as for a module), and every peer entry — an invoke, a
1950
+ * delivery, a coverage read, an export read for it as consumer — opens this door after
1951
+ * `peerScopeGate`.
1952
+ */
1953
+ async openDoor(kind, key, scopeId, gated, pass) {
1954
+ const words = DOOR_WORDS[kind];
1955
+ const what = words.what(key);
1956
+ // A gate this door reads itself reports a hold it could not read; one handed in was the
1957
+ // caller's to report.
1958
+ const regate = async () => {
1959
+ const fresh = await this.doorGate(kind, scopeId, key);
1960
+ if (fresh.error)
1961
+ console.error(`substrat: door of ${what} on ${scopeId}: ${fresh.error}`);
1962
+ return fresh;
1963
+ };
1964
+ let gate = gated ?? (await regate());
1965
+ return {
1966
+ kind,
1967
+ key,
1968
+ through: async (call) => {
1969
+ for (let regates = 0;; regates += 1) {
1970
+ if (gate.held) {
1971
+ // The hold ends when the switch is applied again: a job run waits it out without
1972
+ // spending its retries, because the driver defers on THIS refusal (`doorWait`).
1973
+ const held = substratError('forbidden', `${what} is held off on this scope: it was rewound to before its ` +
1974
+ `${words.switchName} switch was turned off, and it stays off until ` +
1975
+ 'the switch is applied again', { reason: SYSTEM_DOOR_WAIT });
1976
+ this.heldRefusals.add(held);
1977
+ throw this.doorWait(pass, held);
1978
+ }
1979
+ const answer = await call(gate.instance);
1980
+ if (!isSystemDoorMoved(answer))
1981
+ return answer;
1982
+ if (regates >= SYSTEM_DOOR_REGATES) {
1983
+ throw this.doorWait(pass, substratError('unavailable', `the scope kept restarting under the ${words.door} of ${what} ` +
1984
+ `(${regates + 1} attempts); nothing ran. Retry later`, { reason: SYSTEM_DOOR_WAIT }));
1985
+ }
1986
+ gate = await regate();
1987
+ }
1988
+ },
1989
+ };
1990
+ }
1991
+ /**
1992
+ * #1834: the refusals this host's system door threw that mean "not now" (a hold, a scope that
1993
+ * kept restarting), each tied by identity to the job pass whose door threw it. The job driver
1994
+ * defers a pass only on one of these, never on an error's shape, which any step or operation
1995
+ * could copy. A mark is ORIGIN and FRESHNESS together: it is consumed when the driver takes it,
1996
+ * and only the pass it names can take it, so a handler that keeps a refusal and throws it again
1997
+ * on a later pass is an ordinary failure there. Held weakly: an error nobody holds goes.
1998
+ */
1999
+ doorWaits = new WeakMap();
2000
+ /**
2001
+ * #2029: the hold refusals this host's doors threw, by identity — so a peer delivery pauses its
2002
+ * edge, and a coverage read answers "holds nothing", on exactly those and on no error that only
2003
+ * looks like one. Held weakly, as `doorWaits` is.
2004
+ */
2005
+ heldRefusals = new WeakSet();
2006
+ /** #2029: is `err` a hold refusal this host's door threw? */
2007
+ isHeldRefusal(err) {
2008
+ return typeof err === 'object' && err !== null && this.heldRefusals.has(err);
2009
+ }
2010
+ /** #1834: mark this door's own refusal as "not now" for the pass it was opened for, and hand it back. */
2011
+ doorWait(pass, err) {
2012
+ if (pass)
2013
+ this.doorWaits.set(err, pass);
2014
+ return err;
2015
+ }
2016
+ /** #1834: is `err` a "not now" this host's door threw on `pass`? Consumes the mark when it is. */
2017
+ takeDoorWait(err, pass) {
2018
+ if (typeof err !== 'object' || err === null || this.doorWaits.get(err) !== pass)
2019
+ return false;
2020
+ this.doorWaits.delete(err);
2021
+ return true;
2022
+ }
2023
+ /**
2024
+ * #1834: a door's gate — the scope's state read for the subject, the instance that answered it,
2025
+ * and then the rewind hold (#1819), for any subject the scope does not already have off. The
2026
+ * hold is read AFTER the state, which is the order `switchHeld` needs.
2027
+ *
2028
+ * `ungranted` reads the hold too (#2029): a subject held on the scope only by a TENANT-level
2029
+ * grant (#1823, #2030) comes back from a rewind with no scope grant and no marker, and the
2030
+ * tenant grant still authorizes it there. A rewind captured it off, so the hold must keep it so.
2031
+ */
2032
+ async doorGate(kind, scopeId, key) {
2033
+ const stub = this.scopeStub(scopeId);
2034
+ const { state, instance } = await (kind === 'system' ? stub.systemDoorState(key) : stub.peerDoorState(key));
2035
+ const cleared = `${scopeId} ${instance}`;
2036
+ if (state === 'off' || CLEAR_SCOPE_INSTANCES.has(cleared))
2037
+ return { state, instance, held: false };
2038
+ const hold = await this.switchHeld(scopeId, holdKeyOf(kind, key));
2039
+ if (hold.scopeClear)
2040
+ rememberClearInstance(cleared);
2041
+ return { state, instance, held: hold.held, ...(hold.error ? { error: hold.error } : {}) };
1694
2042
  }
1695
2043
  /**
1696
2044
  * #1705: the consumer half. The same lifecycle gate as every door (K-3), then the ScopeDO
@@ -1703,8 +2051,25 @@ export class CloudflareScopeHost {
1703
2051
  // The peer door's own gate (#1706): K-3's pair check, the refusal for a scope served
1704
2052
  // elsewhere, the lifecycle check and the migration. A delivery is a write into the
1705
2053
  // consumer's scope, so it takes the gate an invoke through that door takes.
1706
- await this.peerScopeGate(tenantId, scopeId, 'deliverToPeer');
1707
- const result = importResult.parse(await this.scopeStub(scopeId).importApply(batch, tenantId, scopeId));
2054
+ const record = await this.peerScopeGate(tenantId, scopeId, 'deliverToPeer');
2055
+ // #2004: a copy consumes no other vertical's events, at the door as in the sweep
2056
+ // (`scope-copy.ts`). A pause: nothing runs and the watermark stays.
2057
+ if (await this.isInertScope(tenantId, scopeId, record)) {
2058
+ return { ...emptyImportResult(batch), paused: { reason: INERT_SCOPE_REASON } };
2059
+ }
2060
+ // #2029: the producer is a peer, so the delivery goes through its door: a producer the rewind
2061
+ // hold keeps off pauses the edge, exactly as a switched-off one does inside the DO.
2062
+ const door = await this.openDoor('peer', batch.source.vertical, scopeId);
2063
+ let applied;
2064
+ try {
2065
+ applied = await door.through((instance) => this.scopeStub(scopeId).importApply(batch, tenantId, scopeId, instance));
2066
+ }
2067
+ catch (err) {
2068
+ if (!this.isHeldRefusal(err))
2069
+ throw err;
2070
+ return { ...emptyImportResult(batch), paused: { reason: err.message } };
2071
+ }
2072
+ const result = importResult.parse(applied);
1708
2073
  if (result.delivered > 0) {
1709
2074
  try {
1710
2075
  await this.drainExecutors(tenantId, scopeId, null);
@@ -1748,7 +2113,9 @@ export class CloudflareScopeHost {
1748
2113
  async getVerticalScope(caller, tenantId, scopeId, options) {
1749
2114
  const parsed = verticalCaller.parse(caller);
1750
2115
  await this.peerScopeGate(tenantId, scopeId, 'getVerticalScope');
1751
- return this.buildStub(tenantId, scopeId, undefined, undefined, undefined, options, undefined, undefined, parsed);
2116
+ // #2029: gated against the rewind hold and pinned, as the system door is.
2117
+ const door = await this.openDoor('peer', parsed.vertical, scopeId);
2118
+ return this.buildStub(tenantId, scopeId, undefined, undefined, door, options, undefined, undefined, parsed);
1752
2119
  }
1753
2120
  /**
1754
2121
  * Does peer `vertical` hold each key at this scope's node now (#1706) — the ScopeDO's own
@@ -1756,9 +2123,19 @@ export class CloudflareScopeHost {
1756
2123
  */
1757
2124
  async peerCovers(tenantId, scopeId, vertical, permissions) {
1758
2125
  await this.peerScopeGate(tenantId, scopeId, 'peerCovers');
1759
- return peerCoverage
1760
- .array()
1761
- .parse(await this.scopeStub(scopeId).peerCovers(tenantId, scopeId, verticalSlug.parse(vertical), [...permissions]));
2126
+ const slug = verticalSlug.parse(vertical);
2127
+ // #2029: a peer the rewind hold keeps off holds nothing here, as a switched-off one doesn't.
2128
+ const door = await this.openDoor('peer', slug, scopeId);
2129
+ try {
2130
+ return peerCoverage
2131
+ .array()
2132
+ .parse(await door.through((instance) => this.scopeStub(scopeId).peerCovers(tenantId, scopeId, slug, [...permissions], instance)));
2133
+ }
2134
+ catch (err) {
2135
+ if (this.isHeldRefusal(err))
2136
+ return permissions.map((permission) => ({ permission, held: false }));
2137
+ throw err;
2138
+ }
1762
2139
  }
1763
2140
  /**
1764
2141
  * `ctx.canAssign`'s bound for a principal the host names (#1931) — the ScopeDO's own role read
@@ -1772,7 +2149,7 @@ export class CloudflareScopeHost {
1772
2149
  // provisioning receipt is K-3 here (#1738), read before anything migrates the DO, and the
1773
2150
  // refusal reads as the pure host's does: an unknown scope, not an unknown role.
1774
2151
  if (this.cpLess && !(await this.scopeStub(scopeId).servesTenant(tenantId))) {
1775
- throw substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
2152
+ throw unknownScopeForTenant(tenantId, scopeId);
1776
2153
  }
1777
2154
  await this.peerScopeGate(tenantId, scopeId, 'canAssign');
1778
2155
  const bound = await this.scopeStub(scopeId).canAssignFor(tenantId, scopeId, principalId.parse(principal), roleKey);
@@ -1782,7 +2159,7 @@ export class CloudflareScopeHost {
1782
2159
  }
1783
2160
  async assignScopeRoleBounded(tenantId, scopeId, caller, assignee, roleKey) {
1784
2161
  if (this.cpLess && !(await this.scopeStub(scopeId).servesTenant(tenantId))) {
1785
- throw substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
2162
+ throw unknownScopeForTenant(tenantId, scopeId);
1786
2163
  }
1787
2164
  await this.peerScopeGate(tenantId, scopeId, 'assignScopeRoleBounded');
1788
2165
  const bound = await this.scopeStub(scopeId).assignScopeRoleBoundedFor(tenantId, scopeId, principalId.parse(caller), principalId.parse(assignee), roleKey);
@@ -1804,15 +2181,33 @@ export class CloudflareScopeHost {
1804
2181
  // door's own tenant-confinement test proved. A RECORD crosses intact, so the coordinator
1805
2182
  // reads the record and types the refusal itself. `validateScopeAccess` still runs below
1806
2183
  // and still owns the lifecycle half (a suspended tenant or scope).
2184
+ let record;
1807
2185
  if (!this.cpLess) {
1808
- const record = await this.cp.getScopeRecord(tenantId, scopeId);
2186
+ record = await this.cp.getScopeRecord(tenantId, scopeId);
1809
2187
  if (!record) {
1810
- throw substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
2188
+ throw unknownScopeForTenant(tenantId, scopeId);
1811
2189
  }
1812
2190
  this.assertServedHere(record, scopeId, verb);
1813
2191
  }
1814
2192
  await this.assertLive(tenantId, scopeId);
1815
2193
  await this.migrateAndRecord(scopeId);
2194
+ return record;
2195
+ }
2196
+ /**
2197
+ * Whether this scope is held inert (#2005): a fork, a snapshot or a preview, which causes no
2198
+ * outbound effects and consumes no other vertical's events (#2004). The one primacy question
2199
+ * every such door asks. Where there is a directory its record answers (`record`, the one
2200
+ * `peerScopeGate` returns, when the caller has already read it). A CP-less host has none, so it asks the scope's own storage
2201
+ * whether it is classified a copy (`_substrat_copy_origin`'s `is_copy`, #2009, set on the
2202
+ * directory's word): a preview and a snapshot reach a hosted vertical as restores the
2203
+ * platform marks.
2204
+ */
2205
+ isInertScope(tenantId, scopeId, record) {
2206
+ if (this.cpLess)
2207
+ return this.scopeStub(scopeId).isCopy();
2208
+ if (record)
2209
+ return Promise.resolve(!isPrimaryScopeRow(record));
2210
+ return this.cp.getScopeRecord(tenantId, scopeId).then((row) => !isPrimaryScopeRow(row));
1816
2211
  }
1817
2212
  /**
1818
2213
  * The capability's attachment door (#1686) — `getCapabilityScope`'s gate, then the
@@ -1875,7 +2270,7 @@ export class CloudflareScopeHost {
1875
2270
  async capabilityScopeStub(tenantId, scopeId, verb) {
1876
2271
  const rec = await this.cp.getScopeRecord(tenantId, scopeId);
1877
2272
  if (!rec)
1878
- throw substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
2273
+ throw unknownScopeForTenant(tenantId, scopeId);
1879
2274
  if (this.servesScopesElsewhereNow && rec.vertical !== null) {
1880
2275
  throw substratError('unavailable', `${verb} cannot reach scope ${scopeId}: it is served by the '${rec.vertical}' ` +
1881
2276
  'deployment, and platform capability verbs are not delegated there yet');
@@ -1921,7 +2316,9 @@ export class CloudflareScopeHost {
1921
2316
  if (!rec || rec.status !== 'active')
1922
2317
  return report;
1923
2318
  }
1924
- else if (await this.lifecycleHeld(scopeId)) {
2319
+ else if (lifecycleRefusal(await this.validateScopeAccess(tenantId, scopeId)) !== null) {
2320
+ // #2016: the pair is held to the scope's own record first, so a sweep roster entry whose
2321
+ // tenant disagrees with the scope is refused before the probe reads or writes anything.
1925
2322
  // #1713: a held scope emits nothing, so judging it stale would only raise a false alarm.
1926
2323
  return report;
1927
2324
  }
@@ -1963,13 +2360,15 @@ export class CloudflareScopeHost {
1963
2360
  return report;
1964
2361
  // Only run on a live scope of this tenant; a scope archived between the sweep's
1965
2362
  // enumeration and here simply has nothing due. A CP-less host has no directory
1966
- // to ask (#461), so it reads the lifecycle the platform delivered to the scope (#1713).
2363
+ // to ask (#461), so it reads the lifecycle the platform delivered to the scope (#1713), in
2364
+ // the same call that holds the pair to the scope's own record (#2016): a roster entry whose
2365
+ // tenant disagrees is refused before a grant, a cadence row or a run is read or written.
1967
2366
  if (!this.cpLess) {
1968
2367
  const rec = await this.cp.getScopeRecord(tenantId, scopeId);
1969
2368
  if (!rec || rec.status !== 'active')
1970
2369
  return report;
1971
2370
  }
1972
- else if (await this.lifecycleHeld(scopeId)) {
2371
+ else if (lifecycleRefusal(await this.validateScopeAccess(tenantId, scopeId)) !== null) {
1973
2372
  // #1713: the lifecycle the platform delivered holds the scope. Every schedule is
1974
2373
  // `skipped` and no cadence row moves, as under the kill switch, so a schedule that
1975
2374
  // came due meanwhile fires once on the first pass after the scope is live again.
@@ -1986,15 +2385,15 @@ export class CloudflareScopeHost {
1986
2385
  // never held the module's grant (a foreign vertical's) is a quiet no-op, exactly as
1987
2386
  // before. A scope switched OFF reports every schedule `skipped`, never `failed`, and
1988
2387
  // does not touch its cadence rows — so a restore fires a due schedule on the next pass.
1989
- const state = await stub.systemScheduleState(moduleId);
1990
- if (state === 'ungranted')
2388
+ // #1819: a scope rewound to before its switch was pulled answers `on`, and the rewind held
2389
+ // the module outside the scope. Both are read by the system door's own gate (#1834), which
2390
+ // every fire below then goes through, pinned to the instance this read came from.
2391
+ const gate = await this.doorGate('system', scopeId, moduleId);
2392
+ if (gate.state === 'ungranted')
1991
2393
  return report;
1992
- // #1819: a scope rewound to before its switch was pulled answers `on`. The rewind held the
1993
- // module outside the scope; read AFTER the state, which is the order `switchHeld` needs.
1994
- const hold = state === 'on' ? await this.switchHeld(scopeId, moduleId) : { held: false };
1995
- if (hold.error)
1996
- report.errors.push({ operation: 'switch-hold', error: hold.error });
1997
- if (state === 'off' || hold.held) {
2394
+ if (gate.error)
2395
+ report.errors.push({ operation: 'switch-hold', error: gate.error });
2396
+ if (gate.state === 'off' || gate.held) {
1998
2397
  for (const schedule of schedules) {
1999
2398
  report.skipped += 1;
2000
2399
  report.runs.push({ operation: schedule.operation, outcome: 'skipped' });
@@ -2004,6 +2403,7 @@ export class CloudflareScopeHost {
2004
2403
  }
2005
2404
  const now = Date.now();
2006
2405
  const lines = asyncLinePass();
2406
+ let door;
2007
2407
  for (const schedule of schedules) {
2008
2408
  const last = await stub.scheduleLastRun(schedule.operation);
2009
2409
  const lastRun = last ? Date.parse(last) : null;
@@ -2023,7 +2423,10 @@ export class CloudflareScopeHost {
2023
2423
  let emitted;
2024
2424
  let failure;
2025
2425
  try {
2026
- const scope = await this.getSystemScope(moduleId, tenantId, scopeId);
2426
+ // The gate above already answered for this pass; a fire that meets a restarted scope
2427
+ // is gated again by the door (#1834).
2428
+ door ??= await this.openSystemDoor(moduleId, tenantId, scopeId, gate);
2429
+ const scope = this.buildStub(tenantId, scopeId, undefined, undefined, door);
2027
2430
  await scope.invoke(schedule.operation, schedule.input, { invocationId, onEmitted: (r) => (emitted = r) });
2028
2431
  report.fired += 1;
2029
2432
  }
@@ -2058,7 +2461,13 @@ export class CloudflareScopeHost {
2058
2461
  return report;
2059
2462
  }
2060
2463
  /** The stub body, shared by the principal, connection, system and impersonation doors. */
2061
- buildStub(tenantId, scopeId, principal, connectionId, systemModuleId, options,
2464
+ buildStub(tenantId, scopeId, principal, connectionId,
2465
+ /**
2466
+ * #1834: the door this stub acts through. The ONLY way to make a stub acting as
2467
+ * `system:<moduleId>`: the module is the door's, and every invoke is gated and pinned by it.
2468
+ * #2029: a peer stub's door too (`openDoor('peer', …)`), gating and pinning each invoke the same way.
2469
+ */
2470
+ door, options,
2062
2471
  /**
2063
2472
  * K-42: the session this stub acts under. Re-read from the directory on
2064
2473
  * EVERY invoke rather than captured here — a stub is a capability and
@@ -2079,6 +2488,7 @@ export class CloudflareScopeHost {
2079
2488
  */
2080
2489
  verticalCaller) {
2081
2490
  const stub = this.scopeStub(scopeId);
2491
+ const systemModuleId = door?.kind === 'system' ? door.key : undefined;
2082
2492
  const cp = this.cp;
2083
2493
  const operationEntitlement = this.operationEntitlement;
2084
2494
  // The DO needs SOME principal-shaped value for `ctx.principal`; for a
@@ -2133,7 +2543,15 @@ export class CloudflareScopeHost {
2133
2543
  }
2134
2544
  // K-42: resolved per invoke, so expiry and `endImpersonation` both bite.
2135
2545
  const session = sessionId === undefined ? undefined : await this.resolveImpersonation(sessionId, tenantId, scopeId);
2136
- const envelope = await stub.invoke(operation, input, asPrincipalId, tenantId, scopeId, connectionId, requiredKey, systemModuleId, true, invokeOptions, session, capabilitySession, verticalCaller);
2546
+ const send = (systemDoorInstance) => stub.invoke(operation, input, asPrincipalId, tenantId, scopeId, connectionId, requiredKey, systemModuleId, true, invokeOptions, session, capabilitySession, verticalCaller, systemDoorInstance);
2547
+ // #1834: through the door (a module's, or #2029 a peer's), pinned to the instance its gate
2548
+ // read. A missed pin comes back as an answer, which `through` re-gates on.
2549
+ const envelope = door
2550
+ ? await door.through(async (instance) => {
2551
+ const sent = await send(instance);
2552
+ return sent.systemDoorMoved === true ? SYSTEM_DOOR_MOVED : sent;
2553
+ })
2554
+ : await send();
2137
2555
  // The operation failed and the DO handed the error back as DATA — so it still
2138
2556
  // has its code and extensions, which a throw across this boundary would have
2139
2557
  // stripped down to a message (#113 §3). Rethrown here, where the caller expects
@@ -2182,6 +2600,12 @@ export class CloudflareScopeHost {
2182
2600
  throw substratError('unavailable', `${operation} ran on a scope host that did not understand peer callers. ` +
2183
2601
  'Retry once the scope has been migrated');
2184
2602
  }
2603
+ // #1834, the same shape: a DO too old to know the door's pin ran the call without
2604
+ // checking it met the storage the door's gate read.
2605
+ if (door !== undefined && envelope.systemDoor?.honoured !== true) {
2606
+ throw substratError('unavailable', `${operation} ran on a scope host that did not understand the system door's pin. ` +
2607
+ 'Retry once the scope has been migrated');
2608
+ }
2185
2609
  if (session !== undefined && envelope.impersonation?.honoured !== true) {
2186
2610
  throw substratError('unavailable', `${operation} ran on a scope host that did not understand impersonation — the ` +
2187
2611
  'operation may have run unrecorded and unbounded. Retry once the scope has ' +
@@ -2192,11 +2616,13 @@ export class CloudflareScopeHost {
2192
2616
  // this scope's outbox — and draining anyway would run executors and
2193
2617
  // connectors as a side effect of a session that may not have side effects.
2194
2618
  // Whatever the outbox already held is the drain sweep's own backstop.
2619
+ // #1184: what this call's executors did inline, for `onExecutorOutcomes`.
2620
+ const executorOutcomes = invokeOptions?.onExecutorOutcomes ? [] : undefined;
2195
2621
  const drained = session?.mode === 'read-only'
2196
2622
  ? { attempted: 0, delivered: 0, retrying: 0, deadLettered: 0, routedToPlatform: 0 }
2197
2623
  // #1525: this drain is part of the call that emitted the events — the
2198
2624
  // coordinator's half of the post-commit tail the DO ran the consumers in.
2199
- : await this.drainExecutors(tenantId, scopeId, invokeOptions?.invocationId ?? null);
2625
+ : await this.drainExecutors(tenantId, scopeId, invokeOptions?.invocationId ?? null, executorOutcomes);
2200
2626
  // #458: the operation committed having enqueued platform intents — tell the
2201
2627
  // caller's harness so it can flag the response for the router kick (#381).
2202
2628
  // Routed connector deliveries (#574 phase 3) count too: the inline drain just
@@ -2213,6 +2639,9 @@ export class CloudflareScopeHost {
2213
2639
  invokeOptions?.onIdempotentReplay?.();
2214
2640
  if (envelope.emitted)
2215
2641
  invokeOptions?.onEmitted?.(envelope.emitted);
2642
+ if (executorOutcomes && session?.mode !== 'read-only' && !envelope.idempotency?.replayed) {
2643
+ invokeOptions?.onExecutorOutcomes?.(executorOutcomes);
2644
+ }
2216
2645
  return envelope.result;
2217
2646
  },
2218
2647
  };
@@ -2305,7 +2734,7 @@ export class CloudflareScopeHost {
2305
2734
  */
2306
2735
  const requireOrg = async (tenant, id) => {
2307
2736
  if (!(await this.cp.readOrg(tenant, id))) {
2308
- throw new Error(`unknown org ${id} in tenant ${tenant}`);
2737
+ throw substratError('not_found', `unknown org ${id} in tenant ${tenant}`);
2309
2738
  }
2310
2739
  };
2311
2740
  const mapScope = (r) => scopeSchema.parse({
@@ -2392,202 +2821,224 @@ export class CloudflareScopeHost {
2392
2821
  }
2393
2822
  };
2394
2823
  /**
2395
- * #1666: move one module's schedule switch on one scope — see `HostAdmin.revokeFromSystem`.
2824
+ * Where one scope's kill switch of one kind lands — the ONE rule the operator's switch and
2825
+ * the #1674 re-assert share, so a re-assert moves exactly the switch an operator's OFF would
2826
+ * have. `kind` is a module's schedule switch (#1666) or a peer's (#1706, #2029); `key` is the
2827
+ * module id or the peer's slug.
2396
2828
  *
2397
2829
  * Where the write lands is the whole point of the delegation branch. The shared control
2398
- * plane's own `SCOPE` namespace is the module-less placeholder, so a hosted scope's
2399
- * switch has to be moved in the deployment serving it; writing it here would report a
2400
- * switch pulled while every schedule kept firing. The audit row is written HERE either
2401
- * way — the deployment's host is CP-less, and its `recordAdmin` is a no-op.
2402
- */
2403
- /**
2404
- * #1706: move one PEER's kill switch on one scope — see `HostAdmin.revokeFromPeer`. The
2405
- * schedule switch's audit discipline (intent first, outcome after, every attempt) and,
2406
- * since part 3, its delegation: a scope served by a vertical's own deployment is switched
2407
- * THERE, over `PeerSwitchDelegation`. Without one configured the scope is still refused
2408
- * `unavailable` (`peerScopeGate`) rather than switched in the placeholder namespace.
2830
+ * plane's own `SCOPE` namespace is the module-less placeholder, so a hosted scope's switch
2831
+ * has to be moved in the deployment serving it, over that kind's delegation; writing it here
2832
+ * would report a switch pulled while every schedule kept firing, or every peer call kept
2833
+ * being admitted. A scope bound to no vertical (#1666 review) has no deployment to delegate
2834
+ * to: its store is the DO here, so the switch moves (or answers `held: false`) here too.
2835
+ * Delegating it would throw "no deployment serving scope" instead.
2409
2836
  */
2410
- const switchPeerAt = async (actor, raw, to) => {
2411
- const input = peerSwitch.parse(raw);
2412
- const { tenantId, scopeId } = input.node;
2837
+ const switchTarget = async (kind, tenantId, scopeId) => {
2413
2838
  let vertical = null;
2414
2839
  if (!this.cpLess) {
2415
2840
  const rec = await this.cp.getScopeRecord(tenantId, scopeId);
2416
2841
  if (!rec)
2417
- throw substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
2842
+ throw unknownScopeForTenant(tenantId, scopeId);
2418
2843
  vertical = rec.vertical;
2419
2844
  }
2420
- const action = to === 'off' ? 'revokeFromPeer' : 'restoreToPeer';
2421
- // Where the write lands is the whole point of this branch (#1706 part 3, mirroring
2422
- // #1666). A scope bound to a vertical is served by that vertical's own deployment, and
2423
- // its `vertical:<slug>` grants live there; switching it in the shared control plane's
2424
- // placeholder namespace would tombstone nothing and report a peer cut off while every
2425
- // call it makes kept being admitted. A scope bound to NO vertical has its store here,
2426
- // so it switches here — delegating it would throw "no deployment serving scope".
2427
- // With no delegation configured the old refusal stands, loudly, rather than a write
2428
- // into the wrong namespace.
2429
- const delegation = this.cpLess || vertical !== null ? this.peerSwitchDelegation : undefined;
2430
- if (delegation) {
2431
- // The gate's lifecycle half still runs (a suspended tenant or scope cannot be
2432
- // switched); its `scopeStub` half is what the delegation replaces.
2433
- await this.validateScopeAccess(tenantId, scopeId);
2434
- }
2435
- else {
2436
- await this.peerScopeGate(tenantId, scopeId, action);
2437
- }
2438
- const operationId = ulid();
2439
- const target = { tenantId, scopeId, vertical };
2440
- const base = { operationId, vertical: input.vertical, calls: to };
2441
- await this.recordAdmin(actor, action, target, null, { ...base, phase: 'intent', reason: input.reason });
2442
- let outcome;
2443
- try {
2444
- outcome = delegation
2445
- ? await delegation.switch({ tenantId, scopeId, vertical: input.vertical, to })
2446
- : await this.scopeStub(scopeId).switchPeer(input.vertical, scopeId, to, new Date().toISOString());
2447
- }
2448
- catch (err) {
2449
- await this.recordAdmin(actor, action, target, null, {
2450
- ...base,
2451
- phase: 'failed',
2452
- error: err instanceof Error ? err.message : String(err),
2453
- }).catch(() => undefined);
2454
- throw err;
2455
- }
2456
- await this.recordAdmin(actor, action, target, null, {
2457
- ...base,
2458
- phase: outcome.held ? 'applied' : 'refused',
2459
- changed: outcome.changed,
2460
- permissions: outcome.permissions,
2461
- });
2462
- if (!outcome.held) {
2463
- throw substratError('not_found', `scope ${scopeId} holds no grant for peer '${input.vertical}' — nothing to switch ${to} ` +
2464
- `(check the slug: it is the calling vertical's registry id, as the target's \`peers\` names it)`);
2465
- }
2466
- return {
2467
- operationId,
2468
- vertical: input.vertical,
2469
- calls: to,
2470
- changed: outcome.changed,
2471
- permissions: outcome.permissions,
2845
+ const hosted = this.cpLess || vertical !== null;
2846
+ const systemDelegation = kind === 'system' && hosted ? this.systemSwitchDelegation : undefined;
2847
+ const peerDelegation = kind === 'peer' && hosted ? this.peerSwitchDelegation : undefined;
2848
+ const delegated = systemDelegation !== undefined || peerDelegation !== undefined;
2849
+ // #1819: a scope whose store is here was rewound here, so `switchInScope` releases its
2850
+ // hold here too. A delegated move releases in the deployment serving the scope.
2851
+ // #1823 (#2030 for a peer): a subject whose only authority on the scope is a TENANT-level
2852
+ // grant has nothing in the scope's storage for the switch to find, so the directory says
2853
+ // whether it is held. A caller moving several reads them in one call and passes each in.
2854
+ // An OFF of a tenant-held MODULE is refused, after the far end answers, unless that answer
2855
+ // attests the evaluator that denies a tenant-level grant (`deniesTenantGrants`). A
2856
+ // deployment built before #1823 drops `tenantHeld`, still answers `held: true` (the module's
2857
+ // scope-level grants were there to switch), and keeps authorizing the tenant-level grant —
2858
+ // so its `held` alone would have the platform record a scope off that is not. A peer needs
2859
+ // no attestation: its door refuses on the marker in every build that has the peer switch at
2860
+ // all (`admitPeer` and `switchPeer` arrived together, #1714). Every OFF the platform drives
2861
+ // moves through here: an operator's (`switchSubjectAt`) and a re-assert's, which every carry
2862
+ // ends with.
2863
+ //
2864
+ // #2045 (Codex r3): whether a delegated deployment honours the switch fence is settled
2865
+ // BEFORE anything moves (`attestFence`, once per target), never after the fact by putting a
2866
+ // move back. Without the fence an ON is refused and an OFF goes through unfenced (see
2867
+ // `UnattestedSwitch`). A move a fenced deployment answers without the attestation still
2868
+ // throws, as defence in depth against a rollback between the preflight and the move.
2869
+ const delegation = systemDelegation ?? peerDelegation;
2870
+ let attested;
2871
+ const attestFence = () => (attested ??= delegation ? delegation.fenceSupported({ tenantId, scopeId }) : Promise.resolve(true));
2872
+ const move = async (key, to, at, tenantHeld,
2873
+ /** #2045: the switch call's fence — its operation id, or for a re-assert the record's. */
2874
+ fence) => {
2875
+ const fenced = await attestFence();
2876
+ if (!fenced && to === 'on')
2877
+ throw new UnattestedSwitch(unfencedMessage(scopeId));
2878
+ const held = tenantHeld ?? (await this.cp.tenantHeldOf(kind, tenantId, [key], at)).length > 0;
2879
+ const outcome = await (systemDelegation
2880
+ ? systemDelegation.switch({ tenantId, scopeId, moduleId: key, to, tenantHeld: held, fence })
2881
+ : peerDelegation
2882
+ ? peerDelegation.switch({ tenantId, scopeId, vertical: key, to, tenantHeld: held, fence })
2883
+ : this.switchInScope(kind, scopeId, key, to, at, held, fence));
2884
+ // An unfenced OFF answers without `fenced`, and that is what the callers read it by.
2885
+ if (fenced && outcome.fenced !== true)
2886
+ throw new UnattestedSwitch(unfencedMessage(scopeId));
2887
+ if (outcome.superseded)
2888
+ return outcome;
2889
+ if (kind === 'system' && to === 'off' && held && outcome.held && outcome.deniesTenantGrants !== true) {
2890
+ throw new UnattestedSwitch(undeniedMessage(scopeId, key));
2891
+ }
2892
+ return outcome;
2472
2893
  };
2894
+ return { vertical, delegated, move, attestFence };
2473
2895
  };
2474
2896
  /**
2475
- * Where one scope's schedule switch lands — the ONE rule `switchSystem` and the #1674
2476
- * re-assert share, so a re-assert moves exactly the switch an operator's OFF would have.
2477
- * A scope bound to no vertical (#1666 review) has no deployment to delegate to: its store
2478
- * is the DO here, so the switch moves (or answers `held: false`) here too. Delegating it
2479
- * would throw "no deployment serving scope" instead. A scope WITH a vertical still
2480
- * delegates, and still fails loudly when none serves it.
2897
+ * One kill switch moved on one scope — the ONE body `revokeFromSystem` / `restoreToSystem`
2898
+ * (#1666) and `revokeFromPeer` / `restoreToPeer` (#1706) share, so the two levers cannot
2899
+ * disagree about the record, the audit, the readback or the undo (#2029). The audit row is
2900
+ * written HERE either way — a deployment's host is CP-less, and its `recordAdmin` is a no-op.
2481
2901
  */
2482
- const systemSwitchTarget = async (tenantId, scopeId) => {
2483
- let vertical = null;
2484
- if (!this.cpLess) {
2485
- const rec = await this.cp.getScopeRecord(tenantId, scopeId);
2486
- if (!rec)
2487
- throw substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
2488
- vertical = rec.vertical;
2902
+ const switchSubjectAt = async (actor, kind, node, key, reason, to) => {
2903
+ const { tenantId, scopeId } = node;
2904
+ const { vertical, delegated, move, attestFence } = await switchTarget(kind, tenantId, scopeId);
2905
+ const action = switchActionOf(kind, to);
2906
+ if (kind === 'peer') {
2907
+ // The peer door's gate (#1706): with a delegation, its lifecycle half (a suspended tenant
2908
+ // or scope cannot be switched), the `scopeStub` half being what the delegation replaces.
2909
+ // Without one, a hosted scope is refused `unavailable` rather than switched in the
2910
+ // placeholder namespace, where it would tombstone nothing a peer call reads.
2911
+ if (delegated)
2912
+ await this.validateScopeAccess(tenantId, scopeId);
2913
+ else
2914
+ await this.peerScopeGate(tenantId, scopeId, action);
2489
2915
  }
2490
- const delegation = this.cpLess || vertical !== null ? this.systemSwitchDelegation : undefined;
2491
- // #1819: a scope whose store is here was rewound here, so `switchInScope` releases its
2492
- // hold here too. A delegated move releases in the deployment serving the scope.
2493
- const move = (moduleId, to, at) => delegation
2494
- ? delegation.switch({ tenantId, scopeId, moduleId, to })
2495
- : this.switchInScope(scopeId, moduleId, to, at);
2496
- return { vertical, move };
2497
- };
2498
- const switchSystem = async (actor, raw, to) => {
2499
- const input = systemSwitch.parse(raw);
2500
- const { tenantId, scopeId } = input.node;
2501
- const { vertical, move } = await systemSwitchTarget(tenantId, scopeId);
2502
2916
  // AUDIT FIRST (#1666 review): the intent row lands before anything moves, and the
2503
2917
  // outcome row after — every attempt, a repeat included. The scope's store and the
2504
2918
  // admin log are separate, so no order makes the pair atomic; this one fails toward
2505
2919
  // "an intent with no recorded outcome" and never toward "a switch that moved with no
2506
2920
  // audit row". A retry after a crash re-audits even though it answers `changed: false`.
2507
2921
  const operationId = ulid();
2508
- const action = to === 'off' ? 'revokeFromSystem' : 'restoreToSystem';
2509
2922
  const target = { tenantId, scopeId, vertical };
2510
- const base = { operationId, moduleId: input.moduleId, schedules: to };
2511
- await this.recordAdmin(actor, action, target, null, { ...base, phase: 'intent', reason: input.reason });
2512
- // The directory's record (#1674): ON before the scope moves, OFF after it held — see
2513
- // `recordSystemSwitchedOn` for why that order is the safe one.
2514
- const at = new Date().toISOString();
2515
- const record = { tenantId, scopeId, moduleId: input.moduleId, actor, reason: input.reason, operationId, at };
2923
+ const base = { operationId, ...switchAuditSubject(kind, key, to) };
2924
+ await this.recordAdmin(actor, action, target, null, { ...base, phase: 'intent', reason });
2516
2925
  const errorOf = (err) => (err instanceof Error ? err.message : String(err));
2517
- let prior = null;
2518
- if (to === 'on') {
2519
- try {
2520
- prior = await this.cp.recordSystemSwitchedOn(record);
2521
- }
2522
- catch (err) {
2523
- // Nothing has moved: fail the call here, audited, rather than switch a scope on
2524
- // whose record still says off (the next reconcile would switch it back off).
2525
- await this.recordAdmin(actor, action, target, null, { ...base, phase: 'failed', error: errorOf(err) }).catch(() => undefined);
2526
- throw err;
2926
+ // #2045 (Codex r3): the deployment's fence is attested BEFORE anything is recorded or moved. An
2927
+ // ON on a deployment built before it is refused here, with nothing written anywhere but the
2928
+ // audit. An OFF goes on: the kill switch must work on every deployment (see `UnattestedSwitch`).
2929
+ try {
2930
+ if (!(await attestFence()) && to === 'on') {
2931
+ throw substratError('precondition_failed', `${unfencedMessage(scopeId)} Nothing was switched.`);
2527
2932
  }
2528
2933
  }
2529
- /**
2530
- * A record write AFTER the scope moved (review #5): retried once, and a failure of both
2531
- * is answered rather than swallowed — it lands on the outcome row as `recordError`, and
2532
- * the call does not report plain success. A lost OFF record is exactly what a wipe then
2533
- * turns into a module running again.
2534
- */
2535
- const recordWrite = async (write) => {
2934
+ catch (err) {
2935
+ await this.recordAdmin(actor, action, target, null, { ...base, phase: 'refused', error: errorOf(err) }).catch(() => undefined);
2936
+ throw err;
2937
+ }
2938
+ // The directory's record (#1674; #2029 for a peer), written BEFORE the scope moves, both
2939
+ // ways — see `recordSwitchedOn` and (#1823) `recordSwitchedOff` for why that order is the
2940
+ // safe one. #2045 (Codex r3): the same write marks the subject owed and clears the scope's
2941
+ // reconcile receipt, so from here on the scope converges to this record whatever happens to
2942
+ // the move. The mark goes only when the scope confirms the move under this call's fence.
2943
+ const at = new Date().toISOString();
2944
+ const record = { kind, tenantId, scopeId, key, actor, reason, operationId, at };
2945
+ let prior;
2946
+ try {
2947
+ prior = to === 'on' ? await this.cp.recordSwitchedOn(record) : await this.cp.recordSwitchedOff(record);
2948
+ }
2949
+ catch (err) {
2950
+ // Nothing has moved: fail the call here, audited. An ON must not switch a scope on
2951
+ // whose record still says off (the next reconcile would switch it back off), and an
2952
+ // OFF must not leave a scope off that the record does not know of (#1823).
2953
+ await this.recordAdmin(actor, action, target, null, { ...base, phase: 'failed', error: errorOf(err) }).catch(() => undefined);
2954
+ throw err;
2955
+ }
2956
+ // #2045: a newer call on this subject has recorded its position already: this one writes
2957
+ // nothing, here or in the scope, and says so.
2958
+ if (recordWriteSuperseded(prior, record)) {
2959
+ await this.recordAdmin(actor, action, target, null, { ...base, phase: 'refused', superseded: true });
2960
+ throw substratError('conflict', switchSupersededMessage(kind, scopeId, key, to));
2961
+ }
2962
+ let outcome;
2963
+ try {
2536
2964
  try {
2537
- await write();
2538
- return null;
2965
+ outcome = await move(key, to, at, undefined, operationId);
2539
2966
  }
2540
- catch {
2541
- try {
2542
- await write();
2543
- return null;
2544
- }
2545
- catch (err) {
2546
- return errorOf(err);
2547
- }
2967
+ catch (first) {
2968
+ if (first instanceof UnattestedSwitch)
2969
+ throw first;
2970
+ // #2045 (Codex r2): one retry under this call's own fence — idempotent, and refused by
2971
+ // the scope if a newer call has moved it meanwhile.
2972
+ outcome = await move(key, to, new Date().toISOString(), undefined, operationId);
2548
2973
  }
2549
- };
2550
- let outcome;
2551
- try {
2552
- outcome = await move(input.moduleId, to, at);
2553
2974
  }
2554
2975
  catch (err) {
2555
- const recordError = prior ? await recordWrite(() => this.cp.restoreSystemSwitchRecord(record, prior)) : null;
2556
- // Best effort: the original error is what the caller must see, and the intent row
2557
- // already says an attempt was made.
2976
+ // The move threw twice, or was answered by a deployment that could not attest it. Either
2977
+ // way it may have landed, and an older call's move may still be in flight behind it, so no
2978
+ // readback can settle the scope and nothing is put back: the record is this call's position
2979
+ // under the newest fence, and its write-ahead mark makes the next re-assert move the scope
2980
+ // to it, under that fence, whichever way it lies now.
2558
2981
  await this.recordAdmin(actor, action, target, null, {
2559
2982
  ...base,
2560
- phase: 'failed',
2983
+ phase: err instanceof UnattestedSwitch ? 'refused' : 'failed',
2561
2984
  error: errorOf(err),
2562
- ...(recordError ? { recordError } : {}),
2985
+ recordKept: true,
2986
+ reassertOwed: true,
2563
2987
  }).catch(() => undefined);
2988
+ if (err instanceof UnattestedSwitch)
2989
+ throw substratError('precondition_failed', err.message);
2564
2990
  throw err;
2565
2991
  }
2566
- // OFF is recorded once the scope held it. A refused ON moved nothing, so its record
2567
- // write is undone: left `on`, the next reconcile of a wiped scope would leave the
2568
- // module running.
2569
- const recordError = to === 'off' && outcome.held
2570
- ? await recordWrite(() => this.cp.recordSystemSwitchedOff(record))
2571
- : to === 'on' && !outcome.held && prior
2572
- ? await recordWrite(() => this.cp.restoreSystemSwitchRecord(record, prior))
2573
- : null;
2992
+ // #2045: the scope has applied a newer call on this subject, so this move wrote nothing. The
2993
+ // record is that newer call's too (its write overwrote this one's, mark included), so nothing
2994
+ // is undone and no mark is this call's to clear.
2995
+ if (outcome.superseded) {
2996
+ await this.recordAdmin(actor, action, target, null, { ...base, phase: 'refused', superseded: true });
2997
+ throw substratError('conflict', switchSupersededMessage(kind, scopeId, key, to));
2998
+ }
2999
+ // A call that held nothing moved nothing, so its record write is undone: left `off`, it
3000
+ // would switch the subject off the day it is installed; left `on`, the next reconcile of
3001
+ // a wiped scope would leave it running. Retried once; a failure of both lands on the
3002
+ // outcome row as `recordError`, and the call says so.
3003
+ //
3004
+ // The scope confirmed this call's move (or that there was nothing to move), so its mark goes.
3005
+ // Held nothing, it goes WITH the undo, in one directory transaction: the row put back carries
3006
+ // the prior call's older id, and no later re-assert confirms under a fence older than this
3007
+ // call's mark. An undo that fails twice leaves the record AND the mark this call's, and the
3008
+ // next re-assert confirms both under it. Held, a failed clear is not the call's: the mark only
3009
+ // costs the next re-assert one idempotent move under this same record.
3010
+ //
3011
+ // An unfenced OFF that held something (a deployment built before the fence) confirmed nothing
3012
+ // under its fence, so its mark stays for the first re-assert after the vertical is redeployed.
3013
+ const unfenced = outcome.held && outcome.fenced !== true;
3014
+ let recordError = null;
3015
+ let owedError = null;
3016
+ if (!outcome.held) {
3017
+ const undo = () => this.cp.restoreSwitchRecord(record, prior, true);
3018
+ recordError = await undo().then(() => null, () => undo().then(() => null, errorOf));
3019
+ }
3020
+ else if (!unfenced) {
3021
+ owedError = await this.cp.clearSwitchOwed(kind, tenantId, scopeId, key, operationId).then(() => null, errorOf);
3022
+ }
2574
3023
  await this.recordAdmin(actor, action, target, null, {
2575
3024
  ...base,
2576
3025
  phase: outcome.held ? 'applied' : 'refused',
2577
3026
  changed: outcome.changed,
2578
3027
  permissions: outcome.permissions,
3028
+ ...(unfenced ? { unfenced: true, reassertOwed: true } : {}),
2579
3029
  ...(recordError ? { recordError } : {}),
3030
+ ...(owedError ? { owedError } : {}),
2580
3031
  });
2581
3032
  if (!outcome.held) {
2582
- throw substratError('not_found', `scope ${scopeId} holds no system grant for module '${input.moduleId}' — nothing to switch ${to} ` +
2583
- `(check the module id: it is the module's manifest id, e.g. '@substrat-run/engine-absence')` +
3033
+ throw substratError('not_found', switchNotFoundMessage(kind, scopeId, key, to) +
2584
3034
  (recordError ? `; and its directory record could not be put back (${recordError})` : ''));
2585
3035
  }
2586
- if (recordError) {
2587
- throw substratError('unavailable', `module '${input.moduleId}' is switched ${to} on scope ${scopeId}, but the directory's record of it ` +
2588
- `could not be written (${recordError}) — repeat the call to record it; a wipe of this scope would ` +
2589
- `otherwise lose the switch`);
2590
- }
3036
+ return { operationId, outcome };
3037
+ };
3038
+ /** #1666: move one module's schedule switch on one scope — see `HostAdmin.revokeFromSystem`. */
3039
+ const switchSystem = async (actor, raw, to) => {
3040
+ const input = systemSwitch.parse(raw);
3041
+ const { operationId, outcome } = await switchSubjectAt(actor, 'system', input.node, input.moduleId, input.reason, to);
2591
3042
  return {
2592
3043
  operationId,
2593
3044
  moduleId: input.moduleId,
@@ -2596,6 +3047,18 @@ export class CloudflareScopeHost {
2596
3047
  permissions: outcome.permissions,
2597
3048
  };
2598
3049
  };
3050
+ /** #1706: move one PEER's kill switch on one scope — see `HostAdmin.revokeFromPeer`. */
3051
+ const switchPeerAt = async (actor, raw, to) => {
3052
+ const input = peerSwitch.parse(raw);
3053
+ const { operationId, outcome } = await switchSubjectAt(actor, 'peer', input.node, input.vertical, input.reason, to);
3054
+ return {
3055
+ operationId,
3056
+ vertical: input.vertical,
3057
+ calls: to,
3058
+ changed: outcome.changed,
3059
+ permissions: outcome.permissions,
3060
+ };
3061
+ };
2599
3062
  /**
2600
3063
  * The status read's admin-log join (#1674): the `intent` row of the `revokeFromSystem`
2601
3064
  * still in force for each OFF module — actor, reason, when. Paired to its `applied` row
@@ -2651,7 +3114,7 @@ export class CloudflareScopeHost {
2651
3114
  if (!this.cpLess) {
2652
3115
  const rec = await this.cp.getScopeRecord(tenantId, scopeId);
2653
3116
  if (!rec)
2654
- throw substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
3117
+ throw unknownScopeForTenant(tenantId, scopeId);
2655
3118
  vertical = rec.vertical;
2656
3119
  }
2657
3120
  // A hosted scope (a vertical bound, on a non-cpLess host) has no local storage worth
@@ -2668,7 +3131,7 @@ export class CloudflareScopeHost {
2668
3131
  delegation
2669
3132
  ? delegation.status({ tenantId, scopeId })
2670
3133
  : this.scopeStub(scopeId).systemGrantsStatus().then((rows) => systemScheduleEntry.array().parse(rows)),
2671
- this.cp.systemSwitchRecordsOf(tenantId, scopeId),
3134
+ this.cp.switchRecordsOf('system', tenantId, scopeId),
2672
3135
  ]);
2673
3136
  const offModules = new Set(states.filter((s) => s.schedules === 'off').map((s) => s.moduleId));
2674
3137
  const explanations = offModules.size > 0
@@ -2687,70 +3150,137 @@ export class CloudflareScopeHost {
2687
3150
  };
2688
3151
  /**
2689
3152
  * #1674: put the directory's OFF positions back into one scope — see
2690
- * `HostAdmin.reassertSystemSwitches`. It moves the switch through `systemSwitchTarget`,
2691
- * so it lands wherever an operator's OFF would have — for a hosted scope, over the
2692
- * `/internal/system-switch` route every deployment since #1666 answers.
3153
+ * `HostAdmin.reassertSystemSwitches`. Every kind (#2029): the recorded-off modules, then the
3154
+ * recorded-off peers. Each moves through `switchTarget`, so it lands wherever an operator's
3155
+ * OFF would have — for a hosted scope, over that kind's `/internal/*` switch route.
2693
3156
  */
2694
3157
  const reassertSystemSwitchesOf = async (actor, node, opts) => {
3158
+ const results = [];
3159
+ for (const kind of SWITCH_KINDS)
3160
+ results.push(...(await reassertKind(actor, kind, node, opts)));
3161
+ return results;
3162
+ };
3163
+ /** One kind's half of `reassertSystemSwitchesOf`. */
3164
+ const reassertKind = async (actor, kind, node, opts) => {
2695
3165
  const { tenantId, scopeId } = node;
2696
- const { vertical, move } = await systemSwitchTarget(tenantId, scopeId);
2697
- const recorded = new Map(await this.cp.systemSwitchRecordsOf(tenantId, scopeId));
3166
+ const { vertical, delegated, move, attestFence } = await switchTarget(kind, tenantId, scopeId);
3167
+ const action = reassertActionOf(kind);
3168
+ const target = { tenantId, scopeId, vertical };
3169
+ const recorded = new Map(await this.cp.switchRecordsOf(kind, tenantId, scopeId));
3170
+ // #2045: every switch call's write-ahead mark, with the newest call that wrote it.
3171
+ const owed = new Map(await this.cp.switchesOwedOf(kind, tenantId, scopeId));
2698
3172
  // #1742 review: moves the deployment made from a stale list (restored ON after the list
2699
3173
  // was read), to undo before the OFF pass below.
2700
- const reverts = staleCarryReverts(recorded, opts?.appliedInUnit);
3174
+ const reverts = staleCarryReverts(kind, recorded, opts?.appliedInUnit);
2701
3175
  // A hosted scope with no delegation configured (Copilot review): this host's own
2702
3176
  // namespace is the module-less placeholder, where the switch would answer `held: false`
2703
3177
  // quietly — a re-assert reported done, and a receipt written, while the deployment
2704
- // serving the scope keeps its schedules running. Refused loudly instead, as the status
2705
- // read is, and only when a re-assert is owed: nothing recorded, nothing to refuse.
2706
- const owed = reverts.length > 0 || [...recorded.values()].includes('off');
2707
- if (owed && !this.cpLess && vertical !== null && !this.systemSwitchDelegation) {
3178
+ // serving the scope keeps its schedules running or its peer admitted. Refused loudly
3179
+ // instead, as the status read is, and only when a re-assert is owed: nothing recorded,
3180
+ // nothing to refuse.
3181
+ const reassertOwed = reverts.length > 0 || owed.size > 0 || [...recorded.values()].includes('off');
3182
+ if (reassertOwed && !this.cpLess && vertical !== null && !delegated) {
2708
3183
  throw substratError('unavailable', `no delegation configured for hosted scope ${scopeId} (vertical '${vertical}') — cannot re-assert ` +
2709
- `its switched-off schedules in the deployment serving it`);
3184
+ `its switched-off ${DOOR_WORDS[kind].switched} in the deployment serving it`);
2710
3185
  }
3186
+ // #2045 (Codex r3): a subject's mark goes only once the scope has confirmed a move under the
3187
+ // record's fence — a fenced answer that was not superseded. A throw anywhere below leaves
3188
+ // every mark not yet confirmed for the next pass.
3189
+ // An unfenced OFF (a deployment built before the fence) confirms nothing, so its mark stays.
3190
+ const confirm = async (key, fence, outcome) => {
3191
+ if (owed.has(key) && !outcome.superseded && outcome.fenced === true) {
3192
+ await this.cp.clearSwitchOwed(kind, tenantId, scopeId, key, fence);
3193
+ }
3194
+ };
3195
+ // #2045 follow-up: on a deployment built before the fence no ON is sent (see
3196
+ // `UnattestedSwitch`). Every OFF below still moves, and the pass refuses at its end, so a
3197
+ // sweep records no receipt for a scope whose owed ON was not sent. Asked only when an ON is due.
3198
+ const onWithheld = [];
3199
+ const withholdOn = async (key) => {
3200
+ if (await attestFence())
3201
+ return false;
3202
+ onWithheld.push(key);
3203
+ return true;
3204
+ };
2711
3205
  const at = new Date().toISOString();
2712
3206
  const reverted = new Set();
2713
- for (const moduleId of reverts) {
3207
+ for (const key of reverts) {
2714
3208
  // Re-read immediately before the move: a staff OFF that completed since the read above
2715
- // (record `off`) must not get a transient ON a due schedule could run in.
2716
- const current = new Map(await this.cp.systemSwitchRecordsOf(tenantId, scopeId));
2717
- if (current.get(moduleId) !== 'on')
3209
+ // (record `off`) must not get a transient ON a due schedule, or a peer call, could use.
3210
+ // #2045: under the fence of the ON the record holds, which is the operator's own call — read
3211
+ // with the position, so it is that ON's fence and no other call's.
3212
+ const current = new Map(await this.cp.switchRecordStatesOf(kind, tenantId, scopeId)).get(key);
3213
+ if (current?.position !== 'on')
2718
3214
  continue;
2719
- const outcome = await move(moduleId, 'on', at);
2720
- reverted.add(moduleId);
3215
+ const fence = current.fence;
3216
+ if (await withholdOn(key))
3217
+ continue;
3218
+ const outcome = await move(key, 'on', at, undefined, fence);
3219
+ reverted.add(key);
2721
3220
  if (outcome.changed) {
2722
- await this.recordAdmin(actor, 'reassertSystemSwitch', { tenantId, scopeId, vertical }, null, {
2723
- operationId: ulid(),
2724
- ...staleCarryRevertRow(moduleId, outcome),
2725
- });
3221
+ await this.recordAdmin(actor, action, target, null, { operationId: ulid(), ...staleCarryRevertRow(kind, key, outcome) });
2726
3222
  }
3223
+ await confirm(key, fence, outcome);
2727
3224
  }
2728
3225
  // Read AFTER the reverts: an OFF that landed meanwhile is switched off below.
2729
- const modules = await this.cp.switchedOffModulesOf(tenantId, scopeId);
3226
+ const { keys, tenantHeld, fences } = await this.recordedOff(kind, tenantId, scopeId, at);
2730
3227
  // #1742: what the deployment already switched off inside its own unit, audited here —
2731
- // the move below answers `changed: false` for it and would write no row.
2732
- // A move the revert above undid is not credited as an in-unit OFF.
2733
- const applied = opts?.appliedInUnit?.filter((a) => !reverted.has(a.moduleId));
2734
- for (const row of inUnitMovesToAudit(modules, applied)) {
2735
- await this.recordAdmin(actor, 'reassertSystemSwitch', { tenantId, scopeId, vertical }, null, {
2736
- operationId: ulid(),
2737
- ...row,
2738
- });
3228
+ // the move below answers `changed: false` for it and would write no row. A move the
3229
+ // revert above undid is not credited as an in-unit OFF.
3230
+ for (const row of inUnitMovesToAudit(kind, keys, opts?.appliedInUnit, reverted)) {
3231
+ await this.recordAdmin(actor, action, target, null, { operationId: ulid(), ...row });
2739
3232
  }
3233
+ // #2045: every move here carries the fence of the call the record holds, so a scope a newer call
3234
+ // has moved since refuses it. A deployment from before the fence takes the OFF unfenced.
2740
3235
  const results = [];
2741
- for (const moduleId of modules) {
2742
- const outcome = await move(moduleId, 'off', at);
3236
+ const held = new Set(tenantHeld);
3237
+ for (const key of keys) {
3238
+ const fence = fences.get(key); // `recordedOff` read each key with its own row's fence
3239
+ const outcome = await move(key, 'off', at, held.has(key), fence);
2743
3240
  if (outcome.changed) {
2744
- await this.recordAdmin(actor, 'reassertSystemSwitch', { tenantId, scopeId, vertical }, null, {
3241
+ await this.recordAdmin(actor, action, target, null, {
2745
3242
  operationId: ulid(),
2746
- moduleId,
2747
- schedules: 'off',
2748
- phase: 'applied',
2749
- changed: true,
2750
- permissions: outcome.permissions,
3243
+ ...reassertOffRow(kind, key, outcome.permissions),
3244
+ ...(outcome.fenced === true ? {} : { unfenced: true }),
2751
3245
  });
2752
3246
  }
2753
- results.push({ moduleId, held: outcome.held, changed: outcome.changed });
3247
+ results.push(reassertEntry(kind, key, outcome));
3248
+ await confirm(key, fence, outcome);
3249
+ }
3250
+ // A subject marked owed converges to its record in either direction: OFF above, and here a
3251
+ // record of ON — sent whatever the scope reads now, because a scope that reads ON may still
3252
+ // be behind the record's fence, with an older call's OFF in flight that only the fence can
3253
+ // refuse (Codex r3, finding 1). Idempotent: an ON already in place changes nothing and moves
3254
+ // the fence. Every other record of ON keeps #1674's rule and turns nothing on.
3255
+ const settled = new Set([...keys, ...reverted]);
3256
+ const pending = [...owed.keys()].filter((key) => !settled.has(key));
3257
+ if (pending.length) {
3258
+ // Position and fence in one read: an ON is sent only under the fence of the ON it read.
3259
+ const now = new Map(await this.cp.switchRecordStatesOf(kind, tenantId, scopeId));
3260
+ for (const key of pending) {
3261
+ const row = now.get(key);
3262
+ if (row?.position !== 'on') {
3263
+ // No record (a call that held nothing, whose undo removed its row), or one that turned
3264
+ // OFF since the read above, which the next pass moves: nothing to send on this one. A
3265
+ // missing row's mark goes under its own id, so a newer call's mark stays.
3266
+ if (row === undefined)
3267
+ await this.cp.clearSwitchOwed(kind, tenantId, scopeId, key, owed.get(key));
3268
+ continue;
3269
+ }
3270
+ const fence = row.fence;
3271
+ if (await withholdOn(key))
3272
+ continue;
3273
+ const outcome = await move(key, 'on', at, undefined, fence);
3274
+ if (outcome.changed) {
3275
+ await this.recordAdmin(actor, action, target, null, { operationId: ulid(), ...reassertOnRow(kind, key, outcome.permissions) });
3276
+ results.push(reassertEntry(kind, key, outcome));
3277
+ }
3278
+ await confirm(key, fence, outcome);
3279
+ }
3280
+ }
3281
+ if (onWithheld.length) {
3282
+ throw substratError('precondition_failed', `${unfencedMessage(scopeId)} Its recorded OFF positions were applied; the owed ON of ` +
3283
+ `${onWithheld.map((k) => `'${k}'`).join(', ')} was not sent.`);
2754
3284
  }
2755
3285
  return results;
2756
3286
  };
@@ -2769,7 +3299,7 @@ export class CloudflareScopeHost {
2769
3299
  if (!this.cpLess) {
2770
3300
  const rec = await this.cp.getScopeRecord(tenantId, scopeId);
2771
3301
  if (!rec)
2772
- throw substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
3302
+ throw unknownScopeForTenant(tenantId, scopeId);
2773
3303
  vertical = rec.vertical;
2774
3304
  }
2775
3305
  if (!this.cpLess && vertical !== null && !this.peerSwitchDelegation) {
@@ -2837,18 +3367,21 @@ export class CloudflareScopeHost {
2837
3367
  unassignRole: async (actor, assignment) => {
2838
3368
  // Tombstone (K-21) — the checker skips revoked rows. A no-op returns false so
2839
3369
  // a repeat unassign stays silent (no second audit row, no needless fan-out).
2840
- const subject = `principal:${assignment.principalId}`;
2841
3370
  const relation = `role:${assignment.roleKey}`;
2842
- const now = new Date().toISOString();
2843
- const changed = assignment.node.scopeId
2844
- ? await this.scopeStub(assignment.node.scopeId).revokeTuple(subject, relation, `scope:${assignment.node.scopeId}`, now)
2845
- : await this.cp.revokeTenantTuple(assignment.node.tenantId, subject, relation, `tenant:${assignment.node.tenantId}`, now);
3371
+ const { tenantId, scopeId } = assignment.node;
3372
+ if (!scopeId) {
3373
+ // One ControlPlaneDO method (#1184): the revoke, the removal fence (raised even when
3374
+ // nothing was held) and the audit row, so no add can land between any two of them.
3375
+ const changed = await this.cp.revokeAndFence(tenantId, assignment.principalId, relation, `tenant:${tenantId}`, this.adminEntry(actor, 'unassignRole', { tenantId, scopeId: null }, assignment, null));
3376
+ // A tenant-level revoke changes the projected set — the tombstone must reach scopes.
3377
+ if (changed)
3378
+ await this.fanOut(tenantId);
3379
+ return;
3380
+ }
3381
+ const changed = await this.scopeStub(scopeId).revokeTuple(`principal:${assignment.principalId}`, relation, `scope:${scopeId}`, new Date().toISOString());
2846
3382
  if (!changed)
2847
3383
  return;
2848
- await this.recordAdmin(actor, 'unassignRole', { tenantId: assignment.node.tenantId, scopeId: assignment.node.scopeId }, assignment, null);
2849
- // A tenant-level revoke changes the projected set — the tombstone must reach scopes.
2850
- if (!assignment.node.scopeId)
2851
- await this.fanOut(assignment.node.tenantId);
3384
+ await this.recordAdmin(actor, 'unassignRole', { tenantId, scopeId }, assignment, null);
2852
3385
  },
2853
3386
  grant: async (actor, raw) => {
2854
3387
  // Parsed like its `grantToConnection`/`grantToSystem` siblings, not taken on
@@ -2968,6 +3501,19 @@ export class CloudflareScopeHost {
2968
3501
  await this.recordAccess(actor, 'listSystemSwitches', { tenantId: filter?.tenantId ?? null }, filter ?? null, rows.length);
2969
3502
  return rows.map((r) => systemSwitchRecord.parse(r));
2970
3503
  },
3504
+ tenantHeldSystemModules: async (actor, tenantId, moduleIds) => {
3505
+ const held = moduleIds.length
3506
+ ? await this.cp.tenantHeldOf('system', tenantId, moduleIds, new Date().toISOString())
3507
+ : [];
3508
+ await this.recordAccess(actor, 'tenantHeldSystemModules', { tenantId }, { moduleIds: [...moduleIds] }, held.length);
3509
+ return held;
3510
+ },
3511
+ peerSwitchCarry: async (actor, node) => {
3512
+ const { keys: switchedOffPeers, tenantHeld: tenantHeldPeers, fences: all, } = await this.recordedOff('peer', node.tenantId, node.scopeId, new Date().toISOString());
3513
+ const fences = Object.fromEntries(switchedOffPeers.map((v) => [v, all.get(v)]));
3514
+ await this.recordAccess(actor, 'peerSwitchCarry', node, null, switchedOffPeers.length);
3515
+ return { switchedOffPeers, tenantHeldPeers, fences };
3516
+ },
2971
3517
  reassertSystemSwitches: reassertSystemSwitchesOf,
2972
3518
  // #1672 — the platform's two capability verbs. Audited AFTER the write, on both, and
2973
3519
  // the failure each leaves is the safe one: a mint whose audit row did not land never
@@ -3033,7 +3579,7 @@ export class CloudflareScopeHost {
3033
3579
  // another tenant's traffic. Exception: the holder is ARCHIVED or REAPED (a
3034
3580
  // deleted app, storage since wiped) — it has released the name, so the rebind
3035
3581
  // reclaims it.
3036
- throw new Error(`hostname '${parsed.hostname}' is already bound to another scope`);
3582
+ throw substratError('conflict', `hostname '${parsed.hostname}' is already bound to another scope`);
3037
3583
  }
3038
3584
  // Exactly one canonical per (scope, surface).
3039
3585
  if (parsed.canonical)
@@ -3054,7 +3600,7 @@ export class CloudflareScopeHost {
3054
3600
  const hostname = raw.toLowerCase(); // DNS is case-insensitive; the map is normalized
3055
3601
  const row = await this.cp.readHostname(hostname);
3056
3602
  if (!row)
3057
- throw new Error(`unknown hostname '${hostname}'`);
3603
+ throw substratError('not_found', `unknown hostname '${hostname}'`);
3058
3604
  if (row.status === status)
3059
3605
  return; // idempotent, unaudited
3060
3606
  await this.cp.setHostnameStatus(hostname, status, note ?? null);
@@ -3064,7 +3610,7 @@ export class CloudflareScopeHost {
3064
3610
  const hostname = raw.toLowerCase(); // DNS is case-insensitive; the map is normalized
3065
3611
  const row = await this.cp.readHostname(hostname);
3066
3612
  if (!row)
3067
- throw new Error(`unknown hostname '${hostname}'`);
3613
+ throw substratError('not_found', `unknown hostname '${hostname}'`);
3068
3614
  // A poll that finds nothing changed (same status, same records, id already set)
3069
3615
  // is not an event — skip the write and the audit entry, so the reconcile sweep
3070
3616
  // does not flood the admin log with no-op rows every interval.
@@ -3150,7 +3696,7 @@ export class CloudflareScopeHost {
3150
3696
  if (existing.owner_tenant !== parsed.ownerTenant) {
3151
3697
  throw substratError('conflict', `vertical '${parsed.slug}' is owned by ${existing.owner_tenant ?? 'the platform'}, not ${parsed.ownerTenant ?? 'the platform'}`);
3152
3698
  }
3153
- throw new Error(`vertical '${parsed.slug}' is already registered as ${existing.source}`);
3699
+ throw substratError('conflict', `vertical '${parsed.slug}' is already registered as ${existing.source}`);
3154
3700
  }
3155
3701
  await this.cp.insertVertical(parsed.slug, parsed.name, parsed.source, parsed.ownerTenant, envSpecJson, installSpecJson, parsed.listed ? 1 : 0, new Date().toISOString());
3156
3702
  await this.recordAdmin(actor, 'registerVertical', { tenantId: null }, null, parsed);
@@ -3331,11 +3877,11 @@ export class CloudflareScopeHost {
3331
3877
  // nothing to diff against — the gate is about change, not existence.
3332
3878
  if (outgoing) {
3333
3879
  if (outgoing.permission_digest !== incoming.permission_digest && !ack.permissionChange) {
3334
- throw new Error(`promotion changes the permission surface (${outgoing.permission_digest} → ` +
3880
+ throw substratError('conflict', `promotion changes the permission surface (${outgoing.permission_digest} → ` +
3335
3881
  `${incoming.permission_digest}) — acknowledge it explicitly to promote`);
3336
3882
  }
3337
3883
  if (outgoing.migration_digest !== incoming.migration_digest && !ack.migrationChange) {
3338
- throw new Error(`promotion changes migrations (${outgoing.migration_digest} → ` +
3884
+ throw substratError('conflict', `promotion changes migrations (${outgoing.migration_digest} → ` +
3339
3885
  `${incoming.migration_digest}) — acknowledge it explicitly to promote`);
3340
3886
  }
3341
3887
  // #1705 PR 3: an export an installed consumer imports, dropped or re-versioned.
@@ -3595,7 +4141,7 @@ export class CloudflareScopeHost {
3595
4141
  },
3596
4142
  createOrg: async (actor, input) => {
3597
4143
  const parsed = createOrgInput.parse(input);
3598
- const created = await this.cp.createOrg(parsed.id, parsed.tenantId, parsed.slug, parsed.name, new Date().toISOString());
4144
+ const created = await this.directory('createOrg', parsed.id, parsed.tenantId, parsed.slug, parsed.name, new Date().toISOString());
3599
4145
  if (!created)
3600
4146
  return; // idempotent, and a no-op is not audited
3601
4147
  await this.recordAdmin(actor, 'createOrg', { tenantId: parsed.tenantId }, null, parsed);
@@ -3616,15 +4162,27 @@ export class CloudflareScopeHost {
3616
4162
  await this.recordAdmin(actor, 'addMember', { tenantId }, null, { principal, orgId });
3617
4163
  await this.fanOut(tenantId); // membership is a tenant-level tuple
3618
4164
  },
4165
+ applyMembership: async (actor, change) => {
4166
+ // The row is minted here, where attribution and `causedBy` live, and written by the
4167
+ // ControlPlaneDO in the same synchronous method as the fence, the bound and the tuple
4168
+ // (#1184): one DO unit, so nothing lands between the check and the write.
4169
+ const { tenantId, principal, roleKey, op } = change;
4170
+ const assignment = { principalId: principal, roleKey, node: { tenantId, scopeId: null } };
4171
+ const result = await this.cp.applyMembership(change, op === 'add'
4172
+ ? this.adminEntry(actor, 'assignRole', { tenantId, scopeId: null }, null, assignment)
4173
+ : this.adminEntry(actor, 'unassignRole', { tenantId, scopeId: null }, assignment, null));
4174
+ // The tenant-level tuple, or its tombstone, reaches the projections.
4175
+ if (result.applied && (op === 'add' || result.changed))
4176
+ await this.fanOut(tenantId);
4177
+ return result;
4178
+ },
3619
4179
  removeMember: async (actor, tenantId, principal, orgId) => {
3620
4180
  await requireOrg(tenantId, orgId);
3621
- // Tombstone (K-21), never DELETE. The DO reports whether anything changed
3622
- // so a repeat revoke stays a silent no-op rather than a second audit row.
3623
- const changed = await this.cp.revokeMember(tenantId, `principal:${principal}`, `org:${orgId}`, new Date().toISOString());
3624
- if (!changed)
3625
- return;
3626
- await this.recordAdmin(actor, 'removeMember', { tenantId }, { principal, orgId }, null);
3627
- await this.fanOut(tenantId); // the tombstone must reach the projections
4181
+ // Tombstone (K-21), never DELETE, with the removal fence in the same DO method
4182
+ // (#1184). A repeat revoke stays a silent no-op rather than a second audit row.
4183
+ const changed = await this.cp.revokeAndFence(tenantId, principal, 'member', `org:${orgId}`, this.adminEntry(actor, 'removeMember', { tenantId }, { principal, orgId }, null));
4184
+ if (changed)
4185
+ await this.fanOut(tenantId); // the tombstone must reach the projections
3628
4186
  },
3629
4187
  listMembers: async (actor, tenantId, orgId, options) => {
3630
4188
  await requireOrg(tenantId, orgId);
@@ -3638,20 +4196,20 @@ export class CloudflareScopeHost {
3638
4196
  },
3639
4197
  createTenant: async (actor, input) => {
3640
4198
  const parsed = createTenantInput.parse(input);
3641
- const created = await this.cp.createTenant(parsed.id, parsed.slug, parsed.name, new Date().toISOString(), parsed.provisionedByTenant ?? null);
4199
+ const created = await this.directory('createTenant', parsed.id, parsed.slug, parsed.name, new Date().toISOString(), parsed.provisionedByTenant ?? null);
3642
4200
  // Idempotent: re-creating an existing tenant is a no-op, not audited.
3643
4201
  if (!created)
3644
4202
  return;
3645
4203
  await this.recordAdmin(actor, 'createTenant', { tenantId: parsed.id }, null, created);
3646
4204
  },
3647
4205
  setTenantStatus: async (actor, tenantId, status) => {
3648
- const before = await this.cp.setTenantStatus(tenantId, status);
4206
+ const before = await this.directory('setTenantStatus', tenantId, status);
3649
4207
  await this.recordAdmin(actor, 'setTenantStatus', { tenantId }, { status: before }, { status });
3650
4208
  // #1713: every hosted scope under the tenant holds or resumes its work by this.
3651
4209
  await this.deliverLifecycles(actor, { tenantId });
3652
4210
  },
3653
4211
  setTenantName: async (actor, tenantId, name) => {
3654
- const before = await this.cp.setTenantName(tenantId, name);
4212
+ const before = await this.directory('setTenantName', tenantId, name);
3655
4213
  if (before === name)
3656
4214
  return; // no-op is not audited — nothing changed
3657
4215
  await this.recordAdmin(actor, 'setTenantName', { tenantId }, { name: before }, { name });
@@ -3661,7 +4219,7 @@ export class CloudflareScopeHost {
3661
4219
  // first (archive-if-needed → reapScope in the vertical deployment); the DO clears
3662
4220
  // the tenant's PII/config rows and flips the row to a `reaped` tombstone, keeping
3663
4221
  // the row + admin log. Only a `deleting` tenant may be reaped (checked in the DO).
3664
- const before = await this.cp.reapTenant(tenantId);
4222
+ const before = await this.directory('reapTenant', tenantId);
3665
4223
  await this.recordAdmin(actor, 'reapTenant', { tenantId }, { status: before }, { status: 'reaped' });
3666
4224
  },
3667
4225
  listTenants: async (actor, page) => {
@@ -3745,7 +4303,7 @@ export class CloudflareScopeHost {
3745
4303
  // Not `peerScopeGate`: a read must not migrate, and the phase makes this call once
3746
4304
  // per edge per pass, where an extra round trip is a fleet-wide cost.
3747
4305
  this.assertServedHere(await this.scopeRecordForRead(tenantId, scopeId), scopeId, 'readExportedEvents');
3748
- const batch = exportedBatch.parse(await this.scopeStub(scopeId).exportedEventsRead(input, tenantId, scopeId));
4306
+ const batch = await this.readExportsThroughDoor(input, tenantId, scopeId);
3749
4307
  await this.recordAccess(actor, 'readExportedEvents', { tenantId, scopeId }, { consumer: input.consumer, after: input.after, limit: input.limit }, batch.events.length);
3750
4308
  return batch;
3751
4309
  },
@@ -3925,12 +4483,18 @@ export class CloudflareScopeHost {
3925
4483
  await this.recordAccess(actor, 'lifecycleFlow', { tenantId, scopeId }, delegatedReadParams.lifecycleFlow(input), flow.observation.events);
3926
4484
  return flow;
3927
4485
  },
4486
+ operationSeries: async (actor, tenantId, scopeId, input) => {
4487
+ await this.scopeRecordForRead(tenantId, scopeId);
4488
+ const series = await this.scopeStub(scopeId).operationSeries(input);
4489
+ await this.recordAccess(actor, 'operationSeries', { tenantId, scopeId }, delegatedReadParams.operationSeries(input), operationSeriesCount(series));
4490
+ return series;
4491
+ },
3928
4492
  readScopeTable: async (actor, tenantId, scopeId, input) => {
3929
4493
  await this.scopeRecordForRead(tenantId, scopeId);
3930
4494
  // Checked here as well as in the ScopeDO, so the refusal keeps its code across the hop.
3931
4495
  assertRowLimit('limit', input.limit);
3932
4496
  assertRowOffset('offset', input.offset);
3933
- const page = await this.scopeStub(scopeId).introspectTable(input.table, input.limit, input.offset);
4497
+ const page = unwrapReply(await this.scopeStub(scopeId).introspectTableReply(input.table, input.limit, input.offset));
3934
4498
  await this.recordAccess(actor, 'readScopeTable', { tenantId, scopeId }, { table: input.table, limit: page.limit, offset: page.offset }, page.rows.length);
3935
4499
  return page;
3936
4500
  },
@@ -3971,7 +4535,7 @@ export class CloudflareScopeHost {
3971
4535
  },
3972
4536
  queryScope: async (actor, tenantId, scopeId, input) => {
3973
4537
  await this.scopeRecordForRead(tenantId, scopeId);
3974
- const result = await this.scopeStub(scopeId).introspectQuery(input.sql);
4538
+ const result = unwrapReply(await this.scopeStub(scopeId).introspectQueryReply(input.sql));
3975
4539
  // The statement is the logged argument: the access log is the evidence trail,
3976
4540
  // and for a console read the SQL is the whole story.
3977
4541
  await this.recordAccess(actor, 'queryScope', { tenantId, scopeId }, { sql: input.sql }, result.rows.length);
@@ -4064,7 +4628,7 @@ export class CloudflareScopeHost {
4064
4628
  if (!rec)
4065
4629
  throw substratError('not_found', `unknown scope ${scopeId} in tenant ${tenantId}`);
4066
4630
  if (rec.status !== 'archived') {
4067
- throw new Error(`scope ${scopeId} is ${rec.status}, not archived — only an archived scope may be reaped`);
4631
+ throw substratError('conflict', `scope ${scopeId} is ${rec.status}, not archived — only an archived scope may be reaped`);
4068
4632
  }
4069
4633
  // A serving scope always holds ≥1 bound hostname; a truly-dead one has been
4070
4634
  // unbound. The dashboard delete path unbinds AT archive, but a bare console
@@ -4076,7 +4640,7 @@ export class CloudflareScopeHost {
4076
4640
  // being released anyway; the interactive per-scope reap never sets it.
4077
4641
  const bound = opts?.force ? [] : await this.cp.listHostnames({ scopeId, limit: 1 });
4078
4642
  if (bound.length > 0) {
4079
- throw new Error(`scope ${scopeId} still resolves hostname '${bound[0].hostname}' — ` +
4643
+ throw substratError('conflict', `scope ${scopeId} still resolves hostname '${bound[0].hostname}' — ` +
4080
4644
  `unbind it before reaping (reap wipes storage and cannot be undone)`);
4081
4645
  }
4082
4646
  await this.scopeStub(scopeId).destroyStorage();
@@ -4274,7 +4838,7 @@ export class CloudflareScopeHost {
4274
4838
  },
4275
4839
  registerIdentityPool: async (actor, input) => {
4276
4840
  const parsed = identityPool.parse(input);
4277
- const created = await this.cp.registerIdentityPool(parsed.provider, parsed.topology, parsed.tenantId, new Date().toISOString());
4841
+ const created = await this.directory('registerIdentityPool', parsed.provider, parsed.topology, parsed.tenantId, new Date().toISOString());
4278
4842
  if (!created)
4279
4843
  return; // identical registration is idempotent, unaudited
4280
4844
  // Null tenant for a central pool: it belongs to no single tenant, which is
@@ -4291,9 +4855,9 @@ export class CloudflareScopeHost {
4291
4855
  listIdentityTenants: async (actor, provider, externalId) => {
4292
4856
  const r = await this.cp.readPool(provider);
4293
4857
  if (!r)
4294
- throw new Error(`identity pool '${provider}' is not registered`);
4858
+ throw substratError('not_found', `identity pool '${provider}' is not registered`);
4295
4859
  if (r.topology !== 'central') {
4296
- throw new Error(`identity pool '${provider}' is tenant-bound — enumerating tenants is only ` +
4860
+ throw substratError('forbidden', `identity pool '${provider}' is tenant-bound — enumerating tenants is only ` +
4297
4861
  `meaningful on a central pool, where the same externalId is the same person`);
4298
4862
  }
4299
4863
  const tenants = (await this.cp.identityTenants(provider, externalId));
@@ -4310,9 +4874,9 @@ export class CloudflareScopeHost {
4310
4874
  at: new Date().toISOString(),
4311
4875
  });
4312
4876
  if (topology === null)
4313
- throw new Error(`identity pool '${provider}' is not registered`);
4877
+ throw substratError('not_found', `identity pool '${provider}' is not registered`);
4314
4878
  if (topology !== 'central') {
4315
- throw new Error(`identity pool '${provider}' is tenant-bound — enumerating tenants is only ` +
4879
+ throw substratError('forbidden', `identity pool '${provider}' is tenant-bound — enumerating tenants is only ` +
4316
4880
  `meaningful on a central pool, where the same externalId is the same person`);
4317
4881
  }
4318
4882
  return memberships.map((m) => identityMembership.parse(m));
@@ -4465,14 +5029,14 @@ export class CloudflareScopeHost {
4465
5029
  const parsed = identityLink.parse(input);
4466
5030
  const pool = await this.cp.readPool(parsed.provider);
4467
5031
  if (!pool) {
4468
- throw new Error(`identity pool '${parsed.provider}' is not registered — a pool must declare ` +
5032
+ throw substratError('conflict', `identity pool '${parsed.provider}' is not registered — a pool must declare ` +
4469
5033
  `its topology before it may link (central vs tenant-bound decides whether ` +
4470
5034
  `the same externalId in two tenants is one person or two)`);
4471
5035
  }
4472
5036
  if (pool.topology === 'tenant-bound' && pool.tenant_id !== parsed.tenantId) {
4473
- throw new Error(`identity pool '${parsed.provider}' is bound to tenant ${pool.tenant_id} and cannot link into ${parsed.tenantId}`);
5037
+ throw substratError('conflict', `identity pool '${parsed.provider}' is bound to tenant ${pool.tenant_id} and cannot link into ${parsed.tenantId}`);
4474
5038
  }
4475
- const changed = await this.cp.linkIdentity(parsed.provider, parsed.externalId, parsed.principal, parsed.tenantId, parsed.scopeId ?? null, new Date().toISOString());
5039
+ const changed = await this.directory('linkIdentity', parsed.provider, parsed.externalId, parsed.principal, parsed.tenantId, parsed.scopeId ?? null, new Date().toISOString());
4476
5040
  // Idempotent: an identity already bound is a no-op, not audited.
4477
5041
  if (!changed)
4478
5042
  return;
@@ -4734,18 +5298,46 @@ export class CloudflareScopeHost {
4734
5298
  };
4735
5299
  }
4736
5300
  // -- helpers --------------------------------------------------------------
5301
+ /** A directory write whose refusal must keep its code across the DO hop (#113). */
5302
+ async directory(method, ...args) {
5303
+ return unwrapReply(await this.cp.reply(method, args));
5304
+ }
4737
5305
  /**
4738
5306
  * The getScope gate (control-plane.md §4.1/§4.2) — the pair check, then the tenant's and
4739
5307
  * the scope's lifecycle — thrown on THIS side of the RPC (#1718). The ControlPlaneDO
4740
5308
  * answers its refusal as data, because an error thrown there arrives here flattened, with
4741
- * no code; a record crosses intact. A CP-less host's null control plane answers nothing,
4742
- * so the gate passes: the router already made it from the shared directory.
5309
+ * no code; a record crosses intact.
5310
+ *
5311
+ * A CP-less host's null control plane answers nothing: the router made the pair check from
5312
+ * the shared directory. The scope's own storage then makes it again (#2016), against the
5313
+ * tenant it was provisioned for (`ScopeDO.admission`), so a vertical that passes a scope id
5314
+ * under the wrong tenant meets K-3's refusal at the door instead of relying on the permission
5315
+ * gate's grant data to deny it.
5316
+ *
5317
+ * Answers with the lifecycle the CP-less scope holds (#1713), read in the same DO call as the
5318
+ * pair check, so an `assertLive` door still costs one round-trip; a door that reads the scope
5319
+ * without the lifecycle half pays the one DO call the pair check needs. Null with a directory,
5320
+ * whose refusal already covered the lifecycle.
4743
5321
  */
4744
5322
  async validateScopeAccess(tenantId, scopeId) {
4745
5323
  const refusal = await this.cp.scopeAccessRefusal(tenantId, scopeId);
4746
- if (!refusal)
4747
- return;
4748
- throw refusal.code ? substratError(refusal.code, refusal.message) : new Error(refusal.message);
5324
+ if (refusal) {
5325
+ throw refusal.code
5326
+ ? substratError(refusal.code, refusal.message, refusal.reason ? { reason: refusal.reason } : undefined)
5327
+ : new Error(refusal.message);
5328
+ }
5329
+ if (!this.cpLess)
5330
+ return null;
5331
+ const { verdict, lifecycle } = await this.scopeStub(scopeId).admission(tenantId);
5332
+ if (verdict === 'foreign')
5333
+ throw unknownScopeForTenant(tenantId, scopeId);
5334
+ // A scope with no tenant record and no role rows has nothing to hold the pair against, so it is
5335
+ // let through — a compatibility state, not a steady one: the next reconcile (every push runs
5336
+ // one against the scopes behind it, #1172) or lifecycle delivery records the directory's tenant.
5337
+ // Logged on every admission, so an operator sees a scope that has not converged.
5338
+ if (verdict === 'unknown')
5339
+ console.warn(JSON.stringify(tenantUnrecordedLine(tenantId, scopeId)));
5340
+ return lifecycle;
4749
5341
  }
4750
5342
  /**
4751
5343
  * THE lifecycle gate (#1713) at every door that runs a scope's work: a request's stubs and
@@ -4763,16 +5355,9 @@ export class CloudflareScopeHost {
4763
5355
  * grants) keep `validateScopeAccess` alone: an operator still reads a suspended scope.
4764
5356
  */
4765
5357
  async assertLive(tenantId, scopeId) {
4766
- await this.validateScopeAccess(tenantId, scopeId);
4767
- const refusal = await this.cpLessRefusal(scopeId, { tenantId, scopeId });
5358
+ const refusal = lifecycleRefusal(await this.validateScopeAccess(tenantId, scopeId), { tenantId, scopeId });
4768
5359
  if (refusal)
4769
- throw new Error(refusal);
4770
- }
4771
- /** The delivered lifecycle's refusal on a CP-less host (#1713); always null with a directory. */
4772
- async cpLessRefusal(scopeId, ids) {
4773
- if (!this.cpLess)
4774
- return null;
4775
- return lifecycleRefusal(await this.scopeStub(scopeId).lifecycle(), ids);
5360
+ throw substratError('conflict', refusal, { reason: SCOPE_GATE_REASONS.notActive });
4776
5361
  }
4777
5362
  /**
4778
5363
  * Deliver the directory's lifecycle to the deployments serving the matching scopes (#1713):
@@ -4792,8 +5377,25 @@ export class CloudflareScopeHost {
4792
5377
  if (!delegation || this.cpLess)
4793
5378
  return report;
4794
5379
  let targets;
5380
+ // #2016: the heal also asks the served scopes whose deployment has not answered that it holds a
5381
+ // record of the scope's tenant, a copy as much as a primary (no push reconciles a copy), until
5382
+ // each answers that it does. A bounded slice per pass, after the drift and the holds.
5383
+ const askTenant = new Set();
4795
5384
  try {
4796
5385
  targets = await this.cp.lifecycleTargets(filter);
5386
+ if (filter.drift) {
5387
+ const unrecorded = await this.cp.lifecycleTargets({ unrecorded: true, limit: TENANT_UNRECORDED_PER_PASS });
5388
+ // Noted before asking, so an ask that fails (or a deployment that never answers) rotates
5389
+ // behind the scopes not asked yet, and the next pass reaches them.
5390
+ if (unrecorded.length > 0)
5391
+ await this.cp.recordTenantAsks(unrecorded.map((u) => u.scope_id), new Date().toISOString());
5392
+ const seen = new Set(targets.map((t) => t.scope_id));
5393
+ for (const u of unrecorded) {
5394
+ askTenant.add(u.scope_id);
5395
+ if (!seen.has(u.scope_id))
5396
+ targets.push(u);
5397
+ }
5398
+ }
4797
5399
  }
4798
5400
  catch (err) {
4799
5401
  console.error('substrat: could not read the scopes to deliver a lifecycle to (#1713)', err);
@@ -4815,8 +5417,10 @@ export class CloudflareScopeHost {
4815
5417
  // no lifecycle runs live. That keeps an activation, and every transition before a
4816
5418
  // deployment carries the route, from posting a delivery that changes nothing. After a
4817
5419
  // restore (epoch > 0) no receipt says nothing about what the deployment holds, so it is
4818
- // delivered. A HOLD is always delivered.
4819
- if (lifecycleRefusal(lifecycle) === null && (t.delivered === null ? t.epoch === 0 : t.delivered === lifecycleReceipt(lifecycle))) {
5420
+ // delivered. A HOLD is always delivered, and so is a scope the heal asks for its tenant record.
5421
+ if (!askTenant.has(scopeId) &&
5422
+ lifecycleRefusal(lifecycle) === null &&
5423
+ (t.delivered === null ? t.epoch === 0 : t.delivered === lifecycleReceipt(lifecycle))) {
4820
5424
  continue;
4821
5425
  }
4822
5426
  report.attempted += 1;
@@ -4874,7 +5478,7 @@ export class CloudflareScopeHost {
4874
5478
  }
4875
5479
  }
4876
5480
  // The receipt is what the scope HOLDS: a newer delivery it kept instead is the truth.
4877
- await this.cp.recordLifecycleReceipt(scopeId, lifecycleReceipt(answer.lifecycle), new Date().toISOString());
5481
+ await this.cp.recordLifecycleReceipt(scopeId, lifecycleReceipt(answer.lifecycle), new Date().toISOString(), answer.tenantRecorded === true);
4878
5482
  report.delivered += 1;
4879
5483
  }
4880
5484
  catch (err) {
@@ -4923,7 +5527,7 @@ export class CloudflareScopeHost {
4923
5527
  * points ask the directory themselves.
4924
5528
  */
4925
5529
  async lifecycleHeld(scopeId) {
4926
- return (await this.cpLessRefusal(scopeId)) !== null;
5530
+ return this.cpLess && lifecycleRefusal(await this.scopeStub(scopeId).lifecycle()) !== null;
4927
5531
  }
4928
5532
  /**
4929
5533
  * K-3's cross-check on its own: the (tenant, scope) pair must exist and agree before a
@@ -4933,7 +5537,7 @@ export class CloudflareScopeHost {
4933
5537
  async assertScope(tenantId, scopeId) {
4934
5538
  const rec = await this.cp.getScopeRecord(tenantId, scopeId);
4935
5539
  if (!rec)
4936
- throw substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
5540
+ throw unknownScopeForTenant(tenantId, scopeId);
4937
5541
  }
4938
5542
  /**
4939
5543
  * The same K-3 cross-check, for a read that then opens the scope's STORAGE — every
@@ -4948,9 +5552,9 @@ export class CloudflareScopeHost {
4948
5552
  async scopeRecordForRead(tenantId, scopeId) {
4949
5553
  const row = await this.cp.getScopeRecord(tenantId, scopeId);
4950
5554
  if (!row)
4951
- throw substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
5555
+ throw unknownScopeForTenant(tenantId, scopeId);
4952
5556
  if (row.status === 'reaped') {
4953
- throw new Error(`scope ${scopeId} is reaped — its storage is gone and cannot be read`);
5557
+ throw substratError('conflict', `scope ${scopeId} is reaped — its storage is gone and cannot be read`);
4954
5558
  }
4955
5559
  return row;
4956
5560
  }
@@ -4983,7 +5587,14 @@ export class CloudflareScopeHost {
4983
5587
  });
4984
5588
  }
4985
5589
  async recordAdmin(actor, action, target, before, after) {
4986
- await this.cp.recordAdmin({
5590
+ await this.cp.recordAdmin(this.adminEntry(actor, action, target, before, after));
5591
+ }
5592
+ /**
5593
+ * One admin-log row, minted here where attribution and `causedBy` live — for `recordAdmin`,
5594
+ * and for a ControlPlaneDO method that writes its row in the same unit as its effect (#1184).
5595
+ */
5596
+ adminEntry(actor, action, target, before, after) {
5597
+ return {
4987
5598
  id: ulid(),
4988
5599
  actor,
4989
5600
  action,
@@ -4995,7 +5606,7 @@ export class CloudflareScopeHost {
4995
5606
  before: before ?? null,
4996
5607
  after: after ?? null,
4997
5608
  at: new Date().toISOString(),
4998
- });
5609
+ };
4999
5610
  }
5000
5611
  /**
5001
5612
  * The scope's own answer to "what may this connection do here" (#726 gap 1).
@@ -5120,6 +5731,11 @@ export class CloudflareScopeHost {
5120
5731
  // rewind CLAIMS them on `SWITCH_HOLDS_NAME`, which the rewind cannot reach. A module is held
5121
5732
  // while any claim on it exists, and `runDueSchedules` skips a held module.
5122
5733
  //
5734
+ // #2029: the same for a PEER switched off (`vertical:<slug>`): the rewind claims the scope's
5735
+ // off peers beside its off modules, each under its hold key (`holdKeyOf`), and the peer door
5736
+ // refuses a held peer as the system door refuses a held module. Everything below holds for
5737
+ // both kinds; "module" reads as "subject".
5738
+ //
5123
5739
  // A claim is released only by a switch move KNOWN to survive its rewind (`switchInScope`): a
5124
5740
  // move on the restored storage, after the rewind armed. A move during the settle, or one the
5125
5741
  // doomed instance applied, lands in storage the restart discards, so it releases nothing. A
@@ -5138,7 +5754,7 @@ export class CloudflareScopeHost {
5138
5754
  * reopens only the gap this closes, for a rewind that lands during the outage, and the next
5139
5755
  * reconcile still switches that module off. The error is answered, for the pass to report.
5140
5756
  */
5141
- async switchHeld(scopeId, moduleId) {
5757
+ async switchHeld(scopeId, holdKey) {
5142
5758
  const at = Date.now();
5143
5759
  let error;
5144
5760
  if (!this.holdSnapshot || at - this.holdSnapshot.at > SWITCH_HOLD_SNAPSHOT_MS) {
@@ -5161,8 +5777,11 @@ export class CloudflareScopeHost {
5161
5777
  }
5162
5778
  }
5163
5779
  }
5164
- const held = this.holdSnapshot?.held.has(`${scopeId} ${moduleId}`) ?? false;
5165
- return { held, ...(error ? { error } : {}) };
5780
+ const snapshot = this.holdSnapshot;
5781
+ const held = snapshot?.held.has(`${scopeId} ${holdKey}`) ?? false;
5782
+ // #2029: a good read naming no row for this scope at all, for `doorGate`'s instance cache.
5783
+ const scopeClear = !error && snapshot?.good === true && !snapshot.scopes.has(scopeId);
5784
+ return { held, ...(error ? { error } : {}), ...(scopeClear ? { scopeClear: true } : {}) };
5166
5785
  }
5167
5786
  /** One read of the whole hold into the snapshot, stamped with when it went out. Never throws. */
5168
5787
  async refreshHolds(at) {
@@ -5170,7 +5789,12 @@ export class CloudflareScopeHost {
5170
5789
  const rows = await this.switchHoldsStub().switchHoldsAll();
5171
5790
  // A slow read that lands after a newer one never replaces it.
5172
5791
  if (!this.holdSnapshot || at >= this.holdSnapshot.at) {
5173
- this.holdSnapshot = { at, held: new Set(rows.map((r) => `${r.scopeId} ${r.moduleId}`)) };
5792
+ this.holdSnapshot = {
5793
+ at,
5794
+ held: new Set(rows.map((r) => `${r.scopeId} ${r.moduleId}`)),
5795
+ scopes: new Set(rows.map((r) => r.scopeId)),
5796
+ good: true,
5797
+ };
5174
5798
  }
5175
5799
  return {};
5176
5800
  }
@@ -5181,8 +5805,10 @@ export class CloudflareScopeHost {
5181
5805
  // than hammered per scope. For that long, other scopes reuse this possibly pre-hold set
5182
5806
  // silently, and only the consult that sent this read reports the error: the declared
5183
5807
  // fail-open, attributed once.
5184
- if (!this.holdSnapshot || at >= this.holdSnapshot.at)
5185
- this.holdSnapshot = { at, held: known ?? new Set() };
5808
+ if (!this.holdSnapshot || at >= this.holdSnapshot.at) {
5809
+ const scopes = new Set([...(known ?? [])].map((k) => k.slice(0, k.indexOf(' '))));
5810
+ this.holdSnapshot = { at, held: known ?? new Set(), scopes, good: false };
5811
+ }
5186
5812
  return {
5187
5813
  error: known
5188
5814
  ? `switch hold unreadable (${why}); the ${known.size} hold(s) from this pass's last good read ` +
@@ -5229,7 +5855,11 @@ export class CloudflareScopeHost {
5229
5855
  * its claims before the OFF's join lands. The joined row then holds the module the operator just
5230
5856
  * turned ON, until the next surviving move of that switch releases it (another ON does).
5231
5857
  */
5232
- async switchInScope(scopeId, moduleId, to, at) {
5858
+ async switchInScope(kind, scopeId, key, to, at, tenantHeld = false,
5859
+ /** #2045: the switch call's fence; see `SWITCH_FENCES_DDL`. */
5860
+ fence) {
5861
+ // #2029: a peer's claims live beside the modules', under its own hold key.
5862
+ const holdKey = holdKeyOf(kind, key);
5233
5863
  let scopeClaims;
5234
5864
  let readError;
5235
5865
  try {
@@ -5238,23 +5868,30 @@ export class CloudflareScopeHost {
5238
5868
  catch (err) {
5239
5869
  readError = err;
5240
5870
  }
5241
- const { instance, ...outcome } = await this.scopeStub(scopeId).switchSystemSchedules(moduleId, scopeId, to, at);
5871
+ const stub = this.scopeStub(scopeId);
5872
+ const { instance, ...outcome } = await (kind === 'system'
5873
+ ? stub.switchSystemSchedules(key, scopeId, to, at, tenantHeld, fence)
5874
+ : stub.switchPeer(key, scopeId, to, at, tenantHeld, fence));
5242
5875
  if (!scopeClaims)
5243
5876
  throw readError;
5877
+ // #2045: a move the fence refused wrote nothing, so it is known to survive nothing and to
5878
+ // release nothing; the newer call that superseded it did whatever the hold needed.
5879
+ if (outcome.superseded)
5880
+ return outcome;
5244
5881
  const now = Date.now();
5245
5882
  const pendingLive = (c) => c.state === 'pending' && now - Date.parse(c.heldAt) <= SWITCH_HOLD_PENDING_MAX_MS;
5246
- const before = scopeClaims.filter((c) => c.moduleId === moduleId);
5883
+ const before = scopeClaims.filter((c) => c.moduleId === holdKey);
5247
5884
  if (to === 'on') {
5248
5885
  // Every ON, claims or not: its tombstone is what keeps a rewind's stale read from claiming
5249
5886
  // the module after this (#1839).
5250
5887
  this.holdSnapshot = null;
5251
- await this.switchHoldsStub().switchHoldOn(scopeId, moduleId, before.map((c) => c.claimId));
5888
+ await this.switchHoldsStub().switchHoldOn(scopeId, holdKey, before.map((c) => c.claimId));
5252
5889
  }
5253
5890
  else if (outcome.held && before.length > 0) {
5254
5891
  const survived = before.filter((c) => !pendingLive(c) && c.doomed !== instance);
5255
5892
  if (survived.length > 0) {
5256
5893
  this.holdSnapshot = null;
5257
- await this.switchHoldsStub().switchHoldRelease(scopeId, moduleId, survived.map((c) => c.claimId));
5894
+ await this.switchHoldsStub().switchHoldRelease(scopeId, holdKey, survived.map((c) => c.claimId));
5258
5895
  }
5259
5896
  }
5260
5897
  if (to === 'off' && outcome.changed) {
@@ -5265,7 +5902,7 @@ export class CloudflareScopeHost {
5265
5902
  const joins = new Set([...scopeClaims, ...after].filter((c) => pendingLive(c) || c.doomed === instance).map((c) => c.claimId));
5266
5903
  if (joins.size > 0) {
5267
5904
  this.holdSnapshot = null;
5268
- await this.switchHoldsStub().switchHoldJoin(scopeId, moduleId, [...joins]);
5905
+ await this.switchHoldsStub().switchHoldJoin(scopeId, holdKey, [...joins]);
5269
5906
  }
5270
5907
  }
5271
5908
  return outcome;
@@ -5329,11 +5966,16 @@ export class CloudflareScopeHost {
5329
5966
  await holds.switchHoldArm(scopeId, claimId, result.instance ?? null).catch(() => undefined);
5330
5967
  return { rewindingTo: result.rewindingTo };
5331
5968
  }
5332
- /** The modules this scope's own storage has switched off, as a rewind reads them before it arms. */
5969
+ /**
5970
+ * The subjects this scope's own storage has switched off, as a rewind reads them before it arms:
5971
+ * its modules, and (#2029) its peers, each by its hold key (`holdKeyOf`).
5972
+ */
5333
5973
  async offInStorage(scopeId) {
5334
- return (await this.systemGrantsStatusLocal(scopeId))
5335
- .filter((e) => e.schedules === 'off')
5336
- .map((e) => e.moduleId);
5974
+ const [modules, peers] = await Promise.all([this.systemGrantsStatusLocal(scopeId), this.peerGrantsStatusLocal(scopeId)]);
5975
+ return [
5976
+ ...modules.filter((e) => e.schedules === 'off').map((e) => holdKeyOf('system', e.moduleId)),
5977
+ ...peers.filter((e) => e.calls === 'off').map((e) => holdKeyOf('peer', e.vertical)),
5978
+ ];
5337
5979
  }
5338
5980
  /**
5339
5981
  * #1839: wait until every row of this rewind's claim is `SWITCH_HOLD_SETTLE_MS` old. The settle
@@ -5402,7 +6044,11 @@ export class CloudflareScopeHost {
5402
6044
  * arrive as a different class of answer just because the caller asked for a socket.
5403
6045
  */
5404
6046
  liveReads = {
5405
- subscribe: async ({ tenantId, scopeId, principal, request }) => {
6047
+ subscribe: async ({ tenantId, scopeId, principal, request, within }) => {
6048
+ // Decided before anything else, and thrown: a `within` that is neither a plain
6049
+ // EntityRef nor a value `vouchedWithin` built is a caller bug, and the two ways of
6050
+ // guessing at it both open a feed wider than was asked for (#1853).
6051
+ const narrowed = liveWithinOf(within);
5406
6052
  if (!isUpgradeRequest(request)) {
5407
6053
  return new Response('live reads are a WebSocket surface', {
5408
6054
  status: 426,
@@ -5460,6 +6106,10 @@ export class CloudflareScopeHost {
5460
6106
  headers.set(LIVE_PRINCIPAL_HEADER, principal);
5461
6107
  headers.set(LIVE_TENANT_HEADER, tenantId);
5462
6108
  headers.set(LIVE_SCOPE_HEADER, scopeId);
6109
+ if (narrowed)
6110
+ headers.set(LIVE_WITHIN_HEADER, encodeLiveWithin(narrowed));
6111
+ else
6112
+ headers.delete(LIVE_WITHIN_HEADER);
5463
6113
  const forwarded = new Request(new URL(LIVE_SUBSCRIBE_PATH, 'https://scope.substrat.internal'),
5464
6114
  // `new Request(url, { …, headers })` rather than `new Request(request, …)`:
5465
6115
  // the DO is addressed by its own path, not the client's, and the upgrade
@@ -5538,10 +6188,10 @@ export class CloudflareScopeHost {
5538
6188
  // good. Read first, so a pair the directory does not hold writes nothing.
5539
6189
  const scope = await this.cp.getScopeRecord(tenantId, scopeId);
5540
6190
  if (!scope)
5541
- throw substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
6191
+ throw unknownScopeForTenant(tenantId, scopeId);
5542
6192
  const { roles, tuples, entitlements, identities } = await this.tenantProjection(tenantId);
5543
6193
  const connectionKeys = await this.connectionKeyRows(tenantId, scope.vertical);
5544
- await this.scopeStub(scopeId).applyProjection(tenantId, roles, tuples, entitlements, undefined, identities, connectionKeys);
6194
+ unwrapReply(await this.scopeStub(scopeId).applyProjectionReply(tenantId, roles, tuples, entitlements, undefined, identities, connectionKeys));
5545
6195
  }
5546
6196
  /**
5547
6197
  * Fan the tenant's current state out into ALL its scopes — called after any
@@ -5590,7 +6240,7 @@ export class CloudflareScopeHost {
5590
6240
  if (!keysByVertical.has(vertical)) {
5591
6241
  keysByVertical.set(vertical, this.connectionKeyRows(tenantId, s.vertical));
5592
6242
  }
5593
- await this.scopeStub(s.scope_id).applyProjection(tenantId, roles, tuples, entitlements, undefined, identities, await keysByVertical.get(vertical));
6243
+ unwrapReply(await this.scopeStub(s.scope_id).applyProjectionReply(tenantId, roles, tuples, entitlements, undefined, identities, await keysByVertical.get(vertical)));
5594
6244
  }));
5595
6245
  const failed = settled.flatMap((r, i) => (r.status === 'rejected' ? [{ scope: scopes[i].scope_id, reason: r.reason }] : []));
5596
6246
  if (failed.length === 0)
@@ -5620,10 +6270,11 @@ export class CloudflareScopeHost {
5620
6270
  * from its own storage. No tenant-level tuples, no control plane.
5621
6271
  */
5622
6272
  async provisionScopeLocal(input) {
6273
+ const carry = recordedOffFromWire(input);
5623
6274
  const services = await this.servicePrincipals?.(input.tenantId, input.scopeId);
5624
6275
  const stub = this.scopeStub(input.scopeId);
5625
6276
  await this.migrateAndRecord(input.scopeId); // create the module tables (setMigrationState no-ops on a null CP)
5626
- const switchedOff = await stub.applyProjection(input.tenantId, input.roles.map((r) => ({ role_key: r.key, permissions: JSON.stringify(r.permissions), source: r.source })), [], // no tenant-level tuples — a CP-less vertical grants at scope level only
6277
+ const switchedOff = unwrapReply(await stub.applyProjectionReply(input.tenantId, input.roles.map((r) => ({ role_key: r.key, permissions: JSON.stringify(r.permissions), source: r.source })), [], // no tenant-level tuples — a CP-less vertical grants at scope level only
5627
6278
  // Only PROJECT (and thereby switch on strict enforcement) when the platform actually
5628
6279
  // supplies entitlements. Omitting them leaves the scope un-projected and trusting-
5629
6280
  // upstream, so a vertical whose provision path predates #304 is never denied — it
@@ -5713,12 +6364,10 @@ export class CloudflareScopeHost {
5713
6364
  public_key: k.publicKey,
5714
6365
  }))
5715
6366
  : undefined,
5716
- // #1742: the recorded-off modules, switched off after the seat in the same unit, on the
5717
- // scope this call provisions — the one the seat's tuples name.
5718
- input.switchedOff
5719
- ? { scopeId: input.scopeId, moduleIds: input.switchedOff, at: new Date().toISOString() }
5720
- : undefined, services?.map((id) => `principal:${id}`));
5721
- return input.switchedOff ? { switchedOff } : {};
6367
+ // #1742: the recorded-off modules and (#2029) peers, switched off after the seat in the
6368
+ // same unit, on the scope this call provisions — the one the seat's tuples name.
6369
+ carry ? { scopeId: input.scopeId, at: new Date().toISOString(), ...carry } : undefined, services?.map((id) => `principal:${id}`)));
6370
+ return carry ? { switchedOff } : {};
5722
6371
  }
5723
6372
  /**
5724
6373
  * Resolve an external identity from a scope's PROJECTED links (#406) — the CP-less
@@ -5857,6 +6506,8 @@ export class CloudflareScopeHost {
5857
6506
  async connectorGrantLocal(connectionId, scopeId, permission, expiresAt) {
5858
6507
  await this.writeScopeTuple(scopeId, subjectRef({ kind: 'connection', id: connectionId }), `granted:${permission}`, `scope:${scopeId}`, expiresAt ?? null);
5859
6508
  }
6509
+ /** #2045 (Codex r3): every switch mover here honours the fence — what `/internal/switch-fence` reports. */
6510
+ switchFenced = true;
5860
6511
  /**
5861
6512
  * The far end of the schedule kill switch for a scope served HERE (#1666):
5862
6513
  * `/internal/system-switch` lands on this, from the shared control plane's
@@ -5865,11 +6516,11 @@ export class CloudflareScopeHost {
5865
6516
  * admin log and writes the row once this returns. `held: false` is an answer, not a
5866
6517
  * throw — see `systemSwitchOutcome`.
5867
6518
  */
5868
- async systemSwitchLocal(scopeId, moduleId, to) {
6519
+ async systemSwitchLocal(scopeId, moduleId, to, opts) {
5869
6520
  // Parsed on the way out: this is the wire answer the platform reads, and the DO's
5870
6521
  // plain strings become the published shape here rather than on trust.
5871
6522
  // #1819: `switchInScope` releases a rewind's hold on the module once the switch lands.
5872
- return systemSwitchOutcome.parse(await this.switchInScope(scopeId, moduleId, to, new Date().toISOString()));
6523
+ return systemSwitchOutcome.parse(await this.switchInScope('system', scopeId, moduleId, to, new Date().toISOString(), opts?.tenantHeld ?? false, opts?.fence));
5873
6524
  }
5874
6525
  /**
5875
6526
  * The far end of the status read for a scope served HERE (#1674): `/internal/system-grants`
@@ -5904,9 +6555,12 @@ export class CloudflareScopeHost {
5904
6555
  * Move one peer's kill switch on a scope served HERE (#1706) — the far end of
5905
6556
  * `revokeFromPeer` / `restoreToPeer` for a hosted scope. Nothing is audited here; the
5906
6557
  * platform writes the admin rows around this call, as for the schedule switch.
6558
+ * #2029: through `switchInScope`, so the move releases a rewind's hold on the peer once it
6559
+ * lands, as a module's does. #2030: `tenantHeld`, the platform's word that the peer holds a
6560
+ * live tenant-level grant, which this deployment has no directory to read.
5907
6561
  */
5908
- async peerSwitchLocal(scopeId, vertical, to) {
5909
- return peerSwitchOutcome.parse(await this.scopeStub(scopeId).switchPeer(verticalSlug.parse(vertical), scopeId, to, new Date().toISOString()));
6562
+ async peerSwitchLocal(scopeId, vertical, to, opts) {
6563
+ return peerSwitchOutcome.parse(await this.switchInScope('peer', scopeId, verticalSlug.parse(vertical), to, new Date().toISOString(), opts?.tenantHeld ?? false, opts?.fence));
5910
6564
  }
5911
6565
  // -- the cross-vertical far ends (#1705 PR 2) --------------------------------
5912
6566
  // The shared control plane runs the cross-vertical phase for every hosted edge and reaches the
@@ -5920,7 +6574,25 @@ export class CloudflareScopeHost {
5920
6574
  async exportedEventsLocal(tenantId, scopeId, raw) {
5921
6575
  const input = exportReadInput.parse(raw);
5922
6576
  await this.assertServesLocally(tenantId, scopeId, 'readExportedEvents');
5923
- return exportedBatch.parse(await this.scopeStub(scopeId).exportedEventsRead(input, tenantId, scopeId));
6577
+ return this.readExportsThroughDoor(input, tenantId, scopeId);
6578
+ }
6579
+ /**
6580
+ * #2029: a producer's export read, through the CONSUMER's peer door: the consumer's grants on
6581
+ * this scope decide what leaves it, so a consumer the rewind hold keeps off must hold nothing
6582
+ * here, exactly as a switched-off one does. Gated and pinned like every door call; held, the
6583
+ * producer answers every key missing, so the edge pauses rather than reading past the hold.
6584
+ */
6585
+ async readExportsThroughDoor(input, tenantId, scopeId) {
6586
+ const stub = this.scopeStub(scopeId);
6587
+ const door = await this.openDoor('peer', input.consumer, scopeId);
6588
+ try {
6589
+ return exportedBatch.parse(await door.through((instance) => stub.exportedEventsRead(input, tenantId, scopeId, { instance })));
6590
+ }
6591
+ catch (err) {
6592
+ if (!this.isHeldRefusal(err))
6593
+ throw err;
6594
+ return exportedBatch.parse(await stub.exportedEventsRead(input, tenantId, scopeId, { held: true }));
6595
+ }
5924
6596
  }
5925
6597
  /**
5926
6598
  * The consumer's imports and watermarks (#1705), for a scope served HERE. The served-here
@@ -5970,7 +6642,7 @@ export class CloudflareScopeHost {
5970
6642
  }
5971
6643
  const rec = await this.cp.getScopeRecord(tenantId, scopeId);
5972
6644
  if (!rec)
5973
- throw substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
6645
+ throw unknownScopeForTenant(tenantId, scopeId);
5974
6646
  const source = await importCursorSourceOf((t, v) => this.admin.resolveVerticalInstance(t, v), { tenantId, scopeId, vertical: rec.vertical }, move.from);
5975
6647
  // Where the write lands is the peer switch's rule: a scope bound to a vertical is served by
5976
6648
  // that vertical's deployment, and its watermark lives there. Without a delegation that