@substrat-run/adapter-cloudflare 0.137.0 → 0.139.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/host.js CHANGED
@@ -1,13 +1,14 @@
1
1
  import { env as ambientEnv } from 'cloudflare:workers';
2
2
  import { isRewindRefusal, REWIND_REFUSED } from './rewind-refusal.js';
3
3
  import { isSystemDoorMoved, SYSTEM_DOOR_MOVED, SYSTEM_DOOR_REGATES } from './system-door.js';
4
- import { delegatedReadParams, fromWireFailure, exportReadInput, exportedBatch, importBatch, importResult, importState, importsOfManifestJson, exportsOfManifestJson, importCursorMove, importCursorMoveAt, importCursorMoved, accessLogEntry, adminLogEntry, opsFailureEntry, opsFailureFingerprint, issueEntry, sweepRunEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunsPayload, modelUsageEntry, attachmentRecord, createTenantInput, identityLink, identityPool, createOrgInput, promotionAcknowledgement, bindAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, registerVerticalInput, vertical as verticalSchema, verticalServingState, verticalChannel, verticalVersion, connection, capabilityExchange, mintedCapability, capabilityGrant, principalId, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, systemSwitch, systemSwitchOutcome, systemScheduleEntry, systemSwitchRecord, entitlementGrant, entitlementGrantInput, capabilityFilter, instant, meterReading, subjectRef, createConnectionInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, org as orgSchema, orgMembership, resolvedIdentity, identityMembership, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, delegatedReadRecord, ownerTransferAudit, copyMarkAudit, subjectShredReceipt, connectorDispatchKind, callsOfManifestJson, outboundOfManifestJson, substratError, redrainEventsInput, coverage, peerCoverage, peerGrantsEntry, peerSwitch, peerSwitchOutcome, verticalCaller, verticalResolution, verticalSlug, entityObjectRef, ATTACHMENT_BLOB_BINDING, blobStoreBindingName, lifecycleDelivery, scopeLifecycle, } from '@substrat-run/contracts';
4
+ import { delegatedReadParams, operationSeriesCount, fromWireFailure, exportReadInput, exportedBatch, importBatch, importResult, importState, importsOfManifestJson, exportsOfManifestJson, importCursorMove, importCursorMoveAt, importCursorMoved, accessLogEntry, adminLogEntry, opsFailureEntry, opsFailureFingerprint, issueEntry, findingEntry, findingRuleEntry, sweepRunEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunsPayload, modelUsageEntry, attachmentRecord, createTenantInput, identityLink, identityPool, createOrgInput, promotionAcknowledgement, bindAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, registerVerticalInput, vertical as verticalSchema, verticalServingState, verticalChannel, verticalVersion, connection, capabilityExchange, mintedCapability, capabilityGrant, principalId, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, systemSwitch, systemSwitchOutcome, systemScheduleEntry, systemSwitchRecord, entitlementGrant, entitlementGrantInput, capabilityFilter, instant, meterReading, subjectRef, createConnectionInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, org as orgSchema, orgMembership, resolvedIdentity, identityMembership, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, delegatedReadRecord, ownerTransferAudit, memberChangeAudit, copyMarkAudit, subjectShredReceipt, connectorDispatchKind, callsOfManifestJson, outboundOfManifestJson, SCOPE_GATE_REASONS, substratError, redrainEventsInput, coverage, peerCoverage, peerGrantsEntry, peerSwitch, peerSwitchOutcome, verticalCaller, verticalResolution, verticalSlug, entityObjectRef, ATTACHMENT_BLOB_BINDING, blobStoreBindingName, lifecycleDelivery, scopeLifecycle, } from '@substrat-run/contracts';
5
5
  import { normalizeHostname, toRouteTarget } from './route-resolver.js';
6
- import { attachmentBlobKey, attachmentSha256, entitlementDenial, requiredEntitlementFor, foldMeterReading, parseValidationRecords, resolveScopeRecord, ulid, KEPT_COPY_REFUSAL, capabilityTokenHash, checkBecomeInput, plausibleSessionToken, foldModelUsage, backoffAt, resolveRetryPolicy, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, openSealed, ImpersonationRefused, assertSessionUsable, impersonationRowValues, mapImpersonationRow, newImpersonationSession, jobRunOf, runDueJobRuns, SYSTEM_DOOR_WAIT, startJobRun, attachmentTextJob, assertAttachmentExtractors, assertJobRegistrable, isAttachmentTextRun, searchLimit, searchMatchExpression, globalFetch, assertRedrainWindow, platformRequestOf, undrainedEventsOf, inUnitMovesToAudit, staleCarryRevertRow, staleCarryReverts, withRecorded, systemSwitchedOffMessage, tenantSystemSwitchedOffMessage, CrossVerticalRegistry, importCursorSourceOf, exportBreaksOf, exportBreakRefusal, bindExportBreaksOf, bindExportBreakRefusal, collectPeers, peerSeats, connectorCallRecord, noopConnectorCallRecorder, recordConnectorCall, settleConnectionUse, splitManifestMigrations, unknownRoleError, assertRowLimit, assertRowOffset, INERT_SCOPE_REASON, lifecycleReceipt, lifecycleRefusal, isPrimaryScope, isPrimaryScopeRow, asyncInvocationId, asyncLinePass, } from '@substrat-run/kernel';
7
- import { attributedHost } from '@substrat-run/kernel';
8
- import { isOrangeToOrange, isUpgradeRequest, LIVE_MODE_HEADER, LIVE_PRINCIPAL_HEADER, LIVE_SCOPE_HEADER, LIVE_SUBSCRIBE_PATH, LIVE_TENANT_HEADER, } from './live-reads.js';
6
+ import { attachmentBlobKey, attachmentSha256, entitlementDenial, requiredEntitlementFor, foldMeterReading, parseValidationRecords, resolveScopeRecord, ulid, KEPT_COPY_REFUSAL, capabilityTokenHash, checkBecomeInput, plausibleSessionToken, foldModelUsage, backoffAt, resolveRetryPolicy, executorOutcomeOf, isDeliveryRefusal, refusalJournalText, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, openSealed, ImpersonationRefused, assertSessionUsable, impersonationRowValues, mapImpersonationRow, newImpersonationSession, jobRunOf, assertLeaseMs, runDueJobRuns, SYSTEM_DOOR_WAIT, startJobRun, attachmentTextJob, assertAttachmentExtractors, resolveAttachmentTextBounds, assertJobRegistrable, ATTACHMENT_TEXT_BACKFILL_JOB, ATTACHMENT_TEXT_JOB, kernelJobFor, attachmentTextBackfillJob, searchLimit, searchMatchExpression, globalFetch, assertRedrainWindow, platformRequestOf, undrainedEventsOf, inUnitMovesToAudit, staleCarryRevertRow, staleCarryReverts, recordedOffFromWire, peerSubjectRef, SWITCH_KINDS, reassertActionOf, reassertEntry, reassertOffRow, reassertOnRow, switchActionOf, switchAuditSubject, switchNotFoundMessage, recordWriteSuperseded, switchSubjectOf, switchSupersededMessage, withRecorded, systemSwitchedOffMessage, tenantSystemSwitchedOffMessage, CrossVerticalRegistry, importCursorSourceOf, exportBreaksOf, exportBreakRefusal, bindExportBreaksOf, bindExportBreakRefusal, collectPeers, peerSeats, connectorCallRecord, noopConnectorCallRecorder, recordConnectorCall, settleConnectionUse, splitManifestMigrations, unknownRoleError, assertRowLimit, assertRowOffset, emptyImportResult, INERT_SCOPE_REASON, lifecycleReceipt, lifecycleRefusal, isPrimaryScope, isPrimaryScopeRow, asyncInvocationId, asyncLinePass, memberAddedAudit, } from '@substrat-run/kernel';
7
+ import { attributedView } from '@substrat-run/kernel';
8
+ import { isOrangeToOrange, isUpgradeRequest, LIVE_MODE_HEADER, LIVE_PRINCIPAL_HEADER, LIVE_SCOPE_HEADER, LIVE_SUBSCRIBE_PATH, LIVE_TENANT_HEADER, LIVE_WITHIN_HEADER, encodeLiveWithin, liveWithinOf, } from './live-reads.js';
9
9
  import { tenantStoreDatabaseName } from './d1.js';
10
10
  import { blobStoreBucketName, r2TenantBlobStore } from './r2.js';
11
+ import { unwrapReply } from './do-reply.js';
11
12
  /** DO row → contract shape. Never reads the secrets table — that is the split. */
12
13
  const toConnection = (r) => connection.parse({
13
14
  id: r.id,
@@ -26,12 +27,6 @@ const toConnection = (r) => connection.parse({
26
27
  createdAt: r.created_at,
27
28
  revokedAt: r.revoked_at,
28
29
  });
29
- /** Rethrow a capability attachment verb's failure, rebuilt with its code; else its value. */
30
- function unwrapCapabilityReply(reply) {
31
- if (reply.failure)
32
- throw fromWireFailure(reply.failure);
33
- return reply.value;
34
- }
35
30
  /**
36
31
  * How far ahead of the clock a held lifecycle epoch may be and still be learned past (#1713). An
37
32
  * epoch is the time a directory restore minted it, so a legitimate one is behind its minter's
@@ -92,24 +87,31 @@ function nullControlPlane() {
92
87
  recordAdmin: noop,
93
88
  recordAccess: noop,
94
89
  tenantHoldsEntitlement: async () => true,
95
- // #1674: the schedule switch's record is a DIRECTORY store, and a CP-less host has no
90
+ // #1674, #2029: the switches' record is a DIRECTORY store, and a CP-less host has no
96
91
  // directory — the shared control plane that delegates here keeps it. Same posture as
97
92
  // `recordAdmin`: nothing to write, nothing recorded to read back.
98
- recordSystemSwitchedOff: async () => null,
99
- recordSystemSwitchedOn: async () => null,
100
- restoreSystemSwitchRecord: noop,
93
+ recordSwitchedOff: async () => null,
94
+ recordSwitchedOn: async () => null,
95
+ restoreSwitchRecord: noop,
101
96
  listSystemSwitches: async () => [],
102
- systemSwitchRecordsOf: async () => [],
103
- switchedOffModulesOf: async () => [],
97
+ switchRecordsOf: async () => [],
98
+ switchRecordStatesOf: async () => [],
99
+ switchesOwedOf: async () => [],
100
+ clearSwitchOwed: noop,
104
101
  // #1823: no tenant tuples here either — the platform reads them and sends `tenantHeld`.
105
- tenantHeldSystemModules: async () => [],
102
+ tenantHeldOf: async () => [],
103
+ };
104
+ const unavailable = (method) => new Error(`control plane unavailable: '${method}' — this host is scope-local / CP-less ` +
105
+ `(docs/architecture/scope-local-permissions.md, Phase 3)`);
106
+ // A directory write through `reply` (#113) is named for the write it carries, not for `reply`.
107
+ passthrough.reply = async (method) => {
108
+ throw unavailable(String(method));
106
109
  };
107
110
  return new Proxy({}, {
108
111
  get: (_t, prop) => typeof prop === 'string' && prop in passthrough
109
112
  ? passthrough[prop]
110
113
  : async () => {
111
- throw new Error(`control plane unavailable: '${String(prop)}' — this host is scope-local / CP-less ` +
112
- `(docs/architecture/scope-local-permissions.md, Phase 3)`);
114
+ throw unavailable(String(prop));
113
115
  },
114
116
  });
115
117
  }
@@ -151,19 +153,34 @@ export const SWITCH_HOLD_EXTRA_WAITS = 3;
151
153
  */
152
154
  export const SWITCH_HOLD_PENDING_MAX_MS = 5 * 60_000;
153
155
  /**
154
- * An OFF of a tenant-held module whose far end did not attest `deniesTenantGrants` (#1823): it
155
- * switched, but its evaluator still authorizes the module's tenant-level grant on the scope.
156
- * Carries the far end's outcome, so the operator's OFF knows whether there is a move to put back.
156
+ * A move the deployment serving the scope applied, or may have, without attesting what the
157
+ * platform needs from it (#2045 Codex r3): the switch fence (`fenced`), or for a tenant-held
158
+ * module's OFF the evaluator that denies a tenant-level grant (`deniesTenantGrants`, #1823).
159
+ *
160
+ * The preflight (`switchTarget`'s `attestFence`) tells a deployment built before the fence apart
161
+ * before anything is recorded or moved. On such a deployment only an ON is refused: the race the
162
+ * fence closes is an older OFF landing after a newer ON, and with every ON refused there is no
163
+ * newer ON to land after, so an OFF goes through unfenced (#2045 follow-up). A kill switch that
164
+ * cannot be pulled until every vertical is re-pushed is the outage the fence must never cause.
165
+ * An unfenced OFF keeps its write-ahead owed mark, so the first re-assert after the vertical is
166
+ * redeployed moves the scope again under the record's fence.
167
+ *
168
+ * This error fires when a deployment that attested the fence answers a move without it — a
169
+ * rollback to an old build between the preflight and the move — or when an ON reaches a
170
+ * deployment without the fence. It is NOT compensated with an opposite move: an unfenced
171
+ * compensation can undo a newer call's switch (Codex r3, finding 2). The call's record and its
172
+ * write-ahead owed mark stay, and the call is refused.
157
173
  */
158
- class UnattestedSwitchOff extends Error {
159
- outcome;
160
- constructor(scopeId, moduleId, outcome) {
161
- super(`the deployment serving scope ${scopeId} predates the kill switch's tenant-grant denial (#1823): ` +
162
- `module '${moduleId}' holds a tenant-level grant that its OFF would leave authorizing there. ` +
163
- `Redeploy the vertical, then retry.`);
164
- this.outcome = outcome;
165
- }
174
+ class UnattestedSwitch extends Error {
166
175
  }
176
+ /** A fenced move answered without `fenced`: the deployment predates the fence. */
177
+ const unfencedMessage = (scopeId) => `the deployment serving scope ${scopeId} predates the switch fence (#2045): it cannot refuse an older ` +
178
+ `switch call's OFF landing after this ON, so no switch is turned on there. Switching off still works. ` +
179
+ `Redeploy the vertical, then retry.`;
180
+ /** A tenant-held module's OFF answered without `deniesTenantGrants` (#1823). */
181
+ const undeniedMessage = (scopeId, moduleId) => `the deployment serving scope ${scopeId} predates the kill switch's tenant-grant denial (#1823): ` +
182
+ `module '${moduleId}' holds a tenant-level grant that its OFF would leave authorizing there. ` +
183
+ `Redeploy the vertical, then retry.`;
167
184
  /**
168
185
  * The attachment bucket a host resolves when it was handed no resolver (#1995): the
169
186
  * platform's per-tenant binding, read off this script's own env, named by the one shared
@@ -172,6 +189,49 @@ class UnattestedSwitchOff extends Error {
172
189
  function ambientAttachmentBucket(tenantId) {
173
190
  return ambientEnv[blobStoreBindingName(ATTACHMENT_BLOB_BINDING, tenantId)] ?? null;
174
191
  }
192
+ /**
193
+ * #2029: scope instances whose door has read the rewind hold and found NO claim on the scope at
194
+ * all, so their doors skip the hold read for the rest of the instance's life. Isolate-wide, because
195
+ * a host lives for one request: without it, every peer call would read the deployment's one hold
196
+ * object — an extra hop, on a single object, on the hot path of every scope never rewound.
197
+ *
198
+ * Safe because claims are only ever added for storage a LATER instance serves. A rewind writes its
199
+ * claim before it arms, and arming restarts the scope, so its rewound storage is always served by a
200
+ * new instance with a new id, which reads the hold afresh (`SWITCH_HOLD_SETTLE_MS` puts every row of
201
+ * the claim before that instance existed). A join (`switchHoldJoin`) goes into a rewind's claim too,
202
+ * so into that same later instance. An instance that read no claim therefore never gains one; claims
203
+ * only leave it, by a switch move. Only a GOOD read is remembered: a read that failed open is not.
204
+ * Bounded: past `CLEAR_SCOPE_INSTANCES_MAX` it is emptied, which costs a re-read, nothing else.
205
+ */
206
+ const CLEAR_SCOPE_INSTANCES = new Set();
207
+ const CLEAR_SCOPE_INSTANCES_MAX = 10_000;
208
+ const rememberClearInstance = (key) => {
209
+ if (CLEAR_SCOPE_INSTANCES.size >= CLEAR_SCOPE_INSTANCES_MAX)
210
+ CLEAR_SCOPE_INSTANCES.clear();
211
+ CLEAR_SCOPE_INSTANCES.add(key);
212
+ };
213
+ /**
214
+ * #2045: a carry's fences, by tuple subject (`RecordedOffCarry.fences`): for each recorded-off
215
+ * subject, the operation id of the call the record holds.
216
+ */
217
+ const switchFencesCarried = (kinds) => {
218
+ const out = {};
219
+ for (const [kind, { keys, fences }] of kinds) {
220
+ for (const key of keys) {
221
+ const fence = fences.get(key);
222
+ if (fence !== undefined)
223
+ out[switchSubjectOf(kind, key)] = fence;
224
+ }
225
+ }
226
+ return out;
227
+ };
228
+ /** #2029: how a door's refusals and a re-assert's name each kind of subject. */
229
+ const DOOR_WORDS = {
230
+ system: { what: (key) => `module '${key}'`, door: 'system door', switchName: 'schedule', switched: 'schedules' },
231
+ peer: { what: (key) => `peer vertical '${key}'`, door: 'peer door', switchName: 'peer', switched: 'peers' },
232
+ };
233
+ /** #2029: the rewind hold's key for one subject — a module's id, bare as #1819 stored it, or `vertical:<slug>`. */
234
+ const holdKeyOf = (kind, key) => (kind === 'system' ? key : peerSubjectRef(key));
175
235
  export class CloudflareScopeHost {
176
236
  admin;
177
237
  scopeNs;
@@ -215,6 +275,7 @@ export class CloudflareScopeHost {
215
275
  attachmentBuckets;
216
276
  /** The parsers attachment text is extracted with (K-43); the host's own, never imported here. */
217
277
  attachmentExtractors;
278
+ attachmentTextBounds;
218
279
  executors = new Map();
219
280
  /**
220
281
  * `<moduleId>/<job>` → the pass body and its default step policy (#1577). Host
@@ -225,10 +286,10 @@ export class CloudflareScopeHost {
225
286
  */
226
287
  jobs = new Map();
227
288
  /**
228
- * The event currently being effected, stamped onto admin rows the executor writes.
229
- * Ambient rather than threaded through every HostAdmin signature: set and cleared
230
- * around one await, with executors running sequentially, so there is no window
231
- * where it belongs to a different event.
289
+ * The event an executor is effecting, stamped onto the admin rows it writes (K-22). Like
290
+ * `onBehalfOf`, only ever set on a VIEW (`causedByView`, or `attributed` with a
291
+ * `causedBy`), never on the host: a field set around the handler's `await` stamped every
292
+ * admin call the host served meanwhile — a staff call included — with that event (#2055).
232
293
  */
233
294
  causedBy = null;
234
295
  /**
@@ -282,6 +343,7 @@ export class CloudflareScopeHost {
282
343
  this.attachmentBuckets = options.attachmentBuckets ?? ambientAttachmentBucket;
283
344
  assertAttachmentExtractors(options.attachmentExtractors ?? []);
284
345
  this.attachmentExtractors = options.attachmentExtractors ?? [];
346
+ this.attachmentTextBounds = resolveAttachmentTextBounds(options.attachmentTextBounds);
285
347
  this.fetchImpl = options.fetch ?? globalFetch;
286
348
  this.connectorCalls = options.connectorCalls ?? noopConnectorCallRecorder;
287
349
  this.scopeLocalPermissions = options.scopeLocalPermissions ?? false;
@@ -389,6 +451,22 @@ export class CloudflareScopeHost {
389
451
  },
390
452
  };
391
453
  }
454
+ /**
455
+ * #1184: the scope reads an executor's handler may make. Plain RPCs to the ScopeDO: the
456
+ * handler runs here on the coordinator, after the DO's own task has returned.
457
+ */
458
+ executorScope(tenantId, scopeId) {
459
+ const stub = this.scopeStub(scopeId);
460
+ return {
461
+ history: async (entity, page) => stub.entityHistory({ entityType: entity.entityType, entityId: entity.entityId, limit: page?.limit, cursor: page?.cursor }),
462
+ covers: async (principal, roleKey, level) => {
463
+ const bound = await stub.canAssignFor(tenantId, scopeId, principalId.parse(principal), roleKey, level === 'tenant');
464
+ if (!bound)
465
+ throw unknownRoleError(roleKey);
466
+ return coverage.parse(bound);
467
+ },
468
+ };
469
+ }
392
470
  /**
393
471
  * Drain this scope's outbox into the registered executors (K-22 §4.2).
394
472
  *
@@ -412,7 +490,9 @@ export class CloudflareScopeHost {
412
490
  * own `invocationId` field reads null — an ambient read there would record "no
413
491
  * call" for every attempt an operation's own tail made.
414
492
  */
415
- invocationId) {
493
+ invocationId,
494
+ /** #1184: what each attempt did, for the emitting call's `onExecutorOutcomes`. */
495
+ outcomes) {
416
496
  const report = {
417
497
  attempted: 0,
418
498
  delivered: 0,
@@ -426,27 +506,22 @@ export class CloudflareScopeHost {
426
506
  // attempt runs — the DO only journals it.
427
507
  const lines = asyncLinePass();
428
508
  try {
429
- await this.drainExecutorsPass(tenantId, scopeId, invocationId, report, lines);
509
+ await this.drainExecutorsPass(tenantId, scopeId, invocationId, report, lines, outcomes);
430
510
  }
431
511
  finally {
432
512
  lines.end();
433
513
  }
434
514
  return report;
435
515
  }
436
- async drainExecutorsPass(tenantId, scopeId, invocationId, report, lines) {
516
+ async drainExecutorsPass(tenantId, scopeId, invocationId, report, lines, outcomes) {
437
517
  const stub = this.scopeStub(scopeId);
518
+ const scope = this.executorScope(tenantId, scopeId);
438
519
  // #2005: a non-primary scope (a fork, a snapshot, a preview of either kind) causes no
439
520
  // outbound effects, so its deliveries are journaled terminal with the reason and no
440
- // handler runs. Asked of the directory where there is one. A CP-less host has none, so it
441
- // asks the scope's own storage whether it is classified a copy (`_substrat_copy_origin`'s
442
- // `is_copy`, #2009, set on the directory's word) — a preview and a snapshot reach a hosted
443
- // vertical as restores the platform marks.
444
- // Asked on the first due event, once per pass: most passes have none, and the directory
445
- // is one global object.
521
+ // handler runs (`isInertScope`). Asked on the first due event, once per pass: most passes
522
+ // have none, and the directory is one global object.
446
523
  let inert;
447
- const isInert = () => (inert ??= this.cpLess
448
- ? stub.isCopy()
449
- : this.cp.getScopeRecord(tenantId, scopeId).then((row) => !isPrimaryScopeRow(row)));
524
+ const isInert = () => (inert ??= this.isInertScope(tenantId, scopeId));
450
525
  // #1713: a CP-less scope its lifecycle holds attempts nothing and journals nothing, so every
451
526
  // due delivery stays due for the first pass after it is live again. Asked on the first due
452
527
  // event, like `isInert`. Every caller passed `assertLive` already, so this is for the scope
@@ -477,6 +552,9 @@ export class CloudflareScopeHost {
477
552
  attempt,
478
553
  startedAt: Date.now(),
479
554
  });
555
+ const outcomeOf = (event, outcome, error) => {
556
+ outcomes?.push(executorOutcomeOf(id, event, outcome, error));
557
+ };
480
558
  for (const bad of undecodable) {
481
559
  report.attempted += 1;
482
560
  const attempt = await stub.recordExecutorAttempt(bad.eventId, deliveryId, bad.error, null, invocationId);
@@ -490,11 +568,13 @@ export class CloudflareScopeHost {
490
568
  // No next attempt, like an undecodable row: a scope does not become primary.
491
569
  const attempt = await stub.recordExecutorAttempt(event.id, deliveryId, INERT_SCOPE_REASON, null, invocationId);
492
570
  report.inert = (report.inert ?? 0) + 1;
571
+ outcomeOf(event, 'inert', INERT_SCOPE_REASON);
493
572
  // Its own outcome, never `delivered`: no handler ran (#2005).
494
573
  lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'inert' });
495
574
  continue;
496
575
  }
497
- this.causedBy = event.id;
576
+ // #2055: the handler writes through a view bound to its event, never through the host.
577
+ const caused = this.causedByView(event.id);
498
578
  try {
499
579
  if (executor.kind === 'connector' && this.cpLess) {
500
580
  // #574 phase 3: this host cannot run a connector — no connection
@@ -507,20 +587,34 @@ export class CloudflareScopeHost {
507
587
  // the at-least-once residue, as it already must in-process.
508
588
  await stub.routeExecutorEventToPlatform(event.id, deliveryId, connectorDispatchKind(executor.provider), JSON.stringify({ executorId: id, event }), JSON.stringify({ system: 'connector-dispatch' }), invocationId);
509
589
  report.routedToPlatform += 1;
590
+ outcomeOf(event, 'routed');
510
591
  // Handed on, not run here: the platform's drain owns its attempts from now on.
511
592
  lines.write({ ...unitOf(event.id, 1), startedAt, outcome: 'routed' });
512
593
  }
513
594
  else if (executor.kind === 'connector') {
514
- await executor.handler(await this.connectorContext(tenantId, scopeId, executor.timeoutMs, event.id), event);
595
+ await executor.handler(await caused.connectorContext(tenantId, scopeId, executor.timeoutMs, event.id), event);
515
596
  const attempt = await stub.recordExecutorAttempt(event.id, deliveryId, null, null, invocationId);
516
597
  report.delivered += 1;
598
+ outcomeOf(event, 'delivered');
517
599
  lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'delivered' });
518
600
  }
519
601
  else {
520
- await executor.handler(this.admin, event);
521
- const attempt = await stub.recordExecutorAttempt(event.id, deliveryId, null, null, invocationId);
522
- report.delivered += 1;
523
- lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'delivered' });
602
+ const result = await executor.handler(caused.admin, event, scope);
603
+ // #1184: a refusal is the handler's own terminal decision — journaled with its
604
+ // reason, no next attempt, listed by `executorDeadLetters` beside an exhausted one.
605
+ const refused = isDeliveryRefusal(result) ? result : null;
606
+ const text = refused && refusalJournalText(refused);
607
+ const attempt = await stub.recordExecutorAttempt(event.id, deliveryId, text, null, invocationId);
608
+ if (refused) {
609
+ report.deadLettered += 1;
610
+ outcomeOf(event, 'refused', refused.reason);
611
+ lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'dead-lettered', error: text });
612
+ }
613
+ else {
614
+ report.delivered += 1;
615
+ outcomeOf(event, 'delivered');
616
+ lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'delivered' });
617
+ }
524
618
  }
525
619
  }
526
620
  catch (err) {
@@ -535,6 +629,7 @@ export class CloudflareScopeHost {
535
629
  report.deadLettered += 1;
536
630
  else
537
631
  report.retrying += 1;
632
+ outcomeOf(event, exhausted ? 'dead-lettered' : 'retrying', err);
538
633
  lines.write({
539
634
  ...unitOf(event.id, attempts),
540
635
  startedAt,
@@ -542,9 +637,6 @@ export class CloudflareScopeHost {
542
637
  error: err,
543
638
  });
544
639
  }
545
- finally {
546
- this.causedBy = null;
547
- }
548
640
  }
549
641
  }
550
642
  }
@@ -558,13 +650,14 @@ export class CloudflareScopeHost {
558
650
  // emitting operation's own tail made.
559
651
  return this.drainExecutors(tenantId, scopeId, null);
560
652
  }
561
- registerJob(moduleId, name, handler, retry) {
653
+ registerJob(moduleId, name, handler, retry, options) {
562
654
  // #1575: the kernel's own jobs are dispatched before this registry is read.
563
655
  assertJobRegistrable(moduleId, name);
656
+ assertLeaseMs(options?.leaseMs);
564
657
  const key = `${moduleId}/${name}`;
565
658
  if (this.jobs.has(key))
566
659
  throw new Error(`job '${key}' is already registered`);
567
- this.jobs.set(key, { handler, retry });
660
+ this.jobs.set(key, { handler, retry, leaseMs: options?.leaseMs });
568
661
  }
569
662
  /**
570
663
  * D-14's DURABLE driver: the run store is RPC to the scope DO, and nothing else.
@@ -582,13 +675,15 @@ export class CloudflareScopeHost {
582
675
  const stub = this.scopeStub(scopeId);
583
676
  return {
584
677
  startOrJoin: (key, row) => stub.jobRunStartOrJoin(key.moduleId, key.job, key.instance, row),
585
- get: (id) => stub.jobRunById(id),
586
678
  dueKeys: (now, max) => stub.jobRunsDueKeys(now, max),
679
+ claim: (id, owner, leaseMs) => stub.jobRunClaim(id, owner, leaseMs),
680
+ begin: (id, owner, marginMs) => stub.jobRunBegin(id, owner, marginMs),
681
+ miss: (id, owner, note) => stub.jobRunMiss(id, owner, note),
587
682
  list: (filter) => stub.jobRunList(filter),
588
- patch: (id, patch) => stub.jobRunPatch(id, patch),
589
- commitPass: (id, patch) => stub.jobCommitPass(id, patch),
590
- step: (runId, name) => stub.jobStepRow(runId, name),
591
- recordStep: (runId, name, result, attempts, lastError, at) => stub.jobStepRecord(runId, name, result, attempts, lastError, at),
683
+ patch: (id, patch, owner) => stub.jobRunPatch(id, patch, owner),
684
+ commitPass: (id, patch, owner) => stub.jobCommitPass(id, patch, owner),
685
+ beginStep: (runId, name, owner, leaseMs) => stub.jobStepBegin(runId, name, owner, leaseMs),
686
+ recordStep: (runId, name, result, attempts, lastError, at, owner, leaseMs) => stub.jobStepRecord(runId, name, result, attempts, lastError, at, owner, leaseMs),
592
687
  };
593
688
  }
594
689
  /**
@@ -606,7 +701,7 @@ export class CloudflareScopeHost {
606
701
  write: (attachmentId, outcome) => stub.attachmentTextRecord(attachmentId, outcome),
607
702
  },
608
703
  // K-43: the host's parsers, handed in — this adapter imports none.
609
- this.attachmentExtractors);
704
+ this.attachmentExtractors, this.attachmentTextBounds);
610
705
  }
611
706
  async startJobRun(tenantId, scopeId, input) {
612
707
  await this.assertLive(tenantId, scopeId);
@@ -618,12 +713,18 @@ export class CloudflareScopeHost {
618
713
  // advance, and an archived one's never move again.
619
714
  await this.assertLive(tenantId, scopeId);
620
715
  await this.migrateAndRecord(scopeId);
621
- // #1575: the kernel's extraction job is this host's own, bound to this scope — no
622
- // deployment registers it, and none can shadow it.
623
- const attachmentText = { handler: this.attachmentTextHandler(tenantId, scopeId) };
716
+ // #1575: the kernel's extraction jobs are this host's own, bound to this scope — no
717
+ // deployment registers them, and none can shadow them. The backfill starts here, on
718
+ // the drive, so a scope's first request never pays for attachments that predate it.
719
+ const stub = this.scopeStub(scopeId);
720
+ await stub.attachmentTextBackfillStart();
721
+ const kernelJobs = {
722
+ [ATTACHMENT_TEXT_JOB]: this.attachmentTextHandler(tenantId, scopeId),
723
+ [ATTACHMENT_TEXT_BACKFILL_JOB]: attachmentTextBackfillJob({ queueBatch: (after) => stub.attachmentTextBackfillBatch(after) }),
724
+ };
624
725
  return runDueJobRuns({
625
726
  store: this.jobStore(scopeId),
626
- handlerFor: (run) => isAttachmentTextRun(run) ? attachmentText : this.jobs.get(`${run.module_id}/${run.job}`),
727
+ handlerFor: (run) => kernelJobFor(run, kernelJobs) ?? this.jobs.get(`${run.module_id}/${run.job}`),
627
728
  now: () => new Date().toISOString(),
628
729
  // #1834: the door is opened FOR this pass, so its "not now" is tied to this pass alone.
629
730
  openScope: async (run, pass) => this.buildStub(tenantId, scopeId, undefined, undefined, await this.openSystemDoor(run.module_id, tenantId, scopeId, undefined, pass)),
@@ -645,13 +746,9 @@ export class CloudflareScopeHost {
645
746
  // ran it. On a CP-less host `connectorContext` throws from the null control plane:
646
747
  // fail closed, exactly the hole routing exists to avoid.
647
748
  await this.assertLive(tenantId, scopeId);
648
- this.causedBy = event.id;
649
- try {
650
- await handler(await this.connectorContext(tenantId, scopeId, options?.timeoutMs ?? 30_000, event.id), event);
651
- }
652
- finally {
653
- this.causedBy = null;
654
- }
749
+ // Built on a view bound to the event, so its admin rows carry it and nothing else the
750
+ // host serves does (#2055).
751
+ await handler(await this.causedByView(event.id).connectorContext(tenantId, scopeId, options?.timeoutMs ?? 30_000, event.id), event);
655
752
  }
656
753
  async executorDeadLetters(tenantId, scopeId) {
657
754
  await this.validateScopeAccess(tenantId, scopeId);
@@ -710,7 +807,7 @@ export class CloudflareScopeHost {
710
807
  if (!rec)
711
808
  throw unknownScopeForTenant(tenantId, scopeId);
712
809
  if (rec.status !== 'active' && rec.status !== 'provisioning') {
713
- throw new Error(`scope not migratable (status: ${rec.status}): ${scopeId}`);
810
+ throw substratError('conflict', `scope not migratable (status: ${rec.status}): ${scopeId}`);
714
811
  }
715
812
  const stub = this.scopeStub(scopeId);
716
813
  try {
@@ -752,11 +849,11 @@ export class CloudflareScopeHost {
752
849
  return this.scopeStub(scopeId).introspectTables();
753
850
  }
754
851
  async introspectScopeTable(scopeId, input) {
755
- return this.scopeStub(scopeId).introspectTable(input.table, input.limit, input.offset);
852
+ return unwrapReply(await this.scopeStub(scopeId).introspectTableReply(input.table, input.limit, input.offset));
756
853
  }
757
854
  /** The SQL console's CP-less path (#219) — same trust line as the pair above. */
758
855
  async introspectScopeQuery(scopeId, input) {
759
- return this.scopeStub(scopeId).introspectQuery(input.sql);
856
+ return unwrapReply(await this.scopeStub(scopeId).introspectQueryReply(input.sql));
760
857
  }
761
858
  /**
762
859
  * The K-35 denial log's CP-less path (#867) — same trust line as the reads above.
@@ -853,10 +950,9 @@ export class CloudflareScopeHost {
853
950
  opts) {
854
951
  if (opts?.markCopy)
855
952
  assertCopyLineage(opts.markCopy);
953
+ const carry = opts ? recordedOffFromWire(opts) : undefined;
856
954
  const load = {
857
- switchOff: opts?.switchedOff
858
- ? { moduleIds: opts.switchedOff, at: new Date().toISOString(), tenantHeld: opts.tenantHeld }
859
- : undefined,
955
+ switchOff: carry ? { at: new Date().toISOString(), ...carry } : undefined,
860
956
  sourceScopeId: opts?.sourceScopeId,
861
957
  exact: opts?.exact,
862
958
  loadStamp: opts?.loadStamp,
@@ -870,7 +966,7 @@ export class CloudflareScopeHost {
870
966
  ...(opts?.expect ? { expect: opts.expect } : {}),
871
967
  });
872
968
  if (out.refused === false) {
873
- return { tables: tables.length, ...(opts?.switchedOff ? { switchedOff: out.switchedOff } : {}) };
969
+ return { tables: tables.length, ...(carry ? { switchedOff: out.switchedOff } : {}) };
874
970
  }
875
971
  if (out.refused === 'kept')
876
972
  throw substratError('conflict', KEPT_COPY_REFUSAL);
@@ -993,7 +1089,7 @@ export class CloudflareScopeHost {
993
1089
  * safe; scope-level tuples (the restored grants) are never touched.
994
1090
  */
995
1091
  async projectRolesLocal(tenantId, scopeId, roles) {
996
- await this.scopeStub(scopeId).applyProjection(tenantId, roles.map((r) => ({ role_key: r.key, permissions: JSON.stringify(r.permissions), source: r.source })), []);
1092
+ unwrapReply(await this.scopeStub(scopeId).applyProjectionReply(tenantId, roles.map((r) => ({ role_key: r.key, permissions: JSON.stringify(r.permissions), source: r.source })), []));
997
1093
  }
998
1094
  /**
999
1095
  * Wipe one scope DO's storage in THIS deployment — the reap half of an orchestrated
@@ -1045,6 +1141,10 @@ export class CloudflareScopeHost {
1045
1141
  async lifecycleFlowLocal(scopeId, input) {
1046
1142
  return this.scopeStub(scopeId).lifecycleFlow(input);
1047
1143
  }
1144
+ /** Business volumes per bucket (#1750) on this host's own scope — the vertical-host read. */
1145
+ async operationSeriesLocal(scopeId, input) {
1146
+ return this.scopeStub(scopeId).operationSeries(input);
1147
+ }
1048
1148
  /** One event's causal chain (#1237) on this host's own scope — the vertical-host read. */
1049
1149
  async eventCauseLocal(scopeId, input) {
1050
1150
  return this.scopeStub(scopeId).eventCause(input);
@@ -1197,7 +1297,7 @@ export class CloudflareScopeHost {
1197
1297
  const record = resolveScopeRecord(input);
1198
1298
  // Fail-closed tenant gate throws out of the awaited cp call BEFORE migrate
1199
1299
  // or audit, so a rejected provision creates nothing and writes no audit row.
1200
- const created = await this.cp.provisionScope(input.tenantId, input.scopeId, record, new Date().toISOString());
1300
+ const created = await this.directory('provisionScope', input.tenantId, input.scopeId, record, new Date().toISOString());
1201
1301
  // Instantiate the scope DO and trigger its lazy migration.
1202
1302
  await this.migrateAndRecord(input.scopeId);
1203
1303
  // Scope-local permissions: a freshly-provisioned scope evaluates from its own
@@ -1238,17 +1338,9 @@ export class CloudflareScopeHost {
1238
1338
  // #1742: in the SAME unit as the seat, so no sweep can land between them; the re-assert
1239
1339
  // after it then finds them off, and audits what the unit moved.
1240
1340
  const ownStore = this.cpLess || record.vertical === null;
1241
- const recordedOff = ownStore ? await this.cp.switchedOffModulesOf(input.tenantId, input.scopeId) : [];
1242
1341
  const seatAt = new Date().toISOString();
1243
- const switchedOff = await this.scopeStub(input.scopeId).seatTuples(seats, recordedOff.length
1244
- ? {
1245
- scopeId: input.scopeId,
1246
- moduleIds: recordedOff,
1247
- at: seatAt,
1248
- // #1823: a recorded-off module held only by a tenant grant goes back off too.
1249
- tenantHeld: await this.cp.tenantHeldSystemModules(input.tenantId, recordedOff, seatAt),
1250
- }
1251
- : undefined);
1342
+ const carry = ownStore ? await this.recordedOffCarry(input.tenantId, input.scopeId, seatAt) : undefined;
1343
+ const switchedOff = await this.scopeStub(input.scopeId).seatTuples(seats, carry ? { scopeId: input.scopeId, at: seatAt, ...carry } : undefined);
1252
1344
  if (ownStore) {
1253
1345
  await this.admin.reassertSystemSwitches(actor, { tenantId: input.tenantId, scopeId: input.scopeId }, { appliedInUnit: switchedOff });
1254
1346
  }
@@ -1270,10 +1362,10 @@ export class CloudflareScopeHost {
1270
1362
  // inside the DO's ledger write, which is the serialization point that actually holds.
1271
1363
  const tenant = await this.cp.getTenant(input.tenantId);
1272
1364
  if (!tenant) {
1273
- throw new Error(`cannot provision tenant store under unknown tenant: ${input.tenantId}`);
1365
+ throw substratError('not_found', `cannot provision tenant store under unknown tenant: ${input.tenantId}`);
1274
1366
  }
1275
1367
  if (tenant.status !== 'active') {
1276
- throw new Error(`cannot provision tenant store under non-active tenant (status: ${tenant.status}): ${input.tenantId}`);
1368
+ throw substratError('conflict', `cannot provision tenant store under non-active tenant (status: ${tenant.status}): ${input.tenantId}`);
1277
1369
  }
1278
1370
  // Idempotent on (tenant, vertical, binding): a retried provision re-resolves the SAME
1279
1371
  // store rather than minting a second database (the K-31 ready-gate retries the whole
@@ -1288,7 +1380,7 @@ export class CloudflareScopeHost {
1288
1380
  // the source of truth, carrying the D1 database_id Cloudflare assigned as the ref.
1289
1381
  const name = await tenantStoreDatabaseName(input.tenantId, input.vertical, input.binding);
1290
1382
  const ref = await d1.create(name);
1291
- const stored = await this.cp.putTenantStore({
1383
+ const stored = await this.directory('putTenantStore', {
1292
1384
  tenantId: input.tenantId,
1293
1385
  vertical: input.vertical,
1294
1386
  binding: input.binding,
@@ -1339,17 +1431,17 @@ export class CloudflareScopeHost {
1339
1431
  const r2 = this.requireBlobStores(`provisionBlobStore(tenant=${input.tenantId} vertical=${input.vertical} binding=${input.binding})`);
1340
1432
  const tenant = await this.cp.getTenant(input.tenantId);
1341
1433
  if (!tenant) {
1342
- throw new Error(`cannot provision blob store under unknown tenant: ${input.tenantId}`);
1434
+ throw substratError('not_found', `cannot provision blob store under unknown tenant: ${input.tenantId}`);
1343
1435
  }
1344
1436
  if (tenant.status !== 'active') {
1345
- throw new Error(`cannot provision blob store under non-active tenant (status: ${tenant.status}): ${input.tenantId}`);
1437
+ throw substratError('conflict', `cannot provision blob store under non-active tenant (status: ${tenant.status}): ${input.tenantId}`);
1346
1438
  }
1347
1439
  const existing = await this.cp.getBlobStore(input.tenantId, input.vertical, input.binding);
1348
1440
  if (existing)
1349
1441
  return { binding: input.binding, kind: 'blob', ref: existing.ref };
1350
1442
  const name = await blobStoreBucketName(input.tenantId, input.vertical, input.binding);
1351
1443
  const ref = await r2.create(name);
1352
- const stored = await this.cp.putBlobStore({
1444
+ const stored = await this.directory('putBlobStore', {
1353
1445
  tenantId: input.tenantId,
1354
1446
  vertical: input.vertical,
1355
1447
  binding: input.binding,
@@ -1488,22 +1580,22 @@ export class CloudflareScopeHost {
1488
1580
  // the DO only to be refused and recorded — never to the blob store.
1489
1581
  const hash = subject.capabilitySession;
1490
1582
  return {
1491
- upload: async (input) => unwrapCapabilityReply(await stub.capabilityAttachmentRefuseWrite(hash, tenantId, scopeId, 'attachments.upload', {
1583
+ upload: async (input) => unwrapReply(await stub.capabilityAttachmentRefuseWrite(hash, tenantId, scopeId, 'attachments.upload', {
1492
1584
  entityType: input.entity.entityType,
1493
1585
  })),
1494
- list: async (entity) => unwrapCapabilityReply(await stub.capabilityAttachmentList(entity, hash, tenantId, scopeId)),
1586
+ list: async (entity) => unwrapReply(await stub.capabilityAttachmentList(entity, hash, tenantId, scopeId)),
1495
1587
  open: async (attachmentId) => {
1496
- const record = unwrapCapabilityReply(await stub.capabilityAttachmentOpen(attachmentId, hash, tenantId, scopeId));
1588
+ const record = unwrapReply(await stub.capabilityAttachmentOpen(attachmentId, hash, tenantId, scopeId));
1497
1589
  return record ? this.openAttachmentBytes(store, scopeId, record) : null;
1498
1590
  },
1499
- remove: async (attachmentId) => unwrapCapabilityReply(await stub.capabilityAttachmentRefuseWrite(hash, tenantId, scopeId, 'attachments.remove', {
1591
+ remove: async (attachmentId) => unwrapReply(await stub.capabilityAttachmentRefuseWrite(hash, tenantId, scopeId, 'attachments.remove', {
1500
1592
  attachmentId,
1501
1593
  })),
1502
1594
  search: async (term, options) => {
1503
1595
  // Judged here too, so a short term reaches the caller as `SearchTermTooShort`
1504
1596
  // rather than as a message that crossed the RPC boundary. The DO judges again.
1505
1597
  searchMatchExpression(term, 'prefix');
1506
- return unwrapCapabilityReply(await stub.capabilityAttachmentSearch(term, searchLimit(options?.limit), hash, tenantId, scopeId));
1598
+ return unwrapReply(await stub.capabilityAttachmentSearch(term, searchLimit(options?.limit), hash, tenantId, scopeId));
1507
1599
  },
1508
1600
  };
1509
1601
  }
@@ -1558,7 +1650,7 @@ export class CloudflareScopeHost {
1558
1650
  throw new Error('attachments.search is not available on a delivery-scoped surface');
1559
1651
  }
1560
1652
  searchMatchExpression(term, 'prefix');
1561
- return unwrapCapabilityReply(await stub.attachmentSearch(term, searchLimit(options?.limit), asPrincipalId, tenantId, scopeId, connectionId));
1653
+ return unwrapReply(await stub.attachmentSearch(term, searchLimit(options?.limit), asPrincipalId, tenantId, scopeId, connectionId));
1562
1654
  },
1563
1655
  };
1564
1656
  }
@@ -1606,24 +1698,51 @@ export class CloudflareScopeHost {
1606
1698
  // this is the scope's real store, as for the provision's seat above; a hosted scope is
1607
1699
  // restored in its deployment, which the platform carries the same list to.
1608
1700
  const ownStore = this.cpLess || existing.vertical === null;
1609
- const recordedOff = ownStore ? await this.cp.switchedOffModulesOf(tenantId, scopeId) : [];
1610
1701
  const replayAt = new Date().toISOString();
1702
+ const carry = ownStore ? await this.recordedOffCarry(tenantId, scopeId, replayAt) : undefined;
1611
1703
  const switchedOff = await this.scopeStub(scopeId).importDump(dump.tables, scopeId, {
1612
- switchOff: recordedOff.length
1613
- ? {
1614
- moduleIds: recordedOff,
1615
- at: replayAt,
1616
- // #1823: a recorded-off module held only by a tenant grant goes back off too.
1617
- tenantHeld: await this.cp.tenantHeldSystemModules(tenantId, recordedOff, replayAt),
1618
- }
1619
- : undefined,
1704
+ switchOff: carry ? { at: replayAt, ...carry } : undefined,
1620
1705
  sourceScopeId: opts?.sourceScopeId ?? dump.scopeId,
1621
1706
  });
1622
1707
  await this.recordAdmin(actor, 'restoreScope', { tenantId, scopeId }, null, { sourceScopeId: dump.scopeId, tables: dump.tables.length, capturedAt: dump.capturedAt });
1623
- if (recordedOff.length) {
1708
+ if (carry) {
1624
1709
  await this.admin.reassertSystemSwitches(actor, { tenantId, scopeId }, { appliedInUnit: switchedOff });
1625
1710
  }
1626
1711
  }
1712
+ /**
1713
+ * #1742, #2029: what this host's own unit carries of the directory's record — the modules and
1714
+ * the peers recorded OFF on the scope, each with the ones held there only by a tenant-level
1715
+ * grant (#1823, #2030). Undefined when nothing is recorded off, so such a unit is unchanged.
1716
+ */
1717
+ async recordedOffCarry(tenantId, scopeId, at) {
1718
+ const [modules, peers] = await Promise.all([
1719
+ this.recordedOff('system', tenantId, scopeId, at),
1720
+ this.recordedOff('peer', tenantId, scopeId, at),
1721
+ ]);
1722
+ if (!modules.keys.length && !peers.keys.length)
1723
+ return undefined;
1724
+ return {
1725
+ moduleIds: modules.keys,
1726
+ tenantHeld: modules.tenantHeld,
1727
+ verticals: peers.keys,
1728
+ tenantHeldVerticals: peers.tenantHeld,
1729
+ fences: switchFencesCarried([
1730
+ ['system', modules],
1731
+ ['peer', peers],
1732
+ ]),
1733
+ };
1734
+ }
1735
+ /** The subjects of one kind the directory records OFF on the scope, and of those the tenant-held ones. */
1736
+ async recordedOff(kind, tenantId, scopeId, at) {
1737
+ // #2045: the OFF rows and their fences in one read, so every key carried has its own row's fence.
1738
+ const fences = new Map((await this.cp.switchRecordStatesOf(kind, tenantId, scopeId))
1739
+ .filter(([, row]) => row.position === 'off')
1740
+ .map(([key, row]) => [key, row.fence]));
1741
+ const keys = [...fences.keys()];
1742
+ if (!keys.length)
1743
+ return { keys, tenantHeld: [], fences };
1744
+ return { keys, tenantHeld: await this.cp.tenantHeldOf(kind, tenantId, keys, at), fences };
1745
+ }
1627
1746
  async snapshotScope(actor, tenantId, scopeId, opts) {
1628
1747
  const source = await this.admin.getScopeRecord(actor, tenantId, scopeId);
1629
1748
  if (!source)
@@ -1654,7 +1773,7 @@ export class CloudflareScopeHost {
1654
1773
  if (!rec)
1655
1774
  throw substratError('not_found', `unknown scope ${scopeId} in tenant ${tenantId}`);
1656
1775
  if (!rec.forkedFrom && rec.kind !== 'preview') {
1657
- throw new Error(`scope ${scopeId} is not a fork or preview — only previews may be deleted; ` +
1776
+ throw substratError('forbidden', `scope ${scopeId} is not a fork or preview — only previews may be deleted; ` +
1658
1777
  `archive a primary scope instead`);
1659
1778
  }
1660
1779
  // Storage BEFORE the directory row: a crash between the two leaves a visible row
@@ -1815,34 +1934,55 @@ export class CloudflareScopeHost {
1815
1934
  /** #1834: the job pass this door is opened for; its "not now" refusals are tied to it. */
1816
1935
  pass) {
1817
1936
  if (!this.moduleIds.has(moduleId)) {
1818
- throw new Error(`module not registered on this host: ${moduleId}`);
1937
+ throw substratError('not_found', `module not registered on this host: ${moduleId}`);
1819
1938
  }
1820
1939
  await this.assertLive(tenantId, scopeId);
1821
1940
  await this.migrateAndRecord(scopeId);
1941
+ return this.openDoor('system', moduleId, scopeId, gated, pass);
1942
+ }
1943
+ /**
1944
+ * #1834: one door, for either kind of switched subject (#2029). The gate is read here unless the
1945
+ * caller read it (`runDueSchedules` reports a hold it could not read on its pass report); a held
1946
+ * subject is refused, and a call landing on another instance than the gate's re-gates, at most
1947
+ * `SYSTEM_DOOR_REGATES` times, then fails closed.
1948
+ *
1949
+ * The peer door (`kind: 'peer'`) is the system door's for a peer vertical. A PITR rewind to before
1950
+ * a peer was switched off brings its grants back live and its OFF marker gone, so the rewind holds
1951
+ * it outside the scope (`rewindHolding`, as for a module), and every peer entry — an invoke, a
1952
+ * delivery, a coverage read, an export read for it as consumer — opens this door after
1953
+ * `peerScopeGate`.
1954
+ */
1955
+ async openDoor(kind, key, scopeId, gated, pass) {
1956
+ const words = DOOR_WORDS[kind];
1957
+ const what = words.what(key);
1822
1958
  // A gate this door reads itself reports a hold it could not read; one handed in was the
1823
- // caller's to report (`runDueSchedules` puts it on its pass report).
1959
+ // caller's to report.
1824
1960
  const regate = async () => {
1825
- const fresh = await this.systemDoorGate(scopeId, moduleId);
1961
+ const fresh = await this.doorGate(kind, scopeId, key);
1826
1962
  if (fresh.error)
1827
- console.error(`substrat: system door ${moduleId} on ${scopeId}: ${fresh.error}`);
1963
+ console.error(`substrat: door of ${what} on ${scopeId}: ${fresh.error}`);
1828
1964
  return fresh;
1829
1965
  };
1830
1966
  let gate = gated ?? (await regate());
1831
1967
  return {
1832
- moduleId,
1968
+ kind,
1969
+ key,
1833
1970
  through: async (call) => {
1834
1971
  for (let regates = 0;; regates += 1) {
1835
1972
  if (gate.held) {
1836
1973
  // The hold ends when the switch is applied again: a job run waits it out without
1837
1974
  // spending its retries, because the driver defers on THIS refusal (`doorWait`).
1838
- throw this.doorWait(pass, substratError('forbidden', `module '${moduleId}' is held off on this scope: it was rewound to before its schedule ` +
1839
- 'switch was turned off, and it stays off until the switch is applied again', { reason: SYSTEM_DOOR_WAIT }));
1975
+ const held = substratError('forbidden', `${what} is held off on this scope: it was rewound to before its ` +
1976
+ `${words.switchName} switch was turned off, and it stays off until ` +
1977
+ 'the switch is applied again', { reason: SYSTEM_DOOR_WAIT });
1978
+ this.heldRefusals.add(held);
1979
+ throw this.doorWait(pass, held);
1840
1980
  }
1841
1981
  const answer = await call(gate.instance);
1842
1982
  if (!isSystemDoorMoved(answer))
1843
1983
  return answer;
1844
1984
  if (regates >= SYSTEM_DOOR_REGATES) {
1845
- throw this.doorWait(pass, substratError('unavailable', `the scope kept restarting under the system door of module '${moduleId}' ` +
1985
+ throw this.doorWait(pass, substratError('unavailable', `the scope kept restarting under the ${words.door} of ${what} ` +
1846
1986
  `(${regates + 1} attempts); nothing ran. Retry later`, { reason: SYSTEM_DOOR_WAIT }));
1847
1987
  }
1848
1988
  gate = await regate();
@@ -1859,6 +1999,16 @@ export class CloudflareScopeHost {
1859
1999
  * on a later pass is an ordinary failure there. Held weakly: an error nobody holds goes.
1860
2000
  */
1861
2001
  doorWaits = new WeakMap();
2002
+ /**
2003
+ * #2029: the hold refusals this host's doors threw, by identity — so a peer delivery pauses its
2004
+ * edge, and a coverage read answers "holds nothing", on exactly those and on no error that only
2005
+ * looks like one. Held weakly, as `doorWaits` is.
2006
+ */
2007
+ heldRefusals = new WeakSet();
2008
+ /** #2029: is `err` a hold refusal this host's door threw? */
2009
+ isHeldRefusal(err) {
2010
+ return typeof err === 'object' && err !== null && this.heldRefusals.has(err);
2011
+ }
1862
2012
  /** #1834: mark this door's own refusal as "not now" for the pass it was opened for, and hand it back. */
1863
2013
  doorWait(pass, err) {
1864
2014
  if (pass)
@@ -1873,13 +2023,23 @@ export class CloudflareScopeHost {
1873
2023
  return true;
1874
2024
  }
1875
2025
  /**
1876
- * #1834: the system door's gate — the scope's state read for the module, the instance that
1877
- * answered it, and then (only for a module the scope has on) the rewind hold (#1819). The
2026
+ * #1834: a door's gate — the scope's state read for the subject, the instance that answered it,
2027
+ * and then the rewind hold (#1819), for any subject the scope does not already have off. The
1878
2028
  * hold is read AFTER the state, which is the order `switchHeld` needs.
2029
+ *
2030
+ * `ungranted` reads the hold too (#2029): a subject held on the scope only by a TENANT-level
2031
+ * grant (#1823, #2030) comes back from a rewind with no scope grant and no marker, and the
2032
+ * tenant grant still authorizes it there. A rewind captured it off, so the hold must keep it so.
1879
2033
  */
1880
- async systemDoorGate(scopeId, moduleId) {
1881
- const { state, instance } = await this.scopeStub(scopeId).systemDoorState(moduleId);
1882
- const hold = state === 'on' ? await this.switchHeld(scopeId, moduleId) : { held: false };
2034
+ async doorGate(kind, scopeId, key) {
2035
+ const stub = this.scopeStub(scopeId);
2036
+ const { state, instance } = await (kind === 'system' ? stub.systemDoorState(key) : stub.peerDoorState(key));
2037
+ const cleared = `${scopeId} ${instance}`;
2038
+ if (state === 'off' || CLEAR_SCOPE_INSTANCES.has(cleared))
2039
+ return { state, instance, held: false };
2040
+ const hold = await this.switchHeld(scopeId, holdKeyOf(kind, key));
2041
+ if (hold.scopeClear)
2042
+ rememberClearInstance(cleared);
1883
2043
  return { state, instance, held: hold.held, ...(hold.error ? { error: hold.error } : {}) };
1884
2044
  }
1885
2045
  /**
@@ -1893,8 +2053,25 @@ export class CloudflareScopeHost {
1893
2053
  // The peer door's own gate (#1706): K-3's pair check, the refusal for a scope served
1894
2054
  // elsewhere, the lifecycle check and the migration. A delivery is a write into the
1895
2055
  // consumer's scope, so it takes the gate an invoke through that door takes.
1896
- await this.peerScopeGate(tenantId, scopeId, 'deliverToPeer');
1897
- const result = importResult.parse(await this.scopeStub(scopeId).importApply(batch, tenantId, scopeId));
2056
+ const record = await this.peerScopeGate(tenantId, scopeId, 'deliverToPeer');
2057
+ // #2004: a copy consumes no other vertical's events, at the door as in the sweep
2058
+ // (`scope-copy.ts`). A pause: nothing runs and the watermark stays.
2059
+ if (await this.isInertScope(tenantId, scopeId, record)) {
2060
+ return { ...emptyImportResult(batch), paused: { reason: INERT_SCOPE_REASON } };
2061
+ }
2062
+ // #2029: the producer is a peer, so the delivery goes through its door: a producer the rewind
2063
+ // hold keeps off pauses the edge, exactly as a switched-off one does inside the DO.
2064
+ const door = await this.openDoor('peer', batch.source.vertical, scopeId);
2065
+ let applied;
2066
+ try {
2067
+ applied = await door.through((instance) => this.scopeStub(scopeId).importApply(batch, tenantId, scopeId, instance));
2068
+ }
2069
+ catch (err) {
2070
+ if (!this.isHeldRefusal(err))
2071
+ throw err;
2072
+ return { ...emptyImportResult(batch), paused: { reason: err.message } };
2073
+ }
2074
+ const result = importResult.parse(applied);
1898
2075
  if (result.delivered > 0) {
1899
2076
  try {
1900
2077
  await this.drainExecutors(tenantId, scopeId, null);
@@ -1938,7 +2115,9 @@ export class CloudflareScopeHost {
1938
2115
  async getVerticalScope(caller, tenantId, scopeId, options) {
1939
2116
  const parsed = verticalCaller.parse(caller);
1940
2117
  await this.peerScopeGate(tenantId, scopeId, 'getVerticalScope');
1941
- return this.buildStub(tenantId, scopeId, undefined, undefined, undefined, options, undefined, undefined, parsed);
2118
+ // #2029: gated against the rewind hold and pinned, as the system door is.
2119
+ const door = await this.openDoor('peer', parsed.vertical, scopeId);
2120
+ return this.buildStub(tenantId, scopeId, undefined, undefined, door, options, undefined, undefined, parsed);
1942
2121
  }
1943
2122
  /**
1944
2123
  * Does peer `vertical` hold each key at this scope's node now (#1706) — the ScopeDO's own
@@ -1946,9 +2125,19 @@ export class CloudflareScopeHost {
1946
2125
  */
1947
2126
  async peerCovers(tenantId, scopeId, vertical, permissions) {
1948
2127
  await this.peerScopeGate(tenantId, scopeId, 'peerCovers');
1949
- return peerCoverage
1950
- .array()
1951
- .parse(await this.scopeStub(scopeId).peerCovers(tenantId, scopeId, verticalSlug.parse(vertical), [...permissions]));
2128
+ const slug = verticalSlug.parse(vertical);
2129
+ // #2029: a peer the rewind hold keeps off holds nothing here, as a switched-off one doesn't.
2130
+ const door = await this.openDoor('peer', slug, scopeId);
2131
+ try {
2132
+ return peerCoverage
2133
+ .array()
2134
+ .parse(await door.through((instance) => this.scopeStub(scopeId).peerCovers(tenantId, scopeId, slug, [...permissions], instance)));
2135
+ }
2136
+ catch (err) {
2137
+ if (this.isHeldRefusal(err))
2138
+ return permissions.map((permission) => ({ permission, held: false }));
2139
+ throw err;
2140
+ }
1952
2141
  }
1953
2142
  /**
1954
2143
  * `ctx.canAssign`'s bound for a principal the host names (#1931) — the ScopeDO's own role read
@@ -1980,6 +2169,32 @@ export class CloudflareScopeHost {
1980
2169
  throw unknownRoleError(roleKey);
1981
2170
  return coverage.parse(bound);
1982
2171
  }
2172
+ async listScopeRoleHolders(tenantId, scopeId, principal) {
2173
+ await this.scopeRoleGate(tenantId, scopeId, 'listScopeRoleHolders');
2174
+ return this.scopeStub(scopeId).scopeRoleHoldersFor(scopeId, principal === undefined ? undefined : principalId.parse(principal));
2175
+ }
2176
+ async changeScopeRoleBounded(tenantId, scopeId, caller, principal, from, to) {
2177
+ await this.scopeRoleGate(tenantId, scopeId, 'changeScopeRoleBounded');
2178
+ const target = principalId.parse(principal);
2179
+ const bound = await this.scopeStub(scopeId).changeScopeRoleBoundedFor(tenantId, scopeId, principalId.parse(caller), target, from, to);
2180
+ if (bound === 'not-held')
2181
+ throw substratError('conflict', `${target} does not hold '${from}' at this scope`);
2182
+ if (bound === 'unknown-to')
2183
+ throw unknownRoleError(to);
2184
+ return coverage.parse(bound);
2185
+ }
2186
+ async revokeScopeRolesBounded(tenantId, scopeId, caller, principal) {
2187
+ await this.scopeRoleGate(tenantId, scopeId, 'revokeScopeRolesBounded');
2188
+ const result = await this.scopeStub(scopeId).revokeScopeRolesBoundedFor(tenantId, scopeId, principalId.parse(caller), principalId.parse(principal));
2189
+ return { coverage: coverage.parse(result.coverage), revoked: result.revoked };
2190
+ }
2191
+ /** The (tenant, scope) gate the scope-role verbs share — `assignScopeRoleBounded`'s two checks. */
2192
+ async scopeRoleGate(tenantId, scopeId, verb) {
2193
+ if (this.cpLess && !(await this.scopeStub(scopeId).servesTenant(tenantId))) {
2194
+ throw unknownScopeForTenant(tenantId, scopeId);
2195
+ }
2196
+ await this.peerScopeGate(tenantId, scopeId, verb);
2197
+ }
1983
2198
  /**
1984
2199
  * Where a peer verb (#1706) may reach a scope: its own ScopeDO, after the ordinary pair and
1985
2200
  * lifecycle gate. Refused, loudly, on the SHARED control plane for a scope bound to a
@@ -1994,8 +2209,9 @@ export class CloudflareScopeHost {
1994
2209
  // door's own tenant-confinement test proved. A RECORD crosses intact, so the coordinator
1995
2210
  // reads the record and types the refusal itself. `validateScopeAccess` still runs below
1996
2211
  // and still owns the lifecycle half (a suspended tenant or scope).
2212
+ let record;
1997
2213
  if (!this.cpLess) {
1998
- const record = await this.cp.getScopeRecord(tenantId, scopeId);
2214
+ record = await this.cp.getScopeRecord(tenantId, scopeId);
1999
2215
  if (!record) {
2000
2216
  throw unknownScopeForTenant(tenantId, scopeId);
2001
2217
  }
@@ -2003,6 +2219,23 @@ export class CloudflareScopeHost {
2003
2219
  }
2004
2220
  await this.assertLive(tenantId, scopeId);
2005
2221
  await this.migrateAndRecord(scopeId);
2222
+ return record;
2223
+ }
2224
+ /**
2225
+ * Whether this scope is held inert (#2005): a fork, a snapshot or a preview, which causes no
2226
+ * outbound effects and consumes no other vertical's events (#2004). The one primacy question
2227
+ * every such door asks. Where there is a directory its record answers (`record`, the one
2228
+ * `peerScopeGate` returns, when the caller has already read it). A CP-less host has none, so it asks the scope's own storage
2229
+ * whether it is classified a copy (`_substrat_copy_origin`'s `is_copy`, #2009, set on the
2230
+ * directory's word): a preview and a snapshot reach a hosted vertical as restores the
2231
+ * platform marks.
2232
+ */
2233
+ isInertScope(tenantId, scopeId, record) {
2234
+ if (this.cpLess)
2235
+ return this.scopeStub(scopeId).isCopy();
2236
+ if (record)
2237
+ return Promise.resolve(!isPrimaryScopeRow(record));
2238
+ return this.cp.getScopeRecord(tenantId, scopeId).then((row) => !isPrimaryScopeRow(row));
2006
2239
  }
2007
2240
  /**
2008
2241
  * The capability's attachment door (#1686) — `getCapabilityScope`'s gate, then the
@@ -2183,7 +2416,7 @@ export class CloudflareScopeHost {
2183
2416
  // #1819: a scope rewound to before its switch was pulled answers `on`, and the rewind held
2184
2417
  // the module outside the scope. Both are read by the system door's own gate (#1834), which
2185
2418
  // every fire below then goes through, pinned to the instance this read came from.
2186
- const gate = await this.systemDoorGate(scopeId, moduleId);
2419
+ const gate = await this.doorGate('system', scopeId, moduleId);
2187
2420
  if (gate.state === 'ungranted')
2188
2421
  return report;
2189
2422
  if (gate.error)
@@ -2258,10 +2491,11 @@ export class CloudflareScopeHost {
2258
2491
  /** The stub body, shared by the principal, connection, system and impersonation doors. */
2259
2492
  buildStub(tenantId, scopeId, principal, connectionId,
2260
2493
  /**
2261
- * #1834: the system door this stub acts through. The ONLY way to make a stub acting as
2494
+ * #1834: the door this stub acts through. The ONLY way to make a stub acting as
2262
2495
  * `system:<moduleId>`: the module is the door's, and every invoke is gated and pinned by it.
2496
+ * #2029: a peer stub's door too (`openDoor('peer', …)`), gating and pinning each invoke the same way.
2263
2497
  */
2264
- systemDoor, options,
2498
+ door, options,
2265
2499
  /**
2266
2500
  * K-42: the session this stub acts under. Re-read from the directory on
2267
2501
  * EVERY invoke rather than captured here — a stub is a capability and
@@ -2282,7 +2516,7 @@ export class CloudflareScopeHost {
2282
2516
  */
2283
2517
  verticalCaller) {
2284
2518
  const stub = this.scopeStub(scopeId);
2285
- const systemModuleId = systemDoor?.moduleId;
2519
+ const systemModuleId = door?.kind === 'system' ? door.key : undefined;
2286
2520
  const cp = this.cp;
2287
2521
  const operationEntitlement = this.operationEntitlement;
2288
2522
  // The DO needs SOME principal-shaped value for `ctx.principal`; for a
@@ -2338,10 +2572,10 @@ export class CloudflareScopeHost {
2338
2572
  // K-42: resolved per invoke, so expiry and `endImpersonation` both bite.
2339
2573
  const session = sessionId === undefined ? undefined : await this.resolveImpersonation(sessionId, tenantId, scopeId);
2340
2574
  const send = (systemDoorInstance) => stub.invoke(operation, input, asPrincipalId, tenantId, scopeId, connectionId, requiredKey, systemModuleId, true, invokeOptions, session, capabilitySession, verticalCaller, systemDoorInstance);
2341
- // #1834: through the system door, pinned to the instance its gate read. The DO's
2342
- // refusal comes back as a failure envelope, so it is rethrown for the door to re-gate.
2343
- const envelope = systemDoor
2344
- ? await systemDoor.through(async (instance) => {
2575
+ // #1834: through the door (a module's, or #2029 a peer's), pinned to the instance its gate
2576
+ // read. A missed pin comes back as an answer, which `through` re-gates on.
2577
+ const envelope = door
2578
+ ? await door.through(async (instance) => {
2345
2579
  const sent = await send(instance);
2346
2580
  return sent.systemDoorMoved === true ? SYSTEM_DOOR_MOVED : sent;
2347
2581
  })
@@ -2394,9 +2628,9 @@ export class CloudflareScopeHost {
2394
2628
  throw substratError('unavailable', `${operation} ran on a scope host that did not understand peer callers. ` +
2395
2629
  'Retry once the scope has been migrated');
2396
2630
  }
2397
- // #1834, the same shape: a DO too old to know the system door's pin ran the call
2398
- // without checking it met the storage the door's gate read.
2399
- if (systemDoor !== undefined && envelope.systemDoor?.honoured !== true) {
2631
+ // #1834, the same shape: a DO too old to know the door's pin ran the call without
2632
+ // checking it met the storage the door's gate read.
2633
+ if (door !== undefined && envelope.systemDoor?.honoured !== true) {
2400
2634
  throw substratError('unavailable', `${operation} ran on a scope host that did not understand the system door's pin. ` +
2401
2635
  'Retry once the scope has been migrated');
2402
2636
  }
@@ -2410,11 +2644,13 @@ export class CloudflareScopeHost {
2410
2644
  // this scope's outbox — and draining anyway would run executors and
2411
2645
  // connectors as a side effect of a session that may not have side effects.
2412
2646
  // Whatever the outbox already held is the drain sweep's own backstop.
2647
+ // #1184: what this call's executors did inline, for `onExecutorOutcomes`.
2648
+ const executorOutcomes = invokeOptions?.onExecutorOutcomes ? [] : undefined;
2413
2649
  const drained = session?.mode === 'read-only'
2414
2650
  ? { attempted: 0, delivered: 0, retrying: 0, deadLettered: 0, routedToPlatform: 0 }
2415
2651
  // #1525: this drain is part of the call that emitted the events — the
2416
2652
  // coordinator's half of the post-commit tail the DO ran the consumers in.
2417
- : await this.drainExecutors(tenantId, scopeId, invokeOptions?.invocationId ?? null);
2653
+ : await this.drainExecutors(tenantId, scopeId, invokeOptions?.invocationId ?? null, executorOutcomes);
2418
2654
  // #458: the operation committed having enqueued platform intents — tell the
2419
2655
  // caller's harness so it can flag the response for the router kick (#381).
2420
2656
  // Routed connector deliveries (#574 phase 3) count too: the inline drain just
@@ -2431,6 +2667,9 @@ export class CloudflareScopeHost {
2431
2667
  invokeOptions?.onIdempotentReplay?.();
2432
2668
  if (envelope.emitted)
2433
2669
  invokeOptions?.onEmitted?.(envelope.emitted);
2670
+ if (executorOutcomes && session?.mode !== 'read-only' && !envelope.idempotency?.replayed) {
2671
+ invokeOptions?.onExecutorOutcomes?.(executorOutcomes);
2672
+ }
2434
2673
  return envelope.result;
2435
2674
  },
2436
2675
  };
@@ -2440,8 +2679,12 @@ export class CloudflareScopeHost {
2440
2679
  }
2441
2680
  // -- admin surface --------------------------------------------------------
2442
2681
  /** #977: this host, with every admin row it writes naming who the actor acted for. */
2443
- attributed(onBehalfOf) {
2444
- return attributedHost(this, onBehalfOf, this.buildAdmin);
2682
+ attributed(onBehalfOf, options) {
2683
+ return attributedView(this, { onBehalfOf, causedBy: options?.causedBy }, this.buildAdmin);
2684
+ }
2685
+ /** #2055: this host, with every admin row it writes naming `eventId` as its cause. */
2686
+ causedByView(eventId) {
2687
+ return attributedView(this, { causedBy: eventId }, this.buildAdmin);
2445
2688
  }
2446
2689
  buildAdmin() {
2447
2690
  // The directory row → the `scope` contract. Parsed, not cast: the columns are
@@ -2523,7 +2766,7 @@ export class CloudflareScopeHost {
2523
2766
  */
2524
2767
  const requireOrg = async (tenant, id) => {
2525
2768
  if (!(await this.cp.readOrg(tenant, id))) {
2526
- throw new Error(`unknown org ${id} in tenant ${tenant}`);
2769
+ throw substratError('not_found', `unknown org ${id} in tenant ${tenant}`);
2527
2770
  }
2528
2771
  };
2529
2772
  const mapScope = (r) => scopeSchema.parse({
@@ -2610,94 +2853,20 @@ export class CloudflareScopeHost {
2610
2853
  }
2611
2854
  };
2612
2855
  /**
2613
- * #1666: move one module's schedule switch on one scope — see `HostAdmin.revokeFromSystem`.
2856
+ * Where one scope's kill switch of one kind lands — the ONE rule the operator's switch and
2857
+ * the #1674 re-assert share, so a re-assert moves exactly the switch an operator's OFF would
2858
+ * have. `kind` is a module's schedule switch (#1666) or a peer's (#1706, #2029); `key` is the
2859
+ * module id or the peer's slug.
2614
2860
  *
2615
2861
  * Where the write lands is the whole point of the delegation branch. The shared control
2616
- * plane's own `SCOPE` namespace is the module-less placeholder, so a hosted scope's
2617
- * switch has to be moved in the deployment serving it; writing it here would report a
2618
- * switch pulled while every schedule kept firing. The audit row is written HERE either
2619
- * way — the deployment's host is CP-less, and its `recordAdmin` is a no-op.
2620
- */
2621
- /**
2622
- * #1706: move one PEER's kill switch on one scope — see `HostAdmin.revokeFromPeer`. The
2623
- * schedule switch's audit discipline (intent first, outcome after, every attempt) and,
2624
- * since part 3, its delegation: a scope served by a vertical's own deployment is switched
2625
- * THERE, over `PeerSwitchDelegation`. Without one configured the scope is still refused
2626
- * `unavailable` (`peerScopeGate`) rather than switched in the placeholder namespace.
2627
- */
2628
- const switchPeerAt = async (actor, raw, to) => {
2629
- const input = peerSwitch.parse(raw);
2630
- const { tenantId, scopeId } = input.node;
2631
- let vertical = null;
2632
- if (!this.cpLess) {
2633
- const rec = await this.cp.getScopeRecord(tenantId, scopeId);
2634
- if (!rec)
2635
- throw unknownScopeForTenant(tenantId, scopeId);
2636
- vertical = rec.vertical;
2637
- }
2638
- const action = to === 'off' ? 'revokeFromPeer' : 'restoreToPeer';
2639
- // Where the write lands is the whole point of this branch (#1706 part 3, mirroring
2640
- // #1666). A scope bound to a vertical is served by that vertical's own deployment, and
2641
- // its `vertical:<slug>` grants live there; switching it in the shared control plane's
2642
- // placeholder namespace would tombstone nothing and report a peer cut off while every
2643
- // call it makes kept being admitted. A scope bound to NO vertical has its store here,
2644
- // so it switches here — delegating it would throw "no deployment serving scope".
2645
- // With no delegation configured the old refusal stands, loudly, rather than a write
2646
- // into the wrong namespace.
2647
- const delegation = this.cpLess || vertical !== null ? this.peerSwitchDelegation : undefined;
2648
- if (delegation) {
2649
- // The gate's lifecycle half still runs (a suspended tenant or scope cannot be
2650
- // switched); its `scopeStub` half is what the delegation replaces.
2651
- await this.validateScopeAccess(tenantId, scopeId);
2652
- }
2653
- else {
2654
- await this.peerScopeGate(tenantId, scopeId, action);
2655
- }
2656
- const operationId = ulid();
2657
- const target = { tenantId, scopeId, vertical };
2658
- const base = { operationId, vertical: input.vertical, calls: to };
2659
- await this.recordAdmin(actor, action, target, null, { ...base, phase: 'intent', reason: input.reason });
2660
- let outcome;
2661
- try {
2662
- outcome = delegation
2663
- ? await delegation.switch({ tenantId, scopeId, vertical: input.vertical, to })
2664
- : await this.scopeStub(scopeId).switchPeer(input.vertical, scopeId, to, new Date().toISOString());
2665
- }
2666
- catch (err) {
2667
- await this.recordAdmin(actor, action, target, null, {
2668
- ...base,
2669
- phase: 'failed',
2670
- error: err instanceof Error ? err.message : String(err),
2671
- }).catch(() => undefined);
2672
- throw err;
2673
- }
2674
- await this.recordAdmin(actor, action, target, null, {
2675
- ...base,
2676
- phase: outcome.held ? 'applied' : 'refused',
2677
- changed: outcome.changed,
2678
- permissions: outcome.permissions,
2679
- });
2680
- if (!outcome.held) {
2681
- throw substratError('not_found', `scope ${scopeId} holds no grant for peer '${input.vertical}' — nothing to switch ${to} ` +
2682
- `(check the slug: it is the calling vertical's registry id, as the target's \`peers\` names it)`);
2683
- }
2684
- return {
2685
- operationId,
2686
- vertical: input.vertical,
2687
- calls: to,
2688
- changed: outcome.changed,
2689
- permissions: outcome.permissions,
2690
- };
2691
- };
2692
- /**
2693
- * Where one scope's schedule switch lands — the ONE rule `switchSystem` and the #1674
2694
- * re-assert share, so a re-assert moves exactly the switch an operator's OFF would have.
2695
- * A scope bound to no vertical (#1666 review) has no deployment to delegate to: its store
2696
- * is the DO here, so the switch moves (or answers `held: false`) here too. Delegating it
2697
- * would throw "no deployment serving scope" instead. A scope WITH a vertical still
2698
- * delegates, and still fails loudly when none serves it.
2862
+ * plane's own `SCOPE` namespace is the module-less placeholder, so a hosted scope's switch
2863
+ * has to be moved in the deployment serving it, over that kind's delegation; writing it here
2864
+ * would report a switch pulled while every schedule kept firing, or every peer call kept
2865
+ * being admitted. A scope bound to no vertical (#1666 review) has no deployment to delegate
2866
+ * to: its store is the DO here, so the switch moves (or answers `held: false`) here too.
2867
+ * Delegating it would throw "no deployment serving scope" instead.
2699
2868
  */
2700
- const systemSwitchTarget = async (tenantId, scopeId) => {
2869
+ const switchTarget = async (kind, tenantId, scopeId) => {
2701
2870
  let vertical = null;
2702
2871
  if (!this.cpLess) {
2703
2872
  const rec = await this.cp.getScopeRecord(tenantId, scopeId);
@@ -2705,66 +2874,109 @@ export class CloudflareScopeHost {
2705
2874
  throw unknownScopeForTenant(tenantId, scopeId);
2706
2875
  vertical = rec.vertical;
2707
2876
  }
2708
- const delegation = this.cpLess || vertical !== null ? this.systemSwitchDelegation : undefined;
2877
+ const hosted = this.cpLess || vertical !== null;
2878
+ const systemDelegation = kind === 'system' && hosted ? this.systemSwitchDelegation : undefined;
2879
+ const peerDelegation = kind === 'peer' && hosted ? this.peerSwitchDelegation : undefined;
2880
+ const delegated = systemDelegation !== undefined || peerDelegation !== undefined;
2709
2881
  // #1819: a scope whose store is here was rewound here, so `switchInScope` releases its
2710
2882
  // hold here too. A delegated move releases in the deployment serving the scope.
2711
- // #1823: a module whose only authority on the scope is a TENANT-level grant has nothing in
2712
- // the scope's storage for the switch to find, so the directory says whether it is held.
2713
- // A caller moving several modules reads them in one call and passes each answer in.
2714
- // An OFF of a tenant-held module is refused, after the far end answers, unless that answer
2883
+ // #1823 (#2030 for a peer): a subject whose only authority on the scope is a TENANT-level
2884
+ // grant has nothing in the scope's storage for the switch to find, so the directory says
2885
+ // whether it is held. A caller moving several reads them in one call and passes each in.
2886
+ // An OFF of a tenant-held MODULE is refused, after the far end answers, unless that answer
2715
2887
  // attests the evaluator that denies a tenant-level grant (`deniesTenantGrants`). A
2716
2888
  // deployment built before #1823 drops `tenantHeld`, still answers `held: true` (the module's
2717
2889
  // scope-level grants were there to switch), and keeps authorizing the tenant-level grant —
2718
- // so its `held` alone would have the platform record a scope off that is not. Every OFF the
2719
- // platform drives moves through here: an operator's (`switchSystem`, which then puts the
2720
- // switch back) and a re-assert's, which every carry ends with.
2721
- const move = async (moduleId, to, at, tenantHeld) => {
2722
- const held = tenantHeld ?? (await this.cp.tenantHeldSystemModules(tenantId, [moduleId], at)).length > 0;
2723
- const outcome = await (delegation
2724
- ? delegation.switch({ tenantId, scopeId, moduleId, to, tenantHeld: held })
2725
- : this.switchInScope(scopeId, moduleId, to, at, held));
2726
- if (to === 'off' && held && outcome.held && outcome.deniesTenantGrants !== true) {
2727
- throw new UnattestedSwitchOff(scopeId, moduleId, outcome);
2890
+ // so its `held` alone would have the platform record a scope off that is not. A peer needs
2891
+ // no attestation: its door refuses on the marker in every build that has the peer switch at
2892
+ // all (`admitPeer` and `switchPeer` arrived together, #1714). Every OFF the platform drives
2893
+ // moves through here: an operator's (`switchSubjectAt`) and a re-assert's, which every carry
2894
+ // ends with.
2895
+ //
2896
+ // #2045 (Codex r3): whether a delegated deployment honours the switch fence is settled
2897
+ // BEFORE anything moves (`attestFence`, once per target), never after the fact by putting a
2898
+ // move back. Without the fence an ON is refused and an OFF goes through unfenced (see
2899
+ // `UnattestedSwitch`). A move a fenced deployment answers without the attestation still
2900
+ // throws, as defence in depth against a rollback between the preflight and the move.
2901
+ const delegation = systemDelegation ?? peerDelegation;
2902
+ let attested;
2903
+ const attestFence = () => (attested ??= delegation ? delegation.fenceSupported({ tenantId, scopeId }) : Promise.resolve(true));
2904
+ const move = async (key, to, at, tenantHeld,
2905
+ /** #2045: the switch call's fence — its operation id, or for a re-assert the record's. */
2906
+ fence) => {
2907
+ const fenced = await attestFence();
2908
+ if (!fenced && to === 'on')
2909
+ throw new UnattestedSwitch(unfencedMessage(scopeId));
2910
+ const held = tenantHeld ?? (await this.cp.tenantHeldOf(kind, tenantId, [key], at)).length > 0;
2911
+ const outcome = await (systemDelegation
2912
+ ? systemDelegation.switch({ tenantId, scopeId, moduleId: key, to, tenantHeld: held, fence })
2913
+ : peerDelegation
2914
+ ? peerDelegation.switch({ tenantId, scopeId, vertical: key, to, tenantHeld: held, fence })
2915
+ : this.switchInScope(kind, scopeId, key, to, at, held, fence));
2916
+ // An unfenced OFF answers without `fenced`, and that is what the callers read it by.
2917
+ if (fenced && outcome.fenced !== true)
2918
+ throw new UnattestedSwitch(unfencedMessage(scopeId));
2919
+ if (outcome.superseded)
2920
+ return outcome;
2921
+ if (kind === 'system' && to === 'off' && held && outcome.held && outcome.deniesTenantGrants !== true) {
2922
+ throw new UnattestedSwitch(undeniedMessage(scopeId, key));
2728
2923
  }
2729
2924
  return outcome;
2730
2925
  };
2731
- /**
2732
- * Is the module switched off on the scope right now? Read from wherever `move` writes — the
2733
- * deployment's status read or this scope's own — for a move that threw (#1823): the switch
2734
- * may already have moved before the throw (`switchInScope` moves it, then works the rewind
2735
- * hold), and then the directory's record must stay.
2736
- */
2737
- const isOff = async (moduleId) => {
2738
- const rows = delegation
2739
- ? await delegation.status({ tenantId, scopeId })
2740
- : systemScheduleEntry.array().parse(await this.scopeStub(scopeId).systemGrantsStatus());
2741
- return rows.some((r) => r.moduleId === moduleId && r.schedules === 'off');
2742
- };
2743
- return { vertical, move, isOff };
2926
+ return { vertical, delegated, move, attestFence };
2744
2927
  };
2745
- const switchSystem = async (actor, raw, to) => {
2746
- const input = systemSwitch.parse(raw);
2747
- const { tenantId, scopeId } = input.node;
2748
- const { vertical, move, isOff } = await systemSwitchTarget(tenantId, scopeId);
2928
+ /**
2929
+ * One kill switch moved on one scope — the ONE body `revokeFromSystem` / `restoreToSystem`
2930
+ * (#1666) and `revokeFromPeer` / `restoreToPeer` (#1706) share, so the two levers cannot
2931
+ * disagree about the record, the audit, the readback or the undo (#2029). The audit row is
2932
+ * written HERE either way — a deployment's host is CP-less, and its `recordAdmin` is a no-op.
2933
+ */
2934
+ const switchSubjectAt = async (actor, kind, node, key, reason, to) => {
2935
+ const { tenantId, scopeId } = node;
2936
+ const { vertical, delegated, move, attestFence } = await switchTarget(kind, tenantId, scopeId);
2937
+ const action = switchActionOf(kind, to);
2938
+ if (kind === 'peer') {
2939
+ // The peer door's gate (#1706): with a delegation, its lifecycle half (a suspended tenant
2940
+ // or scope cannot be switched), the `scopeStub` half being what the delegation replaces.
2941
+ // Without one, a hosted scope is refused `unavailable` rather than switched in the
2942
+ // placeholder namespace, where it would tombstone nothing a peer call reads.
2943
+ if (delegated)
2944
+ await this.validateScopeAccess(tenantId, scopeId);
2945
+ else
2946
+ await this.peerScopeGate(tenantId, scopeId, action);
2947
+ }
2749
2948
  // AUDIT FIRST (#1666 review): the intent row lands before anything moves, and the
2750
2949
  // outcome row after — every attempt, a repeat included. The scope's store and the
2751
2950
  // admin log are separate, so no order makes the pair atomic; this one fails toward
2752
2951
  // "an intent with no recorded outcome" and never toward "a switch that moved with no
2753
2952
  // audit row". A retry after a crash re-audits even though it answers `changed: false`.
2754
2953
  const operationId = ulid();
2755
- const action = to === 'off' ? 'revokeFromSystem' : 'restoreToSystem';
2756
2954
  const target = { tenantId, scopeId, vertical };
2757
- const base = { operationId, moduleId: input.moduleId, schedules: to };
2758
- await this.recordAdmin(actor, action, target, null, { ...base, phase: 'intent', reason: input.reason });
2759
- // The directory's record (#1674), written BEFORE the scope moves, both ways, and undone
2760
- // if the move throws or holds nothing — see `recordSystemSwitchedOn` and (#1823)
2761
- // `recordSystemSwitchedOff` for why that order is the safe one.
2762
- const at = new Date().toISOString();
2763
- const record = { tenantId, scopeId, moduleId: input.moduleId, actor, reason: input.reason, operationId, at };
2955
+ const base = { operationId, ...switchAuditSubject(kind, key, to) };
2956
+ await this.recordAdmin(actor, action, target, null, { ...base, phase: 'intent', reason });
2764
2957
  const errorOf = (err) => (err instanceof Error ? err.message : String(err));
2958
+ // #2045 (Codex r3): the deployment's fence is attested BEFORE anything is recorded or moved. An
2959
+ // ON on a deployment built before it is refused here, with nothing written anywhere but the
2960
+ // audit. An OFF goes on: the kill switch must work on every deployment (see `UnattestedSwitch`).
2961
+ try {
2962
+ if (!(await attestFence()) && to === 'on') {
2963
+ throw substratError('precondition_failed', `${unfencedMessage(scopeId)} Nothing was switched.`);
2964
+ }
2965
+ }
2966
+ catch (err) {
2967
+ await this.recordAdmin(actor, action, target, null, { ...base, phase: 'refused', error: errorOf(err) }).catch(() => undefined);
2968
+ throw err;
2969
+ }
2970
+ // The directory's record (#1674; #2029 for a peer), written BEFORE the scope moves, both
2971
+ // ways — see `recordSwitchedOn` and (#1823) `recordSwitchedOff` for why that order is the
2972
+ // safe one. #2045 (Codex r3): the same write marks the subject owed and clears the scope's
2973
+ // reconcile receipt, so from here on the scope converges to this record whatever happens to
2974
+ // the move. The mark goes only when the scope confirms the move under this call's fence.
2975
+ const at = new Date().toISOString();
2976
+ const record = { kind, tenantId, scopeId, key, actor, reason, operationId, at };
2765
2977
  let prior;
2766
2978
  try {
2767
- prior = to === 'on' ? await this.cp.recordSystemSwitchedOn(record) : await this.cp.recordSystemSwitchedOff(record);
2979
+ prior = to === 'on' ? await this.cp.recordSwitchedOn(record) : await this.cp.recordSwitchedOff(record);
2768
2980
  }
2769
2981
  catch (err) {
2770
2982
  // Nothing has moved: fail the call here, audited. An ON must not switch a scope on
@@ -2773,92 +2985,92 @@ export class CloudflareScopeHost {
2773
2985
  await this.recordAdmin(actor, action, target, null, { ...base, phase: 'failed', error: errorOf(err) }).catch(() => undefined);
2774
2986
  throw err;
2775
2987
  }
2776
- /**
2777
- * The record's undo, after the scope's move threw or held nothing: retried once, and a
2778
- * failure of both is answered rather than swallowed — it lands on the outcome row as
2779
- * `recordError`, and the call says so.
2780
- */
2781
- const undoRecord = async () => {
2782
- const undo = () => this.cp.restoreSystemSwitchRecord(record, prior);
2988
+ // #2045: a newer call on this subject has recorded its position already: this one writes
2989
+ // nothing, here or in the scope, and says so.
2990
+ if (recordWriteSuperseded(prior, record)) {
2991
+ await this.recordAdmin(actor, action, target, null, { ...base, phase: 'refused', superseded: true });
2992
+ throw substratError('conflict', switchSupersededMessage(kind, scopeId, key, to));
2993
+ }
2994
+ let outcome;
2995
+ try {
2783
2996
  try {
2784
- await undo();
2785
- return null;
2997
+ outcome = await move(key, to, at, undefined, operationId);
2786
2998
  }
2787
- catch {
2788
- try {
2789
- await undo();
2790
- return null;
2791
- }
2792
- catch (err) {
2793
- return errorOf(err);
2794
- }
2999
+ catch (first) {
3000
+ if (first instanceof UnattestedSwitch)
3001
+ throw first;
3002
+ // #2045 (Codex r2): one retry under this call's own fence — idempotent, and refused by
3003
+ // the scope if a newer call has moved it meanwhile.
3004
+ outcome = await move(key, to, new Date().toISOString(), undefined, operationId);
2795
3005
  }
2796
- };
2797
- let outcome;
2798
- try {
2799
- outcome = await move(input.moduleId, to, at);
2800
3006
  }
2801
3007
  catch (err) {
2802
- if (err instanceof UnattestedSwitchOff) {
2803
- // The far end switched, and cannot hold it against the tenant-level grant. Refused:
2804
- // ON gives back exactly what this OFF took (nothing, when it changed nothing), and
2805
- // then the record goes back too — neither the scope nor the directory says off. An ON
2806
- // that fails here leaves the scope off, and the readback rule below keeps its record.
2807
- const putBack = err.outcome.changed
2808
- ? await move(input.moduleId, 'on', new Date().toISOString(), true).then(() => true, () => false)
2809
- : true;
2810
- if (putBack) {
2811
- const recordError = await undoRecord();
2812
- await this.recordAdmin(actor, action, target, null, {
2813
- ...base,
2814
- phase: 'refused',
2815
- error: err.message,
2816
- ...(recordError ? { recordError } : {}),
2817
- }).catch(() => undefined);
2818
- throw substratError('precondition_failed', `${err.message} Nothing was switched` +
2819
- (recordError ? `, but its directory record could not be put back (${recordError}).` : '.'));
2820
- }
2821
- }
2822
- // A throw does not say the scope did not move: the move crosses the scope's DO and then
2823
- // the rewind hold, and a throw from the second leaves the scope's switch moved. So the
2824
- // record is undone only when the scope reads back in the OTHER position — an OFF that
2825
- // reads not off, an ON that still reads off — and kept when it reads back as asked.
2826
- // Undoing an ON that landed would put the record back to `off` beside a scope that is
2827
- // on, and the next re-assert would switch the operator's ON back off. A read that fails
2828
- // too counts as off, both ways, so what is left fails toward off: an OFF keeps its
2829
- // record (never a scope switched off with no record — a wipe or restore would have
2830
- // nothing to re-assert, and #1743's refusal nothing to read), and an ON is undone back
2831
- // to `off`, which the next re-assert completes if the scope did move.
2832
- const off = await isOff(input.moduleId).catch(() => true);
2833
- const recordKept = off === (to === 'off');
2834
- const recordError = recordKept ? null : await undoRecord();
2835
- // Best effort: the original error is what the caller must see, and the intent row
2836
- // already says an attempt was made.
3008
+ // The move threw twice, or was answered by a deployment that could not attest it. Either
3009
+ // way it may have landed, and an older call's move may still be in flight behind it, so no
3010
+ // readback can settle the scope and nothing is put back: the record is this call's position
3011
+ // under the newest fence, and its write-ahead mark makes the next re-assert move the scope
3012
+ // to it, under that fence, whichever way it lies now.
2837
3013
  await this.recordAdmin(actor, action, target, null, {
2838
3014
  ...base,
2839
- phase: 'failed',
3015
+ phase: err instanceof UnattestedSwitch ? 'refused' : 'failed',
2840
3016
  error: errorOf(err),
2841
- ...(recordKept ? { recordKept: true } : {}),
2842
- ...(recordError ? { recordError } : {}),
3017
+ recordKept: true,
3018
+ reassertOwed: true,
2843
3019
  }).catch(() => undefined);
3020
+ if (err instanceof UnattestedSwitch)
3021
+ throw substratError('precondition_failed', err.message);
2844
3022
  throw err;
2845
3023
  }
3024
+ // #2045: the scope has applied a newer call on this subject, so this move wrote nothing. The
3025
+ // record is that newer call's too (its write overwrote this one's, mark included), so nothing
3026
+ // is undone and no mark is this call's to clear.
3027
+ if (outcome.superseded) {
3028
+ await this.recordAdmin(actor, action, target, null, { ...base, phase: 'refused', superseded: true });
3029
+ throw substratError('conflict', switchSupersededMessage(kind, scopeId, key, to));
3030
+ }
2846
3031
  // A call that held nothing moved nothing, so its record write is undone: left `off`, it
2847
- // would switch the module off the day it is installed; left `on`, the next reconcile of
2848
- // a wiped scope would leave the module running.
2849
- const recordError = outcome.held ? null : await undoRecord();
3032
+ // would switch the subject off the day it is installed; left `on`, the next reconcile of
3033
+ // a wiped scope would leave it running. Retried once; a failure of both lands on the
3034
+ // outcome row as `recordError`, and the call says so.
3035
+ //
3036
+ // The scope confirmed this call's move (or that there was nothing to move), so its mark goes.
3037
+ // Held nothing, it goes WITH the undo, in one directory transaction: the row put back carries
3038
+ // the prior call's older id, and no later re-assert confirms under a fence older than this
3039
+ // call's mark. An undo that fails twice leaves the record AND the mark this call's, and the
3040
+ // next re-assert confirms both under it. Held, a failed clear is not the call's: the mark only
3041
+ // costs the next re-assert one idempotent move under this same record.
3042
+ //
3043
+ // An unfenced OFF that held something (a deployment built before the fence) confirmed nothing
3044
+ // under its fence, so its mark stays for the first re-assert after the vertical is redeployed.
3045
+ const unfenced = outcome.held && outcome.fenced !== true;
3046
+ let recordError = null;
3047
+ let owedError = null;
3048
+ if (!outcome.held) {
3049
+ const undo = () => this.cp.restoreSwitchRecord(record, prior, true);
3050
+ recordError = await undo().then(() => null, () => undo().then(() => null, errorOf));
3051
+ }
3052
+ else if (!unfenced) {
3053
+ owedError = await this.cp.clearSwitchOwed(kind, tenantId, scopeId, key, operationId).then(() => null, errorOf);
3054
+ }
2850
3055
  await this.recordAdmin(actor, action, target, null, {
2851
3056
  ...base,
2852
3057
  phase: outcome.held ? 'applied' : 'refused',
2853
3058
  changed: outcome.changed,
2854
3059
  permissions: outcome.permissions,
3060
+ ...(unfenced ? { unfenced: true, reassertOwed: true } : {}),
2855
3061
  ...(recordError ? { recordError } : {}),
3062
+ ...(owedError ? { owedError } : {}),
2856
3063
  });
2857
3064
  if (!outcome.held) {
2858
- throw substratError('not_found', `scope ${scopeId} holds no system grant for module '${input.moduleId}' — nothing to switch ${to} ` +
2859
- `(check the module id: it is the module's manifest id, e.g. '@substrat-run/engine-absence')` +
3065
+ throw substratError('not_found', switchNotFoundMessage(kind, scopeId, key, to) +
2860
3066
  (recordError ? `; and its directory record could not be put back (${recordError})` : ''));
2861
3067
  }
3068
+ return { operationId, outcome };
3069
+ };
3070
+ /** #1666: move one module's schedule switch on one scope — see `HostAdmin.revokeFromSystem`. */
3071
+ const switchSystem = async (actor, raw, to) => {
3072
+ const input = systemSwitch.parse(raw);
3073
+ const { operationId, outcome } = await switchSubjectAt(actor, 'system', input.node, input.moduleId, input.reason, to);
2862
3074
  return {
2863
3075
  operationId,
2864
3076
  moduleId: input.moduleId,
@@ -2867,6 +3079,18 @@ export class CloudflareScopeHost {
2867
3079
  permissions: outcome.permissions,
2868
3080
  };
2869
3081
  };
3082
+ /** #1706: move one PEER's kill switch on one scope — see `HostAdmin.revokeFromPeer`. */
3083
+ const switchPeerAt = async (actor, raw, to) => {
3084
+ const input = peerSwitch.parse(raw);
3085
+ const { operationId, outcome } = await switchSubjectAt(actor, 'peer', input.node, input.vertical, input.reason, to);
3086
+ return {
3087
+ operationId,
3088
+ vertical: input.vertical,
3089
+ calls: to,
3090
+ changed: outcome.changed,
3091
+ permissions: outcome.permissions,
3092
+ };
3093
+ };
2870
3094
  /**
2871
3095
  * The status read's admin-log join (#1674): the `intent` row of the `revokeFromSystem`
2872
3096
  * still in force for each OFF module — actor, reason, when. Paired to its `applied` row
@@ -2939,7 +3163,7 @@ export class CloudflareScopeHost {
2939
3163
  delegation
2940
3164
  ? delegation.status({ tenantId, scopeId })
2941
3165
  : this.scopeStub(scopeId).systemGrantsStatus().then((rows) => systemScheduleEntry.array().parse(rows)),
2942
- this.cp.systemSwitchRecordsOf(tenantId, scopeId),
3166
+ this.cp.switchRecordsOf('system', tenantId, scopeId),
2943
3167
  ]);
2944
3168
  const offModules = new Set(states.filter((s) => s.schedules === 'off').map((s) => s.moduleId));
2945
3169
  const explanations = offModules.size > 0
@@ -2958,71 +3182,137 @@ export class CloudflareScopeHost {
2958
3182
  };
2959
3183
  /**
2960
3184
  * #1674: put the directory's OFF positions back into one scope — see
2961
- * `HostAdmin.reassertSystemSwitches`. It moves the switch through `systemSwitchTarget`,
2962
- * so it lands wherever an operator's OFF would have — for a hosted scope, over the
2963
- * `/internal/system-switch` route every deployment since #1666 answers.
3185
+ * `HostAdmin.reassertSystemSwitches`. Every kind (#2029): the recorded-off modules, then the
3186
+ * recorded-off peers. Each moves through `switchTarget`, so it lands wherever an operator's
3187
+ * OFF would have — for a hosted scope, over that kind's `/internal/*` switch route.
2964
3188
  */
2965
3189
  const reassertSystemSwitchesOf = async (actor, node, opts) => {
3190
+ const results = [];
3191
+ for (const kind of SWITCH_KINDS)
3192
+ results.push(...(await reassertKind(actor, kind, node, opts)));
3193
+ return results;
3194
+ };
3195
+ /** One kind's half of `reassertSystemSwitchesOf`. */
3196
+ const reassertKind = async (actor, kind, node, opts) => {
2966
3197
  const { tenantId, scopeId } = node;
2967
- const { vertical, move } = await systemSwitchTarget(tenantId, scopeId);
2968
- const recorded = new Map(await this.cp.systemSwitchRecordsOf(tenantId, scopeId));
3198
+ const { vertical, delegated, move, attestFence } = await switchTarget(kind, tenantId, scopeId);
3199
+ const action = reassertActionOf(kind);
3200
+ const target = { tenantId, scopeId, vertical };
3201
+ const recorded = new Map(await this.cp.switchRecordsOf(kind, tenantId, scopeId));
3202
+ // #2045: every switch call's write-ahead mark, with the newest call that wrote it.
3203
+ const owed = new Map(await this.cp.switchesOwedOf(kind, tenantId, scopeId));
2969
3204
  // #1742 review: moves the deployment made from a stale list (restored ON after the list
2970
3205
  // was read), to undo before the OFF pass below.
2971
- const reverts = staleCarryReverts(recorded, opts?.appliedInUnit);
3206
+ const reverts = staleCarryReverts(kind, recorded, opts?.appliedInUnit);
2972
3207
  // A hosted scope with no delegation configured (Copilot review): this host's own
2973
3208
  // namespace is the module-less placeholder, where the switch would answer `held: false`
2974
3209
  // quietly — a re-assert reported done, and a receipt written, while the deployment
2975
- // serving the scope keeps its schedules running. Refused loudly instead, as the status
2976
- // read is, and only when a re-assert is owed: nothing recorded, nothing to refuse.
2977
- const owed = reverts.length > 0 || [...recorded.values()].includes('off');
2978
- if (owed && !this.cpLess && vertical !== null && !this.systemSwitchDelegation) {
3210
+ // serving the scope keeps its schedules running or its peer admitted. Refused loudly
3211
+ // instead, as the status read is, and only when a re-assert is owed: nothing recorded,
3212
+ // nothing to refuse.
3213
+ const reassertOwed = reverts.length > 0 || owed.size > 0 || [...recorded.values()].includes('off');
3214
+ if (reassertOwed && !this.cpLess && vertical !== null && !delegated) {
2979
3215
  throw substratError('unavailable', `no delegation configured for hosted scope ${scopeId} (vertical '${vertical}') — cannot re-assert ` +
2980
- `its switched-off schedules in the deployment serving it`);
3216
+ `its switched-off ${DOOR_WORDS[kind].switched} in the deployment serving it`);
2981
3217
  }
3218
+ // #2045 (Codex r3): a subject's mark goes only once the scope has confirmed a move under the
3219
+ // record's fence — a fenced answer that was not superseded. A throw anywhere below leaves
3220
+ // every mark not yet confirmed for the next pass.
3221
+ // An unfenced OFF (a deployment built before the fence) confirms nothing, so its mark stays.
3222
+ const confirm = async (key, fence, outcome) => {
3223
+ if (owed.has(key) && !outcome.superseded && outcome.fenced === true) {
3224
+ await this.cp.clearSwitchOwed(kind, tenantId, scopeId, key, fence);
3225
+ }
3226
+ };
3227
+ // #2045 follow-up: on a deployment built before the fence no ON is sent (see
3228
+ // `UnattestedSwitch`). Every OFF below still moves, and the pass refuses at its end, so a
3229
+ // sweep records no receipt for a scope whose owed ON was not sent. Asked only when an ON is due.
3230
+ const onWithheld = [];
3231
+ const withholdOn = async (key) => {
3232
+ if (await attestFence())
3233
+ return false;
3234
+ onWithheld.push(key);
3235
+ return true;
3236
+ };
2982
3237
  const at = new Date().toISOString();
2983
3238
  const reverted = new Set();
2984
- for (const moduleId of reverts) {
3239
+ for (const key of reverts) {
2985
3240
  // Re-read immediately before the move: a staff OFF that completed since the read above
2986
- // (record `off`) must not get a transient ON a due schedule could run in.
2987
- const current = new Map(await this.cp.systemSwitchRecordsOf(tenantId, scopeId));
2988
- if (current.get(moduleId) !== 'on')
3241
+ // (record `off`) must not get a transient ON a due schedule, or a peer call, could use.
3242
+ // #2045: under the fence of the ON the record holds, which is the operator's own call — read
3243
+ // with the position, so it is that ON's fence and no other call's.
3244
+ const current = new Map(await this.cp.switchRecordStatesOf(kind, tenantId, scopeId)).get(key);
3245
+ if (current?.position !== 'on')
2989
3246
  continue;
2990
- const outcome = await move(moduleId, 'on', at);
2991
- reverted.add(moduleId);
3247
+ const fence = current.fence;
3248
+ if (await withholdOn(key))
3249
+ continue;
3250
+ const outcome = await move(key, 'on', at, undefined, fence);
3251
+ reverted.add(key);
2992
3252
  if (outcome.changed) {
2993
- await this.recordAdmin(actor, 'reassertSystemSwitch', { tenantId, scopeId, vertical }, null, {
2994
- operationId: ulid(),
2995
- ...staleCarryRevertRow(moduleId, outcome),
2996
- });
3253
+ await this.recordAdmin(actor, action, target, null, { operationId: ulid(), ...staleCarryRevertRow(kind, key, outcome) });
2997
3254
  }
3255
+ await confirm(key, fence, outcome);
2998
3256
  }
2999
3257
  // Read AFTER the reverts: an OFF that landed meanwhile is switched off below.
3000
- const modules = await this.cp.switchedOffModulesOf(tenantId, scopeId);
3258
+ const { keys, tenantHeld, fences } = await this.recordedOff(kind, tenantId, scopeId, at);
3001
3259
  // #1742: what the deployment already switched off inside its own unit, audited here —
3002
- // the move below answers `changed: false` for it and would write no row.
3003
- // A move the revert above undid is not credited as an in-unit OFF.
3004
- const applied = opts?.appliedInUnit?.filter((a) => !reverted.has(a.moduleId));
3005
- for (const row of inUnitMovesToAudit(modules, applied)) {
3006
- await this.recordAdmin(actor, 'reassertSystemSwitch', { tenantId, scopeId, vertical }, null, {
3007
- operationId: ulid(),
3008
- ...row,
3009
- });
3260
+ // the move below answers `changed: false` for it and would write no row. A move the
3261
+ // revert above undid is not credited as an in-unit OFF.
3262
+ for (const row of inUnitMovesToAudit(kind, keys, opts?.appliedInUnit, reverted)) {
3263
+ await this.recordAdmin(actor, action, target, null, { operationId: ulid(), ...row });
3010
3264
  }
3265
+ // #2045: every move here carries the fence of the call the record holds, so a scope a newer call
3266
+ // has moved since refuses it. A deployment from before the fence takes the OFF unfenced.
3011
3267
  const results = [];
3012
- const tenantHeld = new Set(modules.length ? await this.cp.tenantHeldSystemModules(tenantId, modules, at) : []);
3013
- for (const moduleId of modules) {
3014
- const outcome = await move(moduleId, 'off', at, tenantHeld.has(moduleId));
3268
+ const held = new Set(tenantHeld);
3269
+ for (const key of keys) {
3270
+ const fence = fences.get(key); // `recordedOff` read each key with its own row's fence
3271
+ const outcome = await move(key, 'off', at, held.has(key), fence);
3015
3272
  if (outcome.changed) {
3016
- await this.recordAdmin(actor, 'reassertSystemSwitch', { tenantId, scopeId, vertical }, null, {
3273
+ await this.recordAdmin(actor, action, target, null, {
3017
3274
  operationId: ulid(),
3018
- moduleId,
3019
- schedules: 'off',
3020
- phase: 'applied',
3021
- changed: true,
3022
- permissions: outcome.permissions,
3275
+ ...reassertOffRow(kind, key, outcome.permissions),
3276
+ ...(outcome.fenced === true ? {} : { unfenced: true }),
3023
3277
  });
3024
3278
  }
3025
- results.push({ moduleId, held: outcome.held, changed: outcome.changed });
3279
+ results.push(reassertEntry(kind, key, outcome));
3280
+ await confirm(key, fence, outcome);
3281
+ }
3282
+ // A subject marked owed converges to its record in either direction: OFF above, and here a
3283
+ // record of ON — sent whatever the scope reads now, because a scope that reads ON may still
3284
+ // be behind the record's fence, with an older call's OFF in flight that only the fence can
3285
+ // refuse (Codex r3, finding 1). Idempotent: an ON already in place changes nothing and moves
3286
+ // the fence. Every other record of ON keeps #1674's rule and turns nothing on.
3287
+ const settled = new Set([...keys, ...reverted]);
3288
+ const pending = [...owed.keys()].filter((key) => !settled.has(key));
3289
+ if (pending.length) {
3290
+ // Position and fence in one read: an ON is sent only under the fence of the ON it read.
3291
+ const now = new Map(await this.cp.switchRecordStatesOf(kind, tenantId, scopeId));
3292
+ for (const key of pending) {
3293
+ const row = now.get(key);
3294
+ if (row?.position !== 'on') {
3295
+ // No record (a call that held nothing, whose undo removed its row), or one that turned
3296
+ // OFF since the read above, which the next pass moves: nothing to send on this one. A
3297
+ // missing row's mark goes under its own id, so a newer call's mark stays.
3298
+ if (row === undefined)
3299
+ await this.cp.clearSwitchOwed(kind, tenantId, scopeId, key, owed.get(key));
3300
+ continue;
3301
+ }
3302
+ const fence = row.fence;
3303
+ if (await withholdOn(key))
3304
+ continue;
3305
+ const outcome = await move(key, 'on', at, undefined, fence);
3306
+ if (outcome.changed) {
3307
+ await this.recordAdmin(actor, action, target, null, { operationId: ulid(), ...reassertOnRow(kind, key, outcome.permissions) });
3308
+ results.push(reassertEntry(kind, key, outcome));
3309
+ }
3310
+ await confirm(key, fence, outcome);
3311
+ }
3312
+ }
3313
+ if (onWithheld.length) {
3314
+ throw substratError('precondition_failed', `${unfencedMessage(scopeId)} Its recorded OFF positions were applied; the owed ON of ` +
3315
+ `${onWithheld.map((k) => `'${k}'`).join(', ')} was not sent.`);
3026
3316
  }
3027
3317
  return results;
3028
3318
  };
@@ -3109,18 +3399,21 @@ export class CloudflareScopeHost {
3109
3399
  unassignRole: async (actor, assignment) => {
3110
3400
  // Tombstone (K-21) — the checker skips revoked rows. A no-op returns false so
3111
3401
  // a repeat unassign stays silent (no second audit row, no needless fan-out).
3112
- const subject = `principal:${assignment.principalId}`;
3113
3402
  const relation = `role:${assignment.roleKey}`;
3114
- const now = new Date().toISOString();
3115
- const changed = assignment.node.scopeId
3116
- ? await this.scopeStub(assignment.node.scopeId).revokeTuple(subject, relation, `scope:${assignment.node.scopeId}`, now)
3117
- : await this.cp.revokeTenantTuple(assignment.node.tenantId, subject, relation, `tenant:${assignment.node.tenantId}`, now);
3403
+ const { tenantId, scopeId } = assignment.node;
3404
+ if (!scopeId) {
3405
+ // One ControlPlaneDO method (#1184): the revoke, the removal fence (raised even when
3406
+ // nothing was held) and the audit row, so no add can land between any two of them.
3407
+ const changed = await this.cp.revokeAndFence(tenantId, assignment.principalId, relation, `tenant:${tenantId}`, this.adminEntry(actor, 'unassignRole', { tenantId, scopeId: null }, assignment, null));
3408
+ // A tenant-level revoke changes the projected set — the tombstone must reach scopes.
3409
+ if (changed)
3410
+ await this.fanOut(tenantId);
3411
+ return;
3412
+ }
3413
+ const changed = await this.scopeStub(scopeId).revokeTuple(`principal:${assignment.principalId}`, relation, `scope:${scopeId}`, new Date().toISOString());
3118
3414
  if (!changed)
3119
3415
  return;
3120
- await this.recordAdmin(actor, 'unassignRole', { tenantId: assignment.node.tenantId, scopeId: assignment.node.scopeId }, assignment, null);
3121
- // A tenant-level revoke changes the projected set — the tombstone must reach scopes.
3122
- if (!assignment.node.scopeId)
3123
- await this.fanOut(assignment.node.tenantId);
3416
+ await this.recordAdmin(actor, 'unassignRole', { tenantId, scopeId }, assignment, null);
3124
3417
  },
3125
3418
  grant: async (actor, raw) => {
3126
3419
  // Parsed like its `grantToConnection`/`grantToSystem` siblings, not taken on
@@ -3242,11 +3535,17 @@ export class CloudflareScopeHost {
3242
3535
  },
3243
3536
  tenantHeldSystemModules: async (actor, tenantId, moduleIds) => {
3244
3537
  const held = moduleIds.length
3245
- ? await this.cp.tenantHeldSystemModules(tenantId, moduleIds, new Date().toISOString())
3538
+ ? await this.cp.tenantHeldOf('system', tenantId, moduleIds, new Date().toISOString())
3246
3539
  : [];
3247
3540
  await this.recordAccess(actor, 'tenantHeldSystemModules', { tenantId }, { moduleIds: [...moduleIds] }, held.length);
3248
3541
  return held;
3249
3542
  },
3543
+ peerSwitchCarry: async (actor, node) => {
3544
+ const { keys: switchedOffPeers, tenantHeld: tenantHeldPeers, fences: all, } = await this.recordedOff('peer', node.tenantId, node.scopeId, new Date().toISOString());
3545
+ const fences = Object.fromEntries(switchedOffPeers.map((v) => [v, all.get(v)]));
3546
+ await this.recordAccess(actor, 'peerSwitchCarry', node, null, switchedOffPeers.length);
3547
+ return { switchedOffPeers, tenantHeldPeers, fences };
3548
+ },
3250
3549
  reassertSystemSwitches: reassertSystemSwitchesOf,
3251
3550
  // #1672 — the platform's two capability verbs. Audited AFTER the write, on both, and
3252
3551
  // the failure each leaves is the safe one: a mint whose audit row did not land never
@@ -3312,7 +3611,7 @@ export class CloudflareScopeHost {
3312
3611
  // another tenant's traffic. Exception: the holder is ARCHIVED or REAPED (a
3313
3612
  // deleted app, storage since wiped) — it has released the name, so the rebind
3314
3613
  // reclaims it.
3315
- throw new Error(`hostname '${parsed.hostname}' is already bound to another scope`);
3614
+ throw substratError('conflict', `hostname '${parsed.hostname}' is already bound to another scope`);
3316
3615
  }
3317
3616
  // Exactly one canonical per (scope, surface).
3318
3617
  if (parsed.canonical)
@@ -3333,7 +3632,7 @@ export class CloudflareScopeHost {
3333
3632
  const hostname = raw.toLowerCase(); // DNS is case-insensitive; the map is normalized
3334
3633
  const row = await this.cp.readHostname(hostname);
3335
3634
  if (!row)
3336
- throw new Error(`unknown hostname '${hostname}'`);
3635
+ throw substratError('not_found', `unknown hostname '${hostname}'`);
3337
3636
  if (row.status === status)
3338
3637
  return; // idempotent, unaudited
3339
3638
  await this.cp.setHostnameStatus(hostname, status, note ?? null);
@@ -3343,7 +3642,7 @@ export class CloudflareScopeHost {
3343
3642
  const hostname = raw.toLowerCase(); // DNS is case-insensitive; the map is normalized
3344
3643
  const row = await this.cp.readHostname(hostname);
3345
3644
  if (!row)
3346
- throw new Error(`unknown hostname '${hostname}'`);
3645
+ throw substratError('not_found', `unknown hostname '${hostname}'`);
3347
3646
  // A poll that finds nothing changed (same status, same records, id already set)
3348
3647
  // is not an event — skip the write and the audit entry, so the reconcile sweep
3349
3648
  // does not flood the admin log with no-op rows every interval.
@@ -3429,7 +3728,7 @@ export class CloudflareScopeHost {
3429
3728
  if (existing.owner_tenant !== parsed.ownerTenant) {
3430
3729
  throw substratError('conflict', `vertical '${parsed.slug}' is owned by ${existing.owner_tenant ?? 'the platform'}, not ${parsed.ownerTenant ?? 'the platform'}`);
3431
3730
  }
3432
- throw new Error(`vertical '${parsed.slug}' is already registered as ${existing.source}`);
3731
+ throw substratError('conflict', `vertical '${parsed.slug}' is already registered as ${existing.source}`);
3433
3732
  }
3434
3733
  await this.cp.insertVertical(parsed.slug, parsed.name, parsed.source, parsed.ownerTenant, envSpecJson, installSpecJson, parsed.listed ? 1 : 0, new Date().toISOString());
3435
3734
  await this.recordAdmin(actor, 'registerVertical', { tenantId: null }, null, parsed);
@@ -3610,11 +3909,11 @@ export class CloudflareScopeHost {
3610
3909
  // nothing to diff against — the gate is about change, not existence.
3611
3910
  if (outgoing) {
3612
3911
  if (outgoing.permission_digest !== incoming.permission_digest && !ack.permissionChange) {
3613
- throw new Error(`promotion changes the permission surface (${outgoing.permission_digest} → ` +
3912
+ throw substratError('conflict', `promotion changes the permission surface (${outgoing.permission_digest} → ` +
3614
3913
  `${incoming.permission_digest}) — acknowledge it explicitly to promote`);
3615
3914
  }
3616
3915
  if (outgoing.migration_digest !== incoming.migration_digest && !ack.migrationChange) {
3617
- throw new Error(`promotion changes migrations (${outgoing.migration_digest} → ` +
3916
+ throw substratError('conflict', `promotion changes migrations (${outgoing.migration_digest} → ` +
3618
3917
  `${incoming.migration_digest}) — acknowledge it explicitly to promote`);
3619
3918
  }
3620
3919
  // #1705 PR 3: an export an installed consumer imports, dropped or re-versioned.
@@ -3874,7 +4173,7 @@ export class CloudflareScopeHost {
3874
4173
  },
3875
4174
  createOrg: async (actor, input) => {
3876
4175
  const parsed = createOrgInput.parse(input);
3877
- const created = await this.cp.createOrg(parsed.id, parsed.tenantId, parsed.slug, parsed.name, new Date().toISOString());
4176
+ const created = await this.directory('createOrg', parsed.id, parsed.tenantId, parsed.slug, parsed.name, new Date().toISOString());
3878
4177
  if (!created)
3879
4178
  return; // idempotent, and a no-op is not audited
3880
4179
  await this.recordAdmin(actor, 'createOrg', { tenantId: parsed.tenantId }, null, parsed);
@@ -3889,21 +4188,37 @@ export class CloudflareScopeHost {
3889
4188
  await this.recordAccess(actor, 'getOrg', { tenantId }, { orgId }, r ? 1 : 0);
3890
4189
  return r ? mapOrg(r) : undefined;
3891
4190
  },
3892
- addMember: async (actor, tenantId, principal, orgId) => {
4191
+ addMember: async (actor, tenantId, principal, orgId, opts) => {
3893
4192
  await requireOrg(tenantId, orgId);
3894
- await this.cp.writeTenantTuple(tenantId, `principal:${principal}`, 'member', `org:${orgId}`, null);
3895
- await this.recordAdmin(actor, 'addMember', { tenantId }, null, { principal, orgId });
4193
+ const expiresAt = opts?.expiresAt ?? null;
4194
+ await this.cp.writeTenantTuple(tenantId, `principal:${principal}`, 'member', `org:${orgId}`, expiresAt);
4195
+ await this.recordAdmin(actor, 'addMember', { tenantId }, null, memberAddedAudit(principal, orgId, expiresAt));
3896
4196
  await this.fanOut(tenantId); // membership is a tenant-level tuple
3897
4197
  },
4198
+ applyMembership: async (actor, change) => {
4199
+ // The row is minted here, where attribution and `causedBy` live, and written by the
4200
+ // ControlPlaneDO in the same synchronous method as the fence, the bound and the tuple
4201
+ // (#1184): one DO unit, so nothing lands between the check and the write.
4202
+ const { tenantId, principal, roleKey, op, orgId } = change;
4203
+ const assignment = { principalId: principal, roleKey, node: { tenantId, scopeId: null } };
4204
+ const result = await this.cp.applyMembership(change, op === 'add'
4205
+ ? this.adminEntry(actor, 'assignRole', { tenantId, scopeId: null }, null, assignment)
4206
+ : this.adminEntry(actor, 'unassignRole', { tenantId, scopeId: null }, assignment, null),
4207
+ // #2047: the org's own row, when the change joins or leaves one. The unit fills in its
4208
+ // `before`/`after` and writes it only for what it does — a join's carries the expiry.
4209
+ orgId ? this.adminEntry(actor, op === 'add' ? 'addMember' : 'removeMember', { tenantId }, null, null) : undefined);
4210
+ // The tenant-level tuple, or its tombstone, reaches the projections.
4211
+ if (result.applied && (op === 'add' || result.changed))
4212
+ await this.fanOut(tenantId);
4213
+ return result;
4214
+ },
3898
4215
  removeMember: async (actor, tenantId, principal, orgId) => {
3899
4216
  await requireOrg(tenantId, orgId);
3900
- // Tombstone (K-21), never DELETE. The DO reports whether anything changed
3901
- // so a repeat revoke stays a silent no-op rather than a second audit row.
3902
- const changed = await this.cp.revokeMember(tenantId, `principal:${principal}`, `org:${orgId}`, new Date().toISOString());
3903
- if (!changed)
3904
- return;
3905
- await this.recordAdmin(actor, 'removeMember', { tenantId }, { principal, orgId }, null);
3906
- await this.fanOut(tenantId); // the tombstone must reach the projections
4217
+ // Tombstone (K-21), never DELETE, with the removal fence in the same DO method
4218
+ // (#1184). A repeat revoke stays a silent no-op rather than a second audit row.
4219
+ const changed = await this.cp.revokeAndFence(tenantId, principal, 'member', `org:${orgId}`, this.adminEntry(actor, 'removeMember', { tenantId }, { principal, orgId }, null));
4220
+ if (changed)
4221
+ await this.fanOut(tenantId); // the tombstone must reach the projections
3907
4222
  },
3908
4223
  listMembers: async (actor, tenantId, orgId, options) => {
3909
4224
  await requireOrg(tenantId, orgId);
@@ -3913,24 +4228,25 @@ export class CloudflareScopeHost {
3913
4228
  principal: r.subject.slice('principal:'.length),
3914
4229
  orgId,
3915
4230
  revokedAt: r.revoked_at,
4231
+ expiresAt: r.expires_at,
3916
4232
  }));
3917
4233
  },
3918
4234
  createTenant: async (actor, input) => {
3919
4235
  const parsed = createTenantInput.parse(input);
3920
- const created = await this.cp.createTenant(parsed.id, parsed.slug, parsed.name, new Date().toISOString(), parsed.provisionedByTenant ?? null);
4236
+ const created = await this.directory('createTenant', parsed.id, parsed.slug, parsed.name, new Date().toISOString(), parsed.provisionedByTenant ?? null);
3921
4237
  // Idempotent: re-creating an existing tenant is a no-op, not audited.
3922
4238
  if (!created)
3923
4239
  return;
3924
4240
  await this.recordAdmin(actor, 'createTenant', { tenantId: parsed.id }, null, created);
3925
4241
  },
3926
4242
  setTenantStatus: async (actor, tenantId, status) => {
3927
- const before = await this.cp.setTenantStatus(tenantId, status);
4243
+ const before = await this.directory('setTenantStatus', tenantId, status);
3928
4244
  await this.recordAdmin(actor, 'setTenantStatus', { tenantId }, { status: before }, { status });
3929
4245
  // #1713: every hosted scope under the tenant holds or resumes its work by this.
3930
4246
  await this.deliverLifecycles(actor, { tenantId });
3931
4247
  },
3932
4248
  setTenantName: async (actor, tenantId, name) => {
3933
- const before = await this.cp.setTenantName(tenantId, name);
4249
+ const before = await this.directory('setTenantName', tenantId, name);
3934
4250
  if (before === name)
3935
4251
  return; // no-op is not audited — nothing changed
3936
4252
  await this.recordAdmin(actor, 'setTenantName', { tenantId }, { name: before }, { name });
@@ -3940,7 +4256,7 @@ export class CloudflareScopeHost {
3940
4256
  // first (archive-if-needed → reapScope in the vertical deployment); the DO clears
3941
4257
  // the tenant's PII/config rows and flips the row to a `reaped` tombstone, keeping
3942
4258
  // the row + admin log. Only a `deleting` tenant may be reaped (checked in the DO).
3943
- const before = await this.cp.reapTenant(tenantId);
4259
+ const before = await this.directory('reapTenant', tenantId);
3944
4260
  await this.recordAdmin(actor, 'reapTenant', { tenantId }, { status: before }, { status: 'reaped' });
3945
4261
  },
3946
4262
  listTenants: async (actor, page) => {
@@ -4024,7 +4340,7 @@ export class CloudflareScopeHost {
4024
4340
  // Not `peerScopeGate`: a read must not migrate, and the phase makes this call once
4025
4341
  // per edge per pass, where an extra round trip is a fleet-wide cost.
4026
4342
  this.assertServedHere(await this.scopeRecordForRead(tenantId, scopeId), scopeId, 'readExportedEvents');
4027
- const batch = exportedBatch.parse(await this.scopeStub(scopeId).exportedEventsRead(input, tenantId, scopeId));
4343
+ const batch = await this.readExportsThroughDoor(input, tenantId, scopeId);
4028
4344
  await this.recordAccess(actor, 'readExportedEvents', { tenantId, scopeId }, { consumer: input.consumer, after: input.after, limit: input.limit }, batch.events.length);
4029
4345
  return batch;
4030
4346
  },
@@ -4204,12 +4520,18 @@ export class CloudflareScopeHost {
4204
4520
  await this.recordAccess(actor, 'lifecycleFlow', { tenantId, scopeId }, delegatedReadParams.lifecycleFlow(input), flow.observation.events);
4205
4521
  return flow;
4206
4522
  },
4523
+ operationSeries: async (actor, tenantId, scopeId, input) => {
4524
+ await this.scopeRecordForRead(tenantId, scopeId);
4525
+ const series = await this.scopeStub(scopeId).operationSeries(input);
4526
+ await this.recordAccess(actor, 'operationSeries', { tenantId, scopeId }, delegatedReadParams.operationSeries(input), operationSeriesCount(series));
4527
+ return series;
4528
+ },
4207
4529
  readScopeTable: async (actor, tenantId, scopeId, input) => {
4208
4530
  await this.scopeRecordForRead(tenantId, scopeId);
4209
4531
  // Checked here as well as in the ScopeDO, so the refusal keeps its code across the hop.
4210
4532
  assertRowLimit('limit', input.limit);
4211
4533
  assertRowOffset('offset', input.offset);
4212
- const page = await this.scopeStub(scopeId).introspectTable(input.table, input.limit, input.offset);
4534
+ const page = unwrapReply(await this.scopeStub(scopeId).introspectTableReply(input.table, input.limit, input.offset));
4213
4535
  await this.recordAccess(actor, 'readScopeTable', { tenantId, scopeId }, { table: input.table, limit: page.limit, offset: page.offset }, page.rows.length);
4214
4536
  return page;
4215
4537
  },
@@ -4250,7 +4572,7 @@ export class CloudflareScopeHost {
4250
4572
  },
4251
4573
  queryScope: async (actor, tenantId, scopeId, input) => {
4252
4574
  await this.scopeRecordForRead(tenantId, scopeId);
4253
- const result = await this.scopeStub(scopeId).introspectQuery(input.sql);
4575
+ const result = unwrapReply(await this.scopeStub(scopeId).introspectQueryReply(input.sql));
4254
4576
  // The statement is the logged argument: the access log is the evidence trail,
4255
4577
  // and for a console read the SQL is the whole story.
4256
4578
  await this.recordAccess(actor, 'queryScope', { tenantId, scopeId }, { sql: input.sql }, result.rows.length);
@@ -4343,7 +4665,7 @@ export class CloudflareScopeHost {
4343
4665
  if (!rec)
4344
4666
  throw substratError('not_found', `unknown scope ${scopeId} in tenant ${tenantId}`);
4345
4667
  if (rec.status !== 'archived') {
4346
- throw new Error(`scope ${scopeId} is ${rec.status}, not archived — only an archived scope may be reaped`);
4668
+ throw substratError('conflict', `scope ${scopeId} is ${rec.status}, not archived — only an archived scope may be reaped`);
4347
4669
  }
4348
4670
  // A serving scope always holds ≥1 bound hostname; a truly-dead one has been
4349
4671
  // unbound. The dashboard delete path unbinds AT archive, but a bare console
@@ -4355,7 +4677,7 @@ export class CloudflareScopeHost {
4355
4677
  // being released anyway; the interactive per-scope reap never sets it.
4356
4678
  const bound = opts?.force ? [] : await this.cp.listHostnames({ scopeId, limit: 1 });
4357
4679
  if (bound.length > 0) {
4358
- throw new Error(`scope ${scopeId} still resolves hostname '${bound[0].hostname}' — ` +
4680
+ throw substratError('conflict', `scope ${scopeId} still resolves hostname '${bound[0].hostname}' — ` +
4359
4681
  `unbind it before reaping (reap wipes storage and cannot be undone)`);
4360
4682
  }
4361
4683
  await this.scopeStub(scopeId).destroyStorage();
@@ -4553,7 +4875,7 @@ export class CloudflareScopeHost {
4553
4875
  },
4554
4876
  registerIdentityPool: async (actor, input) => {
4555
4877
  const parsed = identityPool.parse(input);
4556
- const created = await this.cp.registerIdentityPool(parsed.provider, parsed.topology, parsed.tenantId, new Date().toISOString());
4878
+ const created = await this.directory('registerIdentityPool', parsed.provider, parsed.topology, parsed.tenantId, new Date().toISOString());
4557
4879
  if (!created)
4558
4880
  return; // identical registration is idempotent, unaudited
4559
4881
  // Null tenant for a central pool: it belongs to no single tenant, which is
@@ -4570,9 +4892,9 @@ export class CloudflareScopeHost {
4570
4892
  listIdentityTenants: async (actor, provider, externalId) => {
4571
4893
  const r = await this.cp.readPool(provider);
4572
4894
  if (!r)
4573
- throw new Error(`identity pool '${provider}' is not registered`);
4895
+ throw substratError('not_found', `identity pool '${provider}' is not registered`);
4574
4896
  if (r.topology !== 'central') {
4575
- throw new Error(`identity pool '${provider}' is tenant-bound — enumerating tenants is only ` +
4897
+ throw substratError('forbidden', `identity pool '${provider}' is tenant-bound — enumerating tenants is only ` +
4576
4898
  `meaningful on a central pool, where the same externalId is the same person`);
4577
4899
  }
4578
4900
  const tenants = (await this.cp.identityTenants(provider, externalId));
@@ -4589,9 +4911,9 @@ export class CloudflareScopeHost {
4589
4911
  at: new Date().toISOString(),
4590
4912
  });
4591
4913
  if (topology === null)
4592
- throw new Error(`identity pool '${provider}' is not registered`);
4914
+ throw substratError('not_found', `identity pool '${provider}' is not registered`);
4593
4915
  if (topology !== 'central') {
4594
- throw new Error(`identity pool '${provider}' is tenant-bound — enumerating tenants is only ` +
4916
+ throw substratError('forbidden', `identity pool '${provider}' is tenant-bound — enumerating tenants is only ` +
4595
4917
  `meaningful on a central pool, where the same externalId is the same person`);
4596
4918
  }
4597
4919
  return memberships.map((m) => identityMembership.parse(m));
@@ -4744,14 +5066,14 @@ export class CloudflareScopeHost {
4744
5066
  const parsed = identityLink.parse(input);
4745
5067
  const pool = await this.cp.readPool(parsed.provider);
4746
5068
  if (!pool) {
4747
- throw new Error(`identity pool '${parsed.provider}' is not registered — a pool must declare ` +
5069
+ throw substratError('conflict', `identity pool '${parsed.provider}' is not registered — a pool must declare ` +
4748
5070
  `its topology before it may link (central vs tenant-bound decides whether ` +
4749
5071
  `the same externalId in two tenants is one person or two)`);
4750
5072
  }
4751
5073
  if (pool.topology === 'tenant-bound' && pool.tenant_id !== parsed.tenantId) {
4752
- throw new Error(`identity pool '${parsed.provider}' is bound to tenant ${pool.tenant_id} and cannot link into ${parsed.tenantId}`);
5074
+ throw substratError('conflict', `identity pool '${parsed.provider}' is bound to tenant ${pool.tenant_id} and cannot link into ${parsed.tenantId}`);
4753
5075
  }
4754
- const changed = await this.cp.linkIdentity(parsed.provider, parsed.externalId, parsed.principal, parsed.tenantId, parsed.scopeId ?? null, new Date().toISOString());
5076
+ const changed = await this.directory('linkIdentity', parsed.provider, parsed.externalId, parsed.principal, parsed.tenantId, parsed.scopeId ?? null, new Date().toISOString());
4755
5077
  // Idempotent: an identity already bound is a no-op, not audited.
4756
5078
  if (!changed)
4757
5079
  return;
@@ -4800,6 +5122,11 @@ export class CloudflareScopeHost {
4800
5122
  const { tenantId, scopeId, ...after } = ownerTransferAudit.parse(entry);
4801
5123
  await this.recordAdmin(actor, 'transferOwner', { tenantId, scopeId }, null, after);
4802
5124
  },
5125
+ /** #1150: one phase of a dashboard member change, written around the vertical's own. */
5126
+ recordMemberChange: async (actor, entry) => {
5127
+ const { tenantId, scopeId, ...after } = memberChangeAudit.parse(entry);
5128
+ await this.recordAdmin(actor, 'manageScopeMember', { tenantId, scopeId }, null, after);
5129
+ },
4803
5130
  /** #2005: one change to a scope's copy marker, written around the vertical's own change. */
4804
5131
  recordCopyMark: async (actor, entry) => {
4805
5132
  const { tenantId, scopeId, action, ...after } = copyMarkAudit.parse(entry);
@@ -4956,6 +5283,8 @@ export class CloudflareScopeHost {
4956
5283
  },
4957
5284
  // Checked here as well as in the directory, so the refusal keeps its code across the hop.
4958
5285
  pruneTelemetry: async (_actor, limit) => this.cp.pruneTelemetry(assertRowLimit('limit', limit)),
5286
+ // #1748: the DO writes each stale resolution's audit row in the same unit, from this one.
5287
+ pruneFindings: async (actor, limit) => this.cp.pruneFindings(assertRowLimit('limit', limit), this.adminEntry(actor, 'resolveStaleFinding', { tenantId: null }, null, null)),
4959
5288
  listIssues: async (actor, filter) => {
4960
5289
  const rows = await this.cp.listIssues({
4961
5290
  status: filter?.status,
@@ -4977,6 +5306,33 @@ export class CloudflareScopeHost {
4977
5306
  await this.recordAdmin(actor, 'setIssueStatus', { tenantId: null, vertical: after.lastVertical }, { fingerprint, status: before.status }, { fingerprint, status: after.status });
4978
5307
  return after;
4979
5308
  },
5309
+ listFindings: async (actor, filter) => {
5310
+ const rows = await this.cp.listFindings({ ...filter });
5311
+ await this.recordAccess(actor, 'listFindings', { tenantId: filter?.tenantId ?? null }, filter, rows.length);
5312
+ return rows.map((r) => findingEntry.parse(r));
5313
+ },
5314
+ // #1748: each findings mutation writes its audit row in the DO's own unit, from the row
5315
+ // minted here (actor, attribution) — so a failed audit write rolls the mutation back.
5316
+ setFindingStatus: async (actor, tenantId, id, status) => {
5317
+ const audit = this.adminEntry(actor, 'setFindingStatus', { tenantId }, null, null);
5318
+ const change = await this.cp.setFindingStatus(tenantId, id, status, new Date().toISOString(), audit);
5319
+ return change ? findingEntry.parse(change.after) : undefined;
5320
+ },
5321
+ createFindingRule: async (actor, tenantId, input) => {
5322
+ const audit = this.adminEntry(actor, 'createFindingRule', { tenantId }, null, null);
5323
+ const created = await this.directory('createFindingRule', tenantId, input, actor, new Date().toISOString(), audit);
5324
+ return { rule: findingRuleEntry.parse(created.rule), suppressed: created.suppressed.length };
5325
+ },
5326
+ revokeFindingRule: async (actor, tenantId, ruleId) => {
5327
+ const audit = this.adminEntry(actor, 'revokeFindingRule', { tenantId }, null, null);
5328
+ const change = await this.cp.revokeFindingRule(tenantId, ruleId, new Date().toISOString(), audit);
5329
+ return change ? findingRuleEntry.parse(change.after) : undefined;
5330
+ },
5331
+ listFindingRules: async (actor, tenantId, filter) => {
5332
+ const rows = await this.cp.listFindingRules(tenantId, filter?.active ? new Date().toISOString() : undefined, filter?.limit);
5333
+ await this.recordAccess(actor, 'listFindingRules', { tenantId }, filter, rows.length);
5334
+ return rows.map((r) => findingRuleEntry.parse(r));
5335
+ },
4980
5336
  recordModelUsage: async (input) => {
4981
5337
  const l = input.line;
4982
5338
  return this.cp.recordModelUsage({
@@ -5013,6 +5369,10 @@ export class CloudflareScopeHost {
5013
5369
  };
5014
5370
  }
5015
5371
  // -- helpers --------------------------------------------------------------
5372
+ /** A directory write whose refusal must keep its code across the DO hop (#113). */
5373
+ async directory(method, ...args) {
5374
+ return unwrapReply(await this.cp.reply(method, args));
5375
+ }
5016
5376
  /**
5017
5377
  * The getScope gate (control-plane.md §4.1/§4.2) — the pair check, then the tenant's and
5018
5378
  * the scope's lifecycle — thrown on THIS side of the RPC (#1718). The ControlPlaneDO
@@ -5032,8 +5392,11 @@ export class CloudflareScopeHost {
5032
5392
  */
5033
5393
  async validateScopeAccess(tenantId, scopeId) {
5034
5394
  const refusal = await this.cp.scopeAccessRefusal(tenantId, scopeId);
5035
- if (refusal)
5036
- throw refusal.code ? substratError(refusal.code, refusal.message) : new Error(refusal.message);
5395
+ if (refusal) {
5396
+ throw refusal.code
5397
+ ? substratError(refusal.code, refusal.message, refusal.reason ? { reason: refusal.reason } : undefined)
5398
+ : new Error(refusal.message);
5399
+ }
5037
5400
  if (!this.cpLess)
5038
5401
  return null;
5039
5402
  const { verdict, lifecycle } = await this.scopeStub(scopeId).admission(tenantId);
@@ -5065,7 +5428,7 @@ export class CloudflareScopeHost {
5065
5428
  async assertLive(tenantId, scopeId) {
5066
5429
  const refusal = lifecycleRefusal(await this.validateScopeAccess(tenantId, scopeId), { tenantId, scopeId });
5067
5430
  if (refusal)
5068
- throw new Error(refusal);
5431
+ throw substratError('conflict', refusal, { reason: SCOPE_GATE_REASONS.notActive });
5069
5432
  }
5070
5433
  /**
5071
5434
  * Deliver the directory's lifecycle to the deployments serving the matching scopes (#1713):
@@ -5262,7 +5625,7 @@ export class CloudflareScopeHost {
5262
5625
  if (!row)
5263
5626
  throw unknownScopeForTenant(tenantId, scopeId);
5264
5627
  if (row.status === 'reaped') {
5265
- throw new Error(`scope ${scopeId} is reaped — its storage is gone and cannot be read`);
5628
+ throw substratError('conflict', `scope ${scopeId} is reaped — its storage is gone and cannot be read`);
5266
5629
  }
5267
5630
  return row;
5268
5631
  }
@@ -5295,7 +5658,14 @@ export class CloudflareScopeHost {
5295
5658
  });
5296
5659
  }
5297
5660
  async recordAdmin(actor, action, target, before, after) {
5298
- await this.cp.recordAdmin({
5661
+ await this.cp.recordAdmin(this.adminEntry(actor, action, target, before, after));
5662
+ }
5663
+ /**
5664
+ * One admin-log row, minted here where attribution and `causedBy` live — for `recordAdmin`,
5665
+ * and for a ControlPlaneDO method that writes its row in the same unit as its effect (#1184).
5666
+ */
5667
+ adminEntry(actor, action, target, before, after) {
5668
+ return {
5299
5669
  id: ulid(),
5300
5670
  actor,
5301
5671
  action,
@@ -5307,7 +5677,7 @@ export class CloudflareScopeHost {
5307
5677
  before: before ?? null,
5308
5678
  after: after ?? null,
5309
5679
  at: new Date().toISOString(),
5310
- });
5680
+ };
5311
5681
  }
5312
5682
  /**
5313
5683
  * The scope's own answer to "what may this connection do here" (#726 gap 1).
@@ -5432,6 +5802,11 @@ export class CloudflareScopeHost {
5432
5802
  // rewind CLAIMS them on `SWITCH_HOLDS_NAME`, which the rewind cannot reach. A module is held
5433
5803
  // while any claim on it exists, and `runDueSchedules` skips a held module.
5434
5804
  //
5805
+ // #2029: the same for a PEER switched off (`vertical:<slug>`): the rewind claims the scope's
5806
+ // off peers beside its off modules, each under its hold key (`holdKeyOf`), and the peer door
5807
+ // refuses a held peer as the system door refuses a held module. Everything below holds for
5808
+ // both kinds; "module" reads as "subject".
5809
+ //
5435
5810
  // A claim is released only by a switch move KNOWN to survive its rewind (`switchInScope`): a
5436
5811
  // move on the restored storage, after the rewind armed. A move during the settle, or one the
5437
5812
  // doomed instance applied, lands in storage the restart discards, so it releases nothing. A
@@ -5450,7 +5825,7 @@ export class CloudflareScopeHost {
5450
5825
  * reopens only the gap this closes, for a rewind that lands during the outage, and the next
5451
5826
  * reconcile still switches that module off. The error is answered, for the pass to report.
5452
5827
  */
5453
- async switchHeld(scopeId, moduleId) {
5828
+ async switchHeld(scopeId, holdKey) {
5454
5829
  const at = Date.now();
5455
5830
  let error;
5456
5831
  if (!this.holdSnapshot || at - this.holdSnapshot.at > SWITCH_HOLD_SNAPSHOT_MS) {
@@ -5473,8 +5848,11 @@ export class CloudflareScopeHost {
5473
5848
  }
5474
5849
  }
5475
5850
  }
5476
- const held = this.holdSnapshot?.held.has(`${scopeId} ${moduleId}`) ?? false;
5477
- return { held, ...(error ? { error } : {}) };
5851
+ const snapshot = this.holdSnapshot;
5852
+ const held = snapshot?.held.has(`${scopeId} ${holdKey}`) ?? false;
5853
+ // #2029: a good read naming no row for this scope at all, for `doorGate`'s instance cache.
5854
+ const scopeClear = !error && snapshot?.good === true && !snapshot.scopes.has(scopeId);
5855
+ return { held, ...(error ? { error } : {}), ...(scopeClear ? { scopeClear: true } : {}) };
5478
5856
  }
5479
5857
  /** One read of the whole hold into the snapshot, stamped with when it went out. Never throws. */
5480
5858
  async refreshHolds(at) {
@@ -5482,7 +5860,12 @@ export class CloudflareScopeHost {
5482
5860
  const rows = await this.switchHoldsStub().switchHoldsAll();
5483
5861
  // A slow read that lands after a newer one never replaces it.
5484
5862
  if (!this.holdSnapshot || at >= this.holdSnapshot.at) {
5485
- this.holdSnapshot = { at, held: new Set(rows.map((r) => `${r.scopeId} ${r.moduleId}`)) };
5863
+ this.holdSnapshot = {
5864
+ at,
5865
+ held: new Set(rows.map((r) => `${r.scopeId} ${r.moduleId}`)),
5866
+ scopes: new Set(rows.map((r) => r.scopeId)),
5867
+ good: true,
5868
+ };
5486
5869
  }
5487
5870
  return {};
5488
5871
  }
@@ -5493,8 +5876,10 @@ export class CloudflareScopeHost {
5493
5876
  // than hammered per scope. For that long, other scopes reuse this possibly pre-hold set
5494
5877
  // silently, and only the consult that sent this read reports the error: the declared
5495
5878
  // fail-open, attributed once.
5496
- if (!this.holdSnapshot || at >= this.holdSnapshot.at)
5497
- this.holdSnapshot = { at, held: known ?? new Set() };
5879
+ if (!this.holdSnapshot || at >= this.holdSnapshot.at) {
5880
+ const scopes = new Set([...(known ?? [])].map((k) => k.slice(0, k.indexOf(' '))));
5881
+ this.holdSnapshot = { at, held: known ?? new Set(), scopes, good: false };
5882
+ }
5498
5883
  return {
5499
5884
  error: known
5500
5885
  ? `switch hold unreadable (${why}); the ${known.size} hold(s) from this pass's last good read ` +
@@ -5541,7 +5926,11 @@ export class CloudflareScopeHost {
5541
5926
  * its claims before the OFF's join lands. The joined row then holds the module the operator just
5542
5927
  * turned ON, until the next surviving move of that switch releases it (another ON does).
5543
5928
  */
5544
- async switchInScope(scopeId, moduleId, to, at, tenantHeld = false) {
5929
+ async switchInScope(kind, scopeId, key, to, at, tenantHeld = false,
5930
+ /** #2045: the switch call's fence; see `SWITCH_FENCES_DDL`. */
5931
+ fence) {
5932
+ // #2029: a peer's claims live beside the modules', under its own hold key.
5933
+ const holdKey = holdKeyOf(kind, key);
5545
5934
  let scopeClaims;
5546
5935
  let readError;
5547
5936
  try {
@@ -5550,23 +5939,30 @@ export class CloudflareScopeHost {
5550
5939
  catch (err) {
5551
5940
  readError = err;
5552
5941
  }
5553
- const { instance, ...outcome } = await this.scopeStub(scopeId).switchSystemSchedules(moduleId, scopeId, to, at, tenantHeld);
5942
+ const stub = this.scopeStub(scopeId);
5943
+ const { instance, ...outcome } = await (kind === 'system'
5944
+ ? stub.switchSystemSchedules(key, scopeId, to, at, tenantHeld, fence)
5945
+ : stub.switchPeer(key, scopeId, to, at, tenantHeld, fence));
5554
5946
  if (!scopeClaims)
5555
5947
  throw readError;
5948
+ // #2045: a move the fence refused wrote nothing, so it is known to survive nothing and to
5949
+ // release nothing; the newer call that superseded it did whatever the hold needed.
5950
+ if (outcome.superseded)
5951
+ return outcome;
5556
5952
  const now = Date.now();
5557
5953
  const pendingLive = (c) => c.state === 'pending' && now - Date.parse(c.heldAt) <= SWITCH_HOLD_PENDING_MAX_MS;
5558
- const before = scopeClaims.filter((c) => c.moduleId === moduleId);
5954
+ const before = scopeClaims.filter((c) => c.moduleId === holdKey);
5559
5955
  if (to === 'on') {
5560
5956
  // Every ON, claims or not: its tombstone is what keeps a rewind's stale read from claiming
5561
5957
  // the module after this (#1839).
5562
5958
  this.holdSnapshot = null;
5563
- await this.switchHoldsStub().switchHoldOn(scopeId, moduleId, before.map((c) => c.claimId));
5959
+ await this.switchHoldsStub().switchHoldOn(scopeId, holdKey, before.map((c) => c.claimId));
5564
5960
  }
5565
5961
  else if (outcome.held && before.length > 0) {
5566
5962
  const survived = before.filter((c) => !pendingLive(c) && c.doomed !== instance);
5567
5963
  if (survived.length > 0) {
5568
5964
  this.holdSnapshot = null;
5569
- await this.switchHoldsStub().switchHoldRelease(scopeId, moduleId, survived.map((c) => c.claimId));
5965
+ await this.switchHoldsStub().switchHoldRelease(scopeId, holdKey, survived.map((c) => c.claimId));
5570
5966
  }
5571
5967
  }
5572
5968
  if (to === 'off' && outcome.changed) {
@@ -5577,7 +5973,7 @@ export class CloudflareScopeHost {
5577
5973
  const joins = new Set([...scopeClaims, ...after].filter((c) => pendingLive(c) || c.doomed === instance).map((c) => c.claimId));
5578
5974
  if (joins.size > 0) {
5579
5975
  this.holdSnapshot = null;
5580
- await this.switchHoldsStub().switchHoldJoin(scopeId, moduleId, [...joins]);
5976
+ await this.switchHoldsStub().switchHoldJoin(scopeId, holdKey, [...joins]);
5581
5977
  }
5582
5978
  }
5583
5979
  return outcome;
@@ -5641,11 +6037,16 @@ export class CloudflareScopeHost {
5641
6037
  await holds.switchHoldArm(scopeId, claimId, result.instance ?? null).catch(() => undefined);
5642
6038
  return { rewindingTo: result.rewindingTo };
5643
6039
  }
5644
- /** The modules this scope's own storage has switched off, as a rewind reads them before it arms. */
6040
+ /**
6041
+ * The subjects this scope's own storage has switched off, as a rewind reads them before it arms:
6042
+ * its modules, and (#2029) its peers, each by its hold key (`holdKeyOf`).
6043
+ */
5645
6044
  async offInStorage(scopeId) {
5646
- return (await this.systemGrantsStatusLocal(scopeId))
5647
- .filter((e) => e.schedules === 'off')
5648
- .map((e) => e.moduleId);
6045
+ const [modules, peers] = await Promise.all([this.systemGrantsStatusLocal(scopeId), this.peerGrantsStatusLocal(scopeId)]);
6046
+ return [
6047
+ ...modules.filter((e) => e.schedules === 'off').map((e) => holdKeyOf('system', e.moduleId)),
6048
+ ...peers.filter((e) => e.calls === 'off').map((e) => holdKeyOf('peer', e.vertical)),
6049
+ ];
5649
6050
  }
5650
6051
  /**
5651
6052
  * #1839: wait until every row of this rewind's claim is `SWITCH_HOLD_SETTLE_MS` old. The settle
@@ -5714,7 +6115,11 @@ export class CloudflareScopeHost {
5714
6115
  * arrive as a different class of answer just because the caller asked for a socket.
5715
6116
  */
5716
6117
  liveReads = {
5717
- subscribe: async ({ tenantId, scopeId, principal, request }) => {
6118
+ subscribe: async ({ tenantId, scopeId, principal, request, within }) => {
6119
+ // Decided before anything else, and thrown: a `within` that is neither a plain
6120
+ // EntityRef nor a value `vouchedWithin` built is a caller bug, and the two ways of
6121
+ // guessing at it both open a feed wider than was asked for (#1853).
6122
+ const narrowed = liveWithinOf(within);
5718
6123
  if (!isUpgradeRequest(request)) {
5719
6124
  return new Response('live reads are a WebSocket surface', {
5720
6125
  status: 426,
@@ -5772,6 +6177,10 @@ export class CloudflareScopeHost {
5772
6177
  headers.set(LIVE_PRINCIPAL_HEADER, principal);
5773
6178
  headers.set(LIVE_TENANT_HEADER, tenantId);
5774
6179
  headers.set(LIVE_SCOPE_HEADER, scopeId);
6180
+ if (narrowed)
6181
+ headers.set(LIVE_WITHIN_HEADER, encodeLiveWithin(narrowed));
6182
+ else
6183
+ headers.delete(LIVE_WITHIN_HEADER);
5775
6184
  const forwarded = new Request(new URL(LIVE_SUBSCRIBE_PATH, 'https://scope.substrat.internal'),
5776
6185
  // `new Request(url, { …, headers })` rather than `new Request(request, …)`:
5777
6186
  // the DO is addressed by its own path, not the client's, and the upgrade
@@ -5853,7 +6262,7 @@ export class CloudflareScopeHost {
5853
6262
  throw unknownScopeForTenant(tenantId, scopeId);
5854
6263
  const { roles, tuples, entitlements, identities } = await this.tenantProjection(tenantId);
5855
6264
  const connectionKeys = await this.connectionKeyRows(tenantId, scope.vertical);
5856
- await this.scopeStub(scopeId).applyProjection(tenantId, roles, tuples, entitlements, undefined, identities, connectionKeys);
6265
+ unwrapReply(await this.scopeStub(scopeId).applyProjectionReply(tenantId, roles, tuples, entitlements, undefined, identities, connectionKeys));
5857
6266
  }
5858
6267
  /**
5859
6268
  * Fan the tenant's current state out into ALL its scopes — called after any
@@ -5902,7 +6311,7 @@ export class CloudflareScopeHost {
5902
6311
  if (!keysByVertical.has(vertical)) {
5903
6312
  keysByVertical.set(vertical, this.connectionKeyRows(tenantId, s.vertical));
5904
6313
  }
5905
- await this.scopeStub(s.scope_id).applyProjection(tenantId, roles, tuples, entitlements, undefined, identities, await keysByVertical.get(vertical));
6314
+ unwrapReply(await this.scopeStub(s.scope_id).applyProjectionReply(tenantId, roles, tuples, entitlements, undefined, identities, await keysByVertical.get(vertical)));
5906
6315
  }));
5907
6316
  const failed = settled.flatMap((r, i) => (r.status === 'rejected' ? [{ scope: scopes[i].scope_id, reason: r.reason }] : []));
5908
6317
  if (failed.length === 0)
@@ -5932,10 +6341,11 @@ export class CloudflareScopeHost {
5932
6341
  * from its own storage. No tenant-level tuples, no control plane.
5933
6342
  */
5934
6343
  async provisionScopeLocal(input) {
6344
+ const carry = recordedOffFromWire(input);
5935
6345
  const services = await this.servicePrincipals?.(input.tenantId, input.scopeId);
5936
6346
  const stub = this.scopeStub(input.scopeId);
5937
6347
  await this.migrateAndRecord(input.scopeId); // create the module tables (setMigrationState no-ops on a null CP)
5938
- const switchedOff = await stub.applyProjection(input.tenantId, input.roles.map((r) => ({ role_key: r.key, permissions: JSON.stringify(r.permissions), source: r.source })), [], // no tenant-level tuples — a CP-less vertical grants at scope level only
6348
+ const switchedOff = unwrapReply(await stub.applyProjectionReply(input.tenantId, input.roles.map((r) => ({ role_key: r.key, permissions: JSON.stringify(r.permissions), source: r.source })), [], // no tenant-level tuples — a CP-less vertical grants at scope level only
5939
6349
  // Only PROJECT (and thereby switch on strict enforcement) when the platform actually
5940
6350
  // supplies entitlements. Omitting them leaves the scope un-projected and trusting-
5941
6351
  // upstream, so a vertical whose provision path predates #304 is never denied — it
@@ -6025,12 +6435,10 @@ export class CloudflareScopeHost {
6025
6435
  public_key: k.publicKey,
6026
6436
  }))
6027
6437
  : undefined,
6028
- // #1742: the recorded-off modules, switched off after the seat in the same unit, on the
6029
- // scope this call provisions — the one the seat's tuples name.
6030
- input.switchedOff
6031
- ? { scopeId: input.scopeId, moduleIds: input.switchedOff, at: new Date().toISOString(), tenantHeld: input.tenantHeld }
6032
- : undefined, services?.map((id) => `principal:${id}`));
6033
- return input.switchedOff ? { switchedOff } : {};
6438
+ // #1742: the recorded-off modules and (#2029) peers, switched off after the seat in the
6439
+ // same unit, on the scope this call provisions — the one the seat's tuples name.
6440
+ carry ? { scopeId: input.scopeId, at: new Date().toISOString(), ...carry } : undefined, services?.map((id) => `principal:${id}`)));
6441
+ return carry ? { switchedOff } : {};
6034
6442
  }
6035
6443
  /**
6036
6444
  * Resolve an external identity from a scope's PROJECTED links (#406) — the CP-less
@@ -6169,6 +6577,8 @@ export class CloudflareScopeHost {
6169
6577
  async connectorGrantLocal(connectionId, scopeId, permission, expiresAt) {
6170
6578
  await this.writeScopeTuple(scopeId, subjectRef({ kind: 'connection', id: connectionId }), `granted:${permission}`, `scope:${scopeId}`, expiresAt ?? null);
6171
6579
  }
6580
+ /** #2045 (Codex r3): every switch mover here honours the fence — what `/internal/switch-fence` reports. */
6581
+ switchFenced = true;
6172
6582
  /**
6173
6583
  * The far end of the schedule kill switch for a scope served HERE (#1666):
6174
6584
  * `/internal/system-switch` lands on this, from the shared control plane's
@@ -6181,7 +6591,7 @@ export class CloudflareScopeHost {
6181
6591
  // Parsed on the way out: this is the wire answer the platform reads, and the DO's
6182
6592
  // plain strings become the published shape here rather than on trust.
6183
6593
  // #1819: `switchInScope` releases a rewind's hold on the module once the switch lands.
6184
- return systemSwitchOutcome.parse(await this.switchInScope(scopeId, moduleId, to, new Date().toISOString(), opts?.tenantHeld ?? false));
6594
+ return systemSwitchOutcome.parse(await this.switchInScope('system', scopeId, moduleId, to, new Date().toISOString(), opts?.tenantHeld ?? false, opts?.fence));
6185
6595
  }
6186
6596
  /**
6187
6597
  * The far end of the status read for a scope served HERE (#1674): `/internal/system-grants`
@@ -6216,9 +6626,12 @@ export class CloudflareScopeHost {
6216
6626
  * Move one peer's kill switch on a scope served HERE (#1706) — the far end of
6217
6627
  * `revokeFromPeer` / `restoreToPeer` for a hosted scope. Nothing is audited here; the
6218
6628
  * platform writes the admin rows around this call, as for the schedule switch.
6629
+ * #2029: through `switchInScope`, so the move releases a rewind's hold on the peer once it
6630
+ * lands, as a module's does. #2030: `tenantHeld`, the platform's word that the peer holds a
6631
+ * live tenant-level grant, which this deployment has no directory to read.
6219
6632
  */
6220
- async peerSwitchLocal(scopeId, vertical, to) {
6221
- return peerSwitchOutcome.parse(await this.scopeStub(scopeId).switchPeer(verticalSlug.parse(vertical), scopeId, to, new Date().toISOString()));
6633
+ async peerSwitchLocal(scopeId, vertical, to, opts) {
6634
+ return peerSwitchOutcome.parse(await this.switchInScope('peer', scopeId, verticalSlug.parse(vertical), to, new Date().toISOString(), opts?.tenantHeld ?? false, opts?.fence));
6222
6635
  }
6223
6636
  // -- the cross-vertical far ends (#1705 PR 2) --------------------------------
6224
6637
  // The shared control plane runs the cross-vertical phase for every hosted edge and reaches the
@@ -6232,7 +6645,25 @@ export class CloudflareScopeHost {
6232
6645
  async exportedEventsLocal(tenantId, scopeId, raw) {
6233
6646
  const input = exportReadInput.parse(raw);
6234
6647
  await this.assertServesLocally(tenantId, scopeId, 'readExportedEvents');
6235
- return exportedBatch.parse(await this.scopeStub(scopeId).exportedEventsRead(input, tenantId, scopeId));
6648
+ return this.readExportsThroughDoor(input, tenantId, scopeId);
6649
+ }
6650
+ /**
6651
+ * #2029: a producer's export read, through the CONSUMER's peer door: the consumer's grants on
6652
+ * this scope decide what leaves it, so a consumer the rewind hold keeps off must hold nothing
6653
+ * here, exactly as a switched-off one does. Gated and pinned like every door call; held, the
6654
+ * producer answers every key missing, so the edge pauses rather than reading past the hold.
6655
+ */
6656
+ async readExportsThroughDoor(input, tenantId, scopeId) {
6657
+ const stub = this.scopeStub(scopeId);
6658
+ const door = await this.openDoor('peer', input.consumer, scopeId);
6659
+ try {
6660
+ return exportedBatch.parse(await door.through((instance) => stub.exportedEventsRead(input, tenantId, scopeId, { instance })));
6661
+ }
6662
+ catch (err) {
6663
+ if (!this.isHeldRefusal(err))
6664
+ throw err;
6665
+ return exportedBatch.parse(await stub.exportedEventsRead(input, tenantId, scopeId, { held: true }));
6666
+ }
6236
6667
  }
6237
6668
  /**
6238
6669
  * The consumer's imports and watermarks (#1705), for a scope served HERE. The served-here