@substrat-run/adapter-cloudflare 0.137.0 → 0.138.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/host.js CHANGED
@@ -1,13 +1,14 @@
1
1
  import { env as ambientEnv } from 'cloudflare:workers';
2
2
  import { isRewindRefusal, REWIND_REFUSED } from './rewind-refusal.js';
3
3
  import { isSystemDoorMoved, SYSTEM_DOOR_MOVED, SYSTEM_DOOR_REGATES } from './system-door.js';
4
- import { delegatedReadParams, fromWireFailure, exportReadInput, exportedBatch, importBatch, importResult, importState, importsOfManifestJson, exportsOfManifestJson, importCursorMove, importCursorMoveAt, importCursorMoved, accessLogEntry, adminLogEntry, opsFailureEntry, opsFailureFingerprint, issueEntry, sweepRunEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunsPayload, modelUsageEntry, attachmentRecord, createTenantInput, identityLink, identityPool, createOrgInput, promotionAcknowledgement, bindAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, registerVerticalInput, vertical as verticalSchema, verticalServingState, verticalChannel, verticalVersion, connection, capabilityExchange, mintedCapability, capabilityGrant, principalId, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, systemSwitch, systemSwitchOutcome, systemScheduleEntry, systemSwitchRecord, entitlementGrant, entitlementGrantInput, capabilityFilter, instant, meterReading, subjectRef, createConnectionInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, org as orgSchema, orgMembership, resolvedIdentity, identityMembership, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, delegatedReadRecord, ownerTransferAudit, copyMarkAudit, subjectShredReceipt, connectorDispatchKind, callsOfManifestJson, outboundOfManifestJson, substratError, redrainEventsInput, coverage, peerCoverage, peerGrantsEntry, peerSwitch, peerSwitchOutcome, verticalCaller, verticalResolution, verticalSlug, entityObjectRef, ATTACHMENT_BLOB_BINDING, blobStoreBindingName, lifecycleDelivery, scopeLifecycle, } from '@substrat-run/contracts';
4
+ import { delegatedReadParams, operationSeriesCount, fromWireFailure, exportReadInput, exportedBatch, importBatch, importResult, importState, importsOfManifestJson, exportsOfManifestJson, importCursorMove, importCursorMoveAt, importCursorMoved, accessLogEntry, adminLogEntry, opsFailureEntry, opsFailureFingerprint, issueEntry, sweepRunEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunsPayload, modelUsageEntry, attachmentRecord, createTenantInput, identityLink, identityPool, createOrgInput, promotionAcknowledgement, bindAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, registerVerticalInput, vertical as verticalSchema, verticalServingState, verticalChannel, verticalVersion, connection, capabilityExchange, mintedCapability, capabilityGrant, principalId, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, systemSwitch, systemSwitchOutcome, systemScheduleEntry, systemSwitchRecord, entitlementGrant, entitlementGrantInput, capabilityFilter, instant, meterReading, subjectRef, createConnectionInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, org as orgSchema, orgMembership, resolvedIdentity, identityMembership, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, delegatedReadRecord, ownerTransferAudit, copyMarkAudit, subjectShredReceipt, connectorDispatchKind, callsOfManifestJson, outboundOfManifestJson, SCOPE_GATE_REASONS, substratError, redrainEventsInput, coverage, peerCoverage, peerGrantsEntry, peerSwitch, peerSwitchOutcome, verticalCaller, verticalResolution, verticalSlug, entityObjectRef, ATTACHMENT_BLOB_BINDING, blobStoreBindingName, lifecycleDelivery, scopeLifecycle, } from '@substrat-run/contracts';
5
5
  import { normalizeHostname, toRouteTarget } from './route-resolver.js';
6
- import { attachmentBlobKey, attachmentSha256, entitlementDenial, requiredEntitlementFor, foldMeterReading, parseValidationRecords, resolveScopeRecord, ulid, KEPT_COPY_REFUSAL, capabilityTokenHash, checkBecomeInput, plausibleSessionToken, foldModelUsage, backoffAt, resolveRetryPolicy, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, openSealed, ImpersonationRefused, assertSessionUsable, impersonationRowValues, mapImpersonationRow, newImpersonationSession, jobRunOf, runDueJobRuns, SYSTEM_DOOR_WAIT, startJobRun, attachmentTextJob, assertAttachmentExtractors, assertJobRegistrable, isAttachmentTextRun, searchLimit, searchMatchExpression, globalFetch, assertRedrainWindow, platformRequestOf, undrainedEventsOf, inUnitMovesToAudit, staleCarryRevertRow, staleCarryReverts, withRecorded, systemSwitchedOffMessage, tenantSystemSwitchedOffMessage, CrossVerticalRegistry, importCursorSourceOf, exportBreaksOf, exportBreakRefusal, bindExportBreaksOf, bindExportBreakRefusal, collectPeers, peerSeats, connectorCallRecord, noopConnectorCallRecorder, recordConnectorCall, settleConnectionUse, splitManifestMigrations, unknownRoleError, assertRowLimit, assertRowOffset, INERT_SCOPE_REASON, lifecycleReceipt, lifecycleRefusal, isPrimaryScope, isPrimaryScopeRow, asyncInvocationId, asyncLinePass, } from '@substrat-run/kernel';
6
+ import { attachmentBlobKey, attachmentSha256, entitlementDenial, requiredEntitlementFor, foldMeterReading, parseValidationRecords, resolveScopeRecord, ulid, KEPT_COPY_REFUSAL, capabilityTokenHash, checkBecomeInput, plausibleSessionToken, foldModelUsage, backoffAt, resolveRetryPolicy, executorOutcomeOf, isDeliveryRefusal, refusalJournalText, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, openSealed, ImpersonationRefused, assertSessionUsable, impersonationRowValues, mapImpersonationRow, newImpersonationSession, jobRunOf, assertLeaseMs, runDueJobRuns, SYSTEM_DOOR_WAIT, startJobRun, attachmentTextJob, assertAttachmentExtractors, assertJobRegistrable, isAttachmentTextRun, searchLimit, searchMatchExpression, globalFetch, assertRedrainWindow, platformRequestOf, undrainedEventsOf, inUnitMovesToAudit, staleCarryRevertRow, staleCarryReverts, recordedOffFromWire, peerSubjectRef, SWITCH_KINDS, reassertActionOf, reassertEntry, reassertOffRow, reassertOnRow, switchActionOf, switchAuditSubject, switchNotFoundMessage, recordWriteSuperseded, switchSubjectOf, switchSupersededMessage, withRecorded, systemSwitchedOffMessage, tenantSystemSwitchedOffMessage, CrossVerticalRegistry, importCursorSourceOf, exportBreaksOf, exportBreakRefusal, bindExportBreaksOf, bindExportBreakRefusal, collectPeers, peerSeats, connectorCallRecord, noopConnectorCallRecorder, recordConnectorCall, settleConnectionUse, splitManifestMigrations, unknownRoleError, assertRowLimit, assertRowOffset, emptyImportResult, INERT_SCOPE_REASON, lifecycleReceipt, lifecycleRefusal, isPrimaryScope, isPrimaryScopeRow, asyncInvocationId, asyncLinePass, } from '@substrat-run/kernel';
7
7
  import { attributedHost } from '@substrat-run/kernel';
8
- import { isOrangeToOrange, isUpgradeRequest, LIVE_MODE_HEADER, LIVE_PRINCIPAL_HEADER, LIVE_SCOPE_HEADER, LIVE_SUBSCRIBE_PATH, LIVE_TENANT_HEADER, } from './live-reads.js';
8
+ import { isOrangeToOrange, isUpgradeRequest, LIVE_MODE_HEADER, LIVE_PRINCIPAL_HEADER, LIVE_SCOPE_HEADER, LIVE_SUBSCRIBE_PATH, LIVE_TENANT_HEADER, LIVE_WITHIN_HEADER, encodeLiveWithin, liveWithinOf, } from './live-reads.js';
9
9
  import { tenantStoreDatabaseName } from './d1.js';
10
10
  import { blobStoreBucketName, r2TenantBlobStore } from './r2.js';
11
+ import { unwrapReply } from './do-reply.js';
11
12
  /** DO row → contract shape. Never reads the secrets table — that is the split. */
12
13
  const toConnection = (r) => connection.parse({
13
14
  id: r.id,
@@ -26,12 +27,6 @@ const toConnection = (r) => connection.parse({
26
27
  createdAt: r.created_at,
27
28
  revokedAt: r.revoked_at,
28
29
  });
29
- /** Rethrow a capability attachment verb's failure, rebuilt with its code; else its value. */
30
- function unwrapCapabilityReply(reply) {
31
- if (reply.failure)
32
- throw fromWireFailure(reply.failure);
33
- return reply.value;
34
- }
35
30
  /**
36
31
  * How far ahead of the clock a held lifecycle epoch may be and still be learned past (#1713). An
37
32
  * epoch is the time a directory restore minted it, so a legitimate one is behind its minter's
@@ -92,24 +87,31 @@ function nullControlPlane() {
92
87
  recordAdmin: noop,
93
88
  recordAccess: noop,
94
89
  tenantHoldsEntitlement: async () => true,
95
- // #1674: the schedule switch's record is a DIRECTORY store, and a CP-less host has no
90
+ // #1674, #2029: the switches' record is a DIRECTORY store, and a CP-less host has no
96
91
  // directory — the shared control plane that delegates here keeps it. Same posture as
97
92
  // `recordAdmin`: nothing to write, nothing recorded to read back.
98
- recordSystemSwitchedOff: async () => null,
99
- recordSystemSwitchedOn: async () => null,
100
- restoreSystemSwitchRecord: noop,
93
+ recordSwitchedOff: async () => null,
94
+ recordSwitchedOn: async () => null,
95
+ restoreSwitchRecord: noop,
101
96
  listSystemSwitches: async () => [],
102
- systemSwitchRecordsOf: async () => [],
103
- switchedOffModulesOf: async () => [],
97
+ switchRecordsOf: async () => [],
98
+ switchRecordStatesOf: async () => [],
99
+ switchesOwedOf: async () => [],
100
+ clearSwitchOwed: noop,
104
101
  // #1823: no tenant tuples here either — the platform reads them and sends `tenantHeld`.
105
- tenantHeldSystemModules: async () => [],
102
+ tenantHeldOf: async () => [],
103
+ };
104
+ const unavailable = (method) => new Error(`control plane unavailable: '${method}' — this host is scope-local / CP-less ` +
105
+ `(docs/architecture/scope-local-permissions.md, Phase 3)`);
106
+ // A directory write through `reply` (#113) is named for the write it carries, not for `reply`.
107
+ passthrough.reply = async (method) => {
108
+ throw unavailable(String(method));
106
109
  };
107
110
  return new Proxy({}, {
108
111
  get: (_t, prop) => typeof prop === 'string' && prop in passthrough
109
112
  ? passthrough[prop]
110
113
  : async () => {
111
- throw new Error(`control plane unavailable: '${String(prop)}' — this host is scope-local / CP-less ` +
112
- `(docs/architecture/scope-local-permissions.md, Phase 3)`);
114
+ throw unavailable(String(prop));
113
115
  },
114
116
  });
115
117
  }
@@ -151,19 +153,34 @@ export const SWITCH_HOLD_EXTRA_WAITS = 3;
151
153
  */
152
154
  export const SWITCH_HOLD_PENDING_MAX_MS = 5 * 60_000;
153
155
  /**
154
- * An OFF of a tenant-held module whose far end did not attest `deniesTenantGrants` (#1823): it
155
- * switched, but its evaluator still authorizes the module's tenant-level grant on the scope.
156
- * Carries the far end's outcome, so the operator's OFF knows whether there is a move to put back.
156
+ * A move the deployment serving the scope applied, or may have, without attesting what the
157
+ * platform needs from it (#2045 Codex r3): the switch fence (`fenced`), or for a tenant-held
158
+ * module's OFF the evaluator that denies a tenant-level grant (`deniesTenantGrants`, #1823).
159
+ *
160
+ * The preflight (`switchTarget`'s `attestFence`) tells a deployment built before the fence apart
161
+ * before anything is recorded or moved. On such a deployment only an ON is refused: the race the
162
+ * fence closes is an older OFF landing after a newer ON, and with every ON refused there is no
163
+ * newer ON to land after, so an OFF goes through unfenced (#2045 follow-up). A kill switch that
164
+ * cannot be pulled until every vertical is re-pushed is the outage the fence must never cause.
165
+ * An unfenced OFF keeps its write-ahead owed mark, so the first re-assert after the vertical is
166
+ * redeployed moves the scope again under the record's fence.
167
+ *
168
+ * This error fires when a deployment that attested the fence answers a move without it — a
169
+ * rollback to an old build between the preflight and the move — or when an ON reaches a
170
+ * deployment without the fence. It is NOT compensated with an opposite move: an unfenced
171
+ * compensation can undo a newer call's switch (Codex r3, finding 2). The call's record and its
172
+ * write-ahead owed mark stay, and the call is refused.
157
173
  */
158
- class UnattestedSwitchOff extends Error {
159
- outcome;
160
- constructor(scopeId, moduleId, outcome) {
161
- super(`the deployment serving scope ${scopeId} predates the kill switch's tenant-grant denial (#1823): ` +
162
- `module '${moduleId}' holds a tenant-level grant that its OFF would leave authorizing there. ` +
163
- `Redeploy the vertical, then retry.`);
164
- this.outcome = outcome;
165
- }
174
+ class UnattestedSwitch extends Error {
166
175
  }
176
+ /** A fenced move answered without `fenced`: the deployment predates the fence. */
177
+ const unfencedMessage = (scopeId) => `the deployment serving scope ${scopeId} predates the switch fence (#2045): it cannot refuse an older ` +
178
+ `switch call's OFF landing after this ON, so no switch is turned on there. Switching off still works. ` +
179
+ `Redeploy the vertical, then retry.`;
180
+ /** A tenant-held module's OFF answered without `deniesTenantGrants` (#1823). */
181
+ const undeniedMessage = (scopeId, moduleId) => `the deployment serving scope ${scopeId} predates the kill switch's tenant-grant denial (#1823): ` +
182
+ `module '${moduleId}' holds a tenant-level grant that its OFF would leave authorizing there. ` +
183
+ `Redeploy the vertical, then retry.`;
167
184
  /**
168
185
  * The attachment bucket a host resolves when it was handed no resolver (#1995): the
169
186
  * platform's per-tenant binding, read off this script's own env, named by the one shared
@@ -172,6 +189,49 @@ class UnattestedSwitchOff extends Error {
172
189
  function ambientAttachmentBucket(tenantId) {
173
190
  return ambientEnv[blobStoreBindingName(ATTACHMENT_BLOB_BINDING, tenantId)] ?? null;
174
191
  }
192
+ /**
193
+ * #2029: scope instances whose door has read the rewind hold and found NO claim on the scope at
194
+ * all, so their doors skip the hold read for the rest of the instance's life. Isolate-wide, because
195
+ * a host lives for one request: without it, every peer call would read the deployment's one hold
196
+ * object — an extra hop, on a single object, on the hot path of every scope never rewound.
197
+ *
198
+ * Safe because claims are only ever added for storage a LATER instance serves. A rewind writes its
199
+ * claim before it arms, and arming restarts the scope, so its rewound storage is always served by a
200
+ * new instance with a new id, which reads the hold afresh (`SWITCH_HOLD_SETTLE_MS` puts every row of
201
+ * the claim before that instance existed). A join (`switchHoldJoin`) goes into a rewind's claim too,
202
+ * so into that same later instance. An instance that read no claim therefore never gains one; claims
203
+ * only leave it, by a switch move. Only a GOOD read is remembered: a read that failed open is not.
204
+ * Bounded: past `CLEAR_SCOPE_INSTANCES_MAX` it is emptied, which costs a re-read, nothing else.
205
+ */
206
+ const CLEAR_SCOPE_INSTANCES = new Set();
207
+ const CLEAR_SCOPE_INSTANCES_MAX = 10_000;
208
+ const rememberClearInstance = (key) => {
209
+ if (CLEAR_SCOPE_INSTANCES.size >= CLEAR_SCOPE_INSTANCES_MAX)
210
+ CLEAR_SCOPE_INSTANCES.clear();
211
+ CLEAR_SCOPE_INSTANCES.add(key);
212
+ };
213
+ /**
214
+ * #2045: a carry's fences, by tuple subject (`RecordedOffCarry.fences`): for each recorded-off
215
+ * subject, the operation id of the call the record holds.
216
+ */
217
+ const switchFencesCarried = (kinds) => {
218
+ const out = {};
219
+ for (const [kind, { keys, fences }] of kinds) {
220
+ for (const key of keys) {
221
+ const fence = fences.get(key);
222
+ if (fence !== undefined)
223
+ out[switchSubjectOf(kind, key)] = fence;
224
+ }
225
+ }
226
+ return out;
227
+ };
228
+ /** #2029: how a door's refusals and a re-assert's name each kind of subject. */
229
+ const DOOR_WORDS = {
230
+ system: { what: (key) => `module '${key}'`, door: 'system door', switchName: 'schedule', switched: 'schedules' },
231
+ peer: { what: (key) => `peer vertical '${key}'`, door: 'peer door', switchName: 'peer', switched: 'peers' },
232
+ };
233
+ /** #2029: the rewind hold's key for one subject — a module's id, bare as #1819 stored it, or `vertical:<slug>`. */
234
+ const holdKeyOf = (kind, key) => (kind === 'system' ? key : peerSubjectRef(key));
175
235
  export class CloudflareScopeHost {
176
236
  admin;
177
237
  scopeNs;
@@ -389,6 +449,22 @@ export class CloudflareScopeHost {
389
449
  },
390
450
  };
391
451
  }
452
+ /**
453
+ * #1184: the scope reads an executor's handler may make. Plain RPCs to the ScopeDO: the
454
+ * handler runs here on the coordinator, after the DO's own task has returned.
455
+ */
456
+ executorScope(tenantId, scopeId) {
457
+ const stub = this.scopeStub(scopeId);
458
+ return {
459
+ history: async (entity, page) => stub.entityHistory({ entityType: entity.entityType, entityId: entity.entityId, limit: page?.limit, cursor: page?.cursor }),
460
+ covers: async (principal, roleKey, level) => {
461
+ const bound = await stub.canAssignFor(tenantId, scopeId, principalId.parse(principal), roleKey, level === 'tenant');
462
+ if (!bound)
463
+ throw unknownRoleError(roleKey);
464
+ return coverage.parse(bound);
465
+ },
466
+ };
467
+ }
392
468
  /**
393
469
  * Drain this scope's outbox into the registered executors (K-22 §4.2).
394
470
  *
@@ -412,7 +488,9 @@ export class CloudflareScopeHost {
412
488
  * own `invocationId` field reads null — an ambient read there would record "no
413
489
  * call" for every attempt an operation's own tail made.
414
490
  */
415
- invocationId) {
491
+ invocationId,
492
+ /** #1184: what each attempt did, for the emitting call's `onExecutorOutcomes`. */
493
+ outcomes) {
416
494
  const report = {
417
495
  attempted: 0,
418
496
  delivered: 0,
@@ -426,27 +504,22 @@ export class CloudflareScopeHost {
426
504
  // attempt runs — the DO only journals it.
427
505
  const lines = asyncLinePass();
428
506
  try {
429
- await this.drainExecutorsPass(tenantId, scopeId, invocationId, report, lines);
507
+ await this.drainExecutorsPass(tenantId, scopeId, invocationId, report, lines, outcomes);
430
508
  }
431
509
  finally {
432
510
  lines.end();
433
511
  }
434
512
  return report;
435
513
  }
436
- async drainExecutorsPass(tenantId, scopeId, invocationId, report, lines) {
514
+ async drainExecutorsPass(tenantId, scopeId, invocationId, report, lines, outcomes) {
437
515
  const stub = this.scopeStub(scopeId);
516
+ const scope = this.executorScope(tenantId, scopeId);
438
517
  // #2005: a non-primary scope (a fork, a snapshot, a preview of either kind) causes no
439
518
  // outbound effects, so its deliveries are journaled terminal with the reason and no
440
- // handler runs. Asked of the directory where there is one. A CP-less host has none, so it
441
- // asks the scope's own storage whether it is classified a copy (`_substrat_copy_origin`'s
442
- // `is_copy`, #2009, set on the directory's word) — a preview and a snapshot reach a hosted
443
- // vertical as restores the platform marks.
444
- // Asked on the first due event, once per pass: most passes have none, and the directory
445
- // is one global object.
519
+ // handler runs (`isInertScope`). Asked on the first due event, once per pass: most passes
520
+ // have none, and the directory is one global object.
446
521
  let inert;
447
- const isInert = () => (inert ??= this.cpLess
448
- ? stub.isCopy()
449
- : this.cp.getScopeRecord(tenantId, scopeId).then((row) => !isPrimaryScopeRow(row)));
522
+ const isInert = () => (inert ??= this.isInertScope(tenantId, scopeId));
450
523
  // #1713: a CP-less scope its lifecycle holds attempts nothing and journals nothing, so every
451
524
  // due delivery stays due for the first pass after it is live again. Asked on the first due
452
525
  // event, like `isInert`. Every caller passed `assertLive` already, so this is for the scope
@@ -477,6 +550,9 @@ export class CloudflareScopeHost {
477
550
  attempt,
478
551
  startedAt: Date.now(),
479
552
  });
553
+ const outcomeOf = (event, outcome, error) => {
554
+ outcomes?.push(executorOutcomeOf(id, event, outcome, error));
555
+ };
480
556
  for (const bad of undecodable) {
481
557
  report.attempted += 1;
482
558
  const attempt = await stub.recordExecutorAttempt(bad.eventId, deliveryId, bad.error, null, invocationId);
@@ -490,6 +566,7 @@ export class CloudflareScopeHost {
490
566
  // No next attempt, like an undecodable row: a scope does not become primary.
491
567
  const attempt = await stub.recordExecutorAttempt(event.id, deliveryId, INERT_SCOPE_REASON, null, invocationId);
492
568
  report.inert = (report.inert ?? 0) + 1;
569
+ outcomeOf(event, 'inert', INERT_SCOPE_REASON);
493
570
  // Its own outcome, never `delivered`: no handler ran (#2005).
494
571
  lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'inert' });
495
572
  continue;
@@ -507,6 +584,7 @@ export class CloudflareScopeHost {
507
584
  // the at-least-once residue, as it already must in-process.
508
585
  await stub.routeExecutorEventToPlatform(event.id, deliveryId, connectorDispatchKind(executor.provider), JSON.stringify({ executorId: id, event }), JSON.stringify({ system: 'connector-dispatch' }), invocationId);
509
586
  report.routedToPlatform += 1;
587
+ outcomeOf(event, 'routed');
510
588
  // Handed on, not run here: the platform's drain owns its attempts from now on.
511
589
  lines.write({ ...unitOf(event.id, 1), startedAt, outcome: 'routed' });
512
590
  }
@@ -514,13 +592,26 @@ export class CloudflareScopeHost {
514
592
  await executor.handler(await this.connectorContext(tenantId, scopeId, executor.timeoutMs, event.id), event);
515
593
  const attempt = await stub.recordExecutorAttempt(event.id, deliveryId, null, null, invocationId);
516
594
  report.delivered += 1;
595
+ outcomeOf(event, 'delivered');
517
596
  lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'delivered' });
518
597
  }
519
598
  else {
520
- await executor.handler(this.admin, event);
521
- const attempt = await stub.recordExecutorAttempt(event.id, deliveryId, null, null, invocationId);
522
- report.delivered += 1;
523
- lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'delivered' });
599
+ const result = await executor.handler(this.admin, event, scope);
600
+ // #1184: a refusal is the handler's own terminal decision — journaled with its
601
+ // reason, no next attempt, listed by `executorDeadLetters` beside an exhausted one.
602
+ const refused = isDeliveryRefusal(result) ? result : null;
603
+ const text = refused && refusalJournalText(refused);
604
+ const attempt = await stub.recordExecutorAttempt(event.id, deliveryId, text, null, invocationId);
605
+ if (refused) {
606
+ report.deadLettered += 1;
607
+ outcomeOf(event, 'refused', refused.reason);
608
+ lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'dead-lettered', error: text });
609
+ }
610
+ else {
611
+ report.delivered += 1;
612
+ outcomeOf(event, 'delivered');
613
+ lines.write({ ...unitOf(event.id, attempt), startedAt, outcome: 'delivered' });
614
+ }
524
615
  }
525
616
  }
526
617
  catch (err) {
@@ -535,6 +626,7 @@ export class CloudflareScopeHost {
535
626
  report.deadLettered += 1;
536
627
  else
537
628
  report.retrying += 1;
629
+ outcomeOf(event, exhausted ? 'dead-lettered' : 'retrying', err);
538
630
  lines.write({
539
631
  ...unitOf(event.id, attempts),
540
632
  startedAt,
@@ -558,13 +650,14 @@ export class CloudflareScopeHost {
558
650
  // emitting operation's own tail made.
559
651
  return this.drainExecutors(tenantId, scopeId, null);
560
652
  }
561
- registerJob(moduleId, name, handler, retry) {
653
+ registerJob(moduleId, name, handler, retry, options) {
562
654
  // #1575: the kernel's own jobs are dispatched before this registry is read.
563
655
  assertJobRegistrable(moduleId, name);
656
+ assertLeaseMs(options?.leaseMs);
564
657
  const key = `${moduleId}/${name}`;
565
658
  if (this.jobs.has(key))
566
659
  throw new Error(`job '${key}' is already registered`);
567
- this.jobs.set(key, { handler, retry });
660
+ this.jobs.set(key, { handler, retry, leaseMs: options?.leaseMs });
568
661
  }
569
662
  /**
570
663
  * D-14's DURABLE driver: the run store is RPC to the scope DO, and nothing else.
@@ -582,13 +675,15 @@ export class CloudflareScopeHost {
582
675
  const stub = this.scopeStub(scopeId);
583
676
  return {
584
677
  startOrJoin: (key, row) => stub.jobRunStartOrJoin(key.moduleId, key.job, key.instance, row),
585
- get: (id) => stub.jobRunById(id),
586
678
  dueKeys: (now, max) => stub.jobRunsDueKeys(now, max),
679
+ claim: (id, owner, leaseMs) => stub.jobRunClaim(id, owner, leaseMs),
680
+ begin: (id, owner, marginMs) => stub.jobRunBegin(id, owner, marginMs),
681
+ miss: (id, owner, note) => stub.jobRunMiss(id, owner, note),
587
682
  list: (filter) => stub.jobRunList(filter),
588
- patch: (id, patch) => stub.jobRunPatch(id, patch),
589
- commitPass: (id, patch) => stub.jobCommitPass(id, patch),
590
- step: (runId, name) => stub.jobStepRow(runId, name),
591
- recordStep: (runId, name, result, attempts, lastError, at) => stub.jobStepRecord(runId, name, result, attempts, lastError, at),
683
+ patch: (id, patch, owner) => stub.jobRunPatch(id, patch, owner),
684
+ commitPass: (id, patch, owner) => stub.jobCommitPass(id, patch, owner),
685
+ beginStep: (runId, name, owner, leaseMs) => stub.jobStepBegin(runId, name, owner, leaseMs),
686
+ recordStep: (runId, name, result, attempts, lastError, at, owner, leaseMs) => stub.jobStepRecord(runId, name, result, attempts, lastError, at, owner, leaseMs),
592
687
  };
593
688
  }
594
689
  /**
@@ -710,7 +805,7 @@ export class CloudflareScopeHost {
710
805
  if (!rec)
711
806
  throw unknownScopeForTenant(tenantId, scopeId);
712
807
  if (rec.status !== 'active' && rec.status !== 'provisioning') {
713
- throw new Error(`scope not migratable (status: ${rec.status}): ${scopeId}`);
808
+ throw substratError('conflict', `scope not migratable (status: ${rec.status}): ${scopeId}`);
714
809
  }
715
810
  const stub = this.scopeStub(scopeId);
716
811
  try {
@@ -752,11 +847,11 @@ export class CloudflareScopeHost {
752
847
  return this.scopeStub(scopeId).introspectTables();
753
848
  }
754
849
  async introspectScopeTable(scopeId, input) {
755
- return this.scopeStub(scopeId).introspectTable(input.table, input.limit, input.offset);
850
+ return unwrapReply(await this.scopeStub(scopeId).introspectTableReply(input.table, input.limit, input.offset));
756
851
  }
757
852
  /** The SQL console's CP-less path (#219) — same trust line as the pair above. */
758
853
  async introspectScopeQuery(scopeId, input) {
759
- return this.scopeStub(scopeId).introspectQuery(input.sql);
854
+ return unwrapReply(await this.scopeStub(scopeId).introspectQueryReply(input.sql));
760
855
  }
761
856
  /**
762
857
  * The K-35 denial log's CP-less path (#867) — same trust line as the reads above.
@@ -853,10 +948,9 @@ export class CloudflareScopeHost {
853
948
  opts) {
854
949
  if (opts?.markCopy)
855
950
  assertCopyLineage(opts.markCopy);
951
+ const carry = opts ? recordedOffFromWire(opts) : undefined;
856
952
  const load = {
857
- switchOff: opts?.switchedOff
858
- ? { moduleIds: opts.switchedOff, at: new Date().toISOString(), tenantHeld: opts.tenantHeld }
859
- : undefined,
953
+ switchOff: carry ? { at: new Date().toISOString(), ...carry } : undefined,
860
954
  sourceScopeId: opts?.sourceScopeId,
861
955
  exact: opts?.exact,
862
956
  loadStamp: opts?.loadStamp,
@@ -870,7 +964,7 @@ export class CloudflareScopeHost {
870
964
  ...(opts?.expect ? { expect: opts.expect } : {}),
871
965
  });
872
966
  if (out.refused === false) {
873
- return { tables: tables.length, ...(opts?.switchedOff ? { switchedOff: out.switchedOff } : {}) };
967
+ return { tables: tables.length, ...(carry ? { switchedOff: out.switchedOff } : {}) };
874
968
  }
875
969
  if (out.refused === 'kept')
876
970
  throw substratError('conflict', KEPT_COPY_REFUSAL);
@@ -993,7 +1087,7 @@ export class CloudflareScopeHost {
993
1087
  * safe; scope-level tuples (the restored grants) are never touched.
994
1088
  */
995
1089
  async projectRolesLocal(tenantId, scopeId, roles) {
996
- await this.scopeStub(scopeId).applyProjection(tenantId, roles.map((r) => ({ role_key: r.key, permissions: JSON.stringify(r.permissions), source: r.source })), []);
1090
+ unwrapReply(await this.scopeStub(scopeId).applyProjectionReply(tenantId, roles.map((r) => ({ role_key: r.key, permissions: JSON.stringify(r.permissions), source: r.source })), []));
997
1091
  }
998
1092
  /**
999
1093
  * Wipe one scope DO's storage in THIS deployment — the reap half of an orchestrated
@@ -1045,6 +1139,10 @@ export class CloudflareScopeHost {
1045
1139
  async lifecycleFlowLocal(scopeId, input) {
1046
1140
  return this.scopeStub(scopeId).lifecycleFlow(input);
1047
1141
  }
1142
+ /** Business volumes per bucket (#1750) on this host's own scope — the vertical-host read. */
1143
+ async operationSeriesLocal(scopeId, input) {
1144
+ return this.scopeStub(scopeId).operationSeries(input);
1145
+ }
1048
1146
  /** One event's causal chain (#1237) on this host's own scope — the vertical-host read. */
1049
1147
  async eventCauseLocal(scopeId, input) {
1050
1148
  return this.scopeStub(scopeId).eventCause(input);
@@ -1197,7 +1295,7 @@ export class CloudflareScopeHost {
1197
1295
  const record = resolveScopeRecord(input);
1198
1296
  // Fail-closed tenant gate throws out of the awaited cp call BEFORE migrate
1199
1297
  // or audit, so a rejected provision creates nothing and writes no audit row.
1200
- const created = await this.cp.provisionScope(input.tenantId, input.scopeId, record, new Date().toISOString());
1298
+ const created = await this.directory('provisionScope', input.tenantId, input.scopeId, record, new Date().toISOString());
1201
1299
  // Instantiate the scope DO and trigger its lazy migration.
1202
1300
  await this.migrateAndRecord(input.scopeId);
1203
1301
  // Scope-local permissions: a freshly-provisioned scope evaluates from its own
@@ -1238,17 +1336,9 @@ export class CloudflareScopeHost {
1238
1336
  // #1742: in the SAME unit as the seat, so no sweep can land between them; the re-assert
1239
1337
  // after it then finds them off, and audits what the unit moved.
1240
1338
  const ownStore = this.cpLess || record.vertical === null;
1241
- const recordedOff = ownStore ? await this.cp.switchedOffModulesOf(input.tenantId, input.scopeId) : [];
1242
1339
  const seatAt = new Date().toISOString();
1243
- const switchedOff = await this.scopeStub(input.scopeId).seatTuples(seats, recordedOff.length
1244
- ? {
1245
- scopeId: input.scopeId,
1246
- moduleIds: recordedOff,
1247
- at: seatAt,
1248
- // #1823: a recorded-off module held only by a tenant grant goes back off too.
1249
- tenantHeld: await this.cp.tenantHeldSystemModules(input.tenantId, recordedOff, seatAt),
1250
- }
1251
- : undefined);
1340
+ const carry = ownStore ? await this.recordedOffCarry(input.tenantId, input.scopeId, seatAt) : undefined;
1341
+ const switchedOff = await this.scopeStub(input.scopeId).seatTuples(seats, carry ? { scopeId: input.scopeId, at: seatAt, ...carry } : undefined);
1252
1342
  if (ownStore) {
1253
1343
  await this.admin.reassertSystemSwitches(actor, { tenantId: input.tenantId, scopeId: input.scopeId }, { appliedInUnit: switchedOff });
1254
1344
  }
@@ -1270,10 +1360,10 @@ export class CloudflareScopeHost {
1270
1360
  // inside the DO's ledger write, which is the serialization point that actually holds.
1271
1361
  const tenant = await this.cp.getTenant(input.tenantId);
1272
1362
  if (!tenant) {
1273
- throw new Error(`cannot provision tenant store under unknown tenant: ${input.tenantId}`);
1363
+ throw substratError('not_found', `cannot provision tenant store under unknown tenant: ${input.tenantId}`);
1274
1364
  }
1275
1365
  if (tenant.status !== 'active') {
1276
- throw new Error(`cannot provision tenant store under non-active tenant (status: ${tenant.status}): ${input.tenantId}`);
1366
+ throw substratError('conflict', `cannot provision tenant store under non-active tenant (status: ${tenant.status}): ${input.tenantId}`);
1277
1367
  }
1278
1368
  // Idempotent on (tenant, vertical, binding): a retried provision re-resolves the SAME
1279
1369
  // store rather than minting a second database (the K-31 ready-gate retries the whole
@@ -1288,7 +1378,7 @@ export class CloudflareScopeHost {
1288
1378
  // the source of truth, carrying the D1 database_id Cloudflare assigned as the ref.
1289
1379
  const name = await tenantStoreDatabaseName(input.tenantId, input.vertical, input.binding);
1290
1380
  const ref = await d1.create(name);
1291
- const stored = await this.cp.putTenantStore({
1381
+ const stored = await this.directory('putTenantStore', {
1292
1382
  tenantId: input.tenantId,
1293
1383
  vertical: input.vertical,
1294
1384
  binding: input.binding,
@@ -1339,17 +1429,17 @@ export class CloudflareScopeHost {
1339
1429
  const r2 = this.requireBlobStores(`provisionBlobStore(tenant=${input.tenantId} vertical=${input.vertical} binding=${input.binding})`);
1340
1430
  const tenant = await this.cp.getTenant(input.tenantId);
1341
1431
  if (!tenant) {
1342
- throw new Error(`cannot provision blob store under unknown tenant: ${input.tenantId}`);
1432
+ throw substratError('not_found', `cannot provision blob store under unknown tenant: ${input.tenantId}`);
1343
1433
  }
1344
1434
  if (tenant.status !== 'active') {
1345
- throw new Error(`cannot provision blob store under non-active tenant (status: ${tenant.status}): ${input.tenantId}`);
1435
+ throw substratError('conflict', `cannot provision blob store under non-active tenant (status: ${tenant.status}): ${input.tenantId}`);
1346
1436
  }
1347
1437
  const existing = await this.cp.getBlobStore(input.tenantId, input.vertical, input.binding);
1348
1438
  if (existing)
1349
1439
  return { binding: input.binding, kind: 'blob', ref: existing.ref };
1350
1440
  const name = await blobStoreBucketName(input.tenantId, input.vertical, input.binding);
1351
1441
  const ref = await r2.create(name);
1352
- const stored = await this.cp.putBlobStore({
1442
+ const stored = await this.directory('putBlobStore', {
1353
1443
  tenantId: input.tenantId,
1354
1444
  vertical: input.vertical,
1355
1445
  binding: input.binding,
@@ -1488,22 +1578,22 @@ export class CloudflareScopeHost {
1488
1578
  // the DO only to be refused and recorded — never to the blob store.
1489
1579
  const hash = subject.capabilitySession;
1490
1580
  return {
1491
- upload: async (input) => unwrapCapabilityReply(await stub.capabilityAttachmentRefuseWrite(hash, tenantId, scopeId, 'attachments.upload', {
1581
+ upload: async (input) => unwrapReply(await stub.capabilityAttachmentRefuseWrite(hash, tenantId, scopeId, 'attachments.upload', {
1492
1582
  entityType: input.entity.entityType,
1493
1583
  })),
1494
- list: async (entity) => unwrapCapabilityReply(await stub.capabilityAttachmentList(entity, hash, tenantId, scopeId)),
1584
+ list: async (entity) => unwrapReply(await stub.capabilityAttachmentList(entity, hash, tenantId, scopeId)),
1495
1585
  open: async (attachmentId) => {
1496
- const record = unwrapCapabilityReply(await stub.capabilityAttachmentOpen(attachmentId, hash, tenantId, scopeId));
1586
+ const record = unwrapReply(await stub.capabilityAttachmentOpen(attachmentId, hash, tenantId, scopeId));
1497
1587
  return record ? this.openAttachmentBytes(store, scopeId, record) : null;
1498
1588
  },
1499
- remove: async (attachmentId) => unwrapCapabilityReply(await stub.capabilityAttachmentRefuseWrite(hash, tenantId, scopeId, 'attachments.remove', {
1589
+ remove: async (attachmentId) => unwrapReply(await stub.capabilityAttachmentRefuseWrite(hash, tenantId, scopeId, 'attachments.remove', {
1500
1590
  attachmentId,
1501
1591
  })),
1502
1592
  search: async (term, options) => {
1503
1593
  // Judged here too, so a short term reaches the caller as `SearchTermTooShort`
1504
1594
  // rather than as a message that crossed the RPC boundary. The DO judges again.
1505
1595
  searchMatchExpression(term, 'prefix');
1506
- return unwrapCapabilityReply(await stub.capabilityAttachmentSearch(term, searchLimit(options?.limit), hash, tenantId, scopeId));
1596
+ return unwrapReply(await stub.capabilityAttachmentSearch(term, searchLimit(options?.limit), hash, tenantId, scopeId));
1507
1597
  },
1508
1598
  };
1509
1599
  }
@@ -1558,7 +1648,7 @@ export class CloudflareScopeHost {
1558
1648
  throw new Error('attachments.search is not available on a delivery-scoped surface');
1559
1649
  }
1560
1650
  searchMatchExpression(term, 'prefix');
1561
- return unwrapCapabilityReply(await stub.attachmentSearch(term, searchLimit(options?.limit), asPrincipalId, tenantId, scopeId, connectionId));
1651
+ return unwrapReply(await stub.attachmentSearch(term, searchLimit(options?.limit), asPrincipalId, tenantId, scopeId, connectionId));
1562
1652
  },
1563
1653
  };
1564
1654
  }
@@ -1606,24 +1696,51 @@ export class CloudflareScopeHost {
1606
1696
  // this is the scope's real store, as for the provision's seat above; a hosted scope is
1607
1697
  // restored in its deployment, which the platform carries the same list to.
1608
1698
  const ownStore = this.cpLess || existing.vertical === null;
1609
- const recordedOff = ownStore ? await this.cp.switchedOffModulesOf(tenantId, scopeId) : [];
1610
1699
  const replayAt = new Date().toISOString();
1700
+ const carry = ownStore ? await this.recordedOffCarry(tenantId, scopeId, replayAt) : undefined;
1611
1701
  const switchedOff = await this.scopeStub(scopeId).importDump(dump.tables, scopeId, {
1612
- switchOff: recordedOff.length
1613
- ? {
1614
- moduleIds: recordedOff,
1615
- at: replayAt,
1616
- // #1823: a recorded-off module held only by a tenant grant goes back off too.
1617
- tenantHeld: await this.cp.tenantHeldSystemModules(tenantId, recordedOff, replayAt),
1618
- }
1619
- : undefined,
1702
+ switchOff: carry ? { at: replayAt, ...carry } : undefined,
1620
1703
  sourceScopeId: opts?.sourceScopeId ?? dump.scopeId,
1621
1704
  });
1622
1705
  await this.recordAdmin(actor, 'restoreScope', { tenantId, scopeId }, null, { sourceScopeId: dump.scopeId, tables: dump.tables.length, capturedAt: dump.capturedAt });
1623
- if (recordedOff.length) {
1706
+ if (carry) {
1624
1707
  await this.admin.reassertSystemSwitches(actor, { tenantId, scopeId }, { appliedInUnit: switchedOff });
1625
1708
  }
1626
1709
  }
1710
+ /**
1711
+ * #1742, #2029: what this host's own unit carries of the directory's record — the modules and
1712
+ * the peers recorded OFF on the scope, each with the ones held there only by a tenant-level
1713
+ * grant (#1823, #2030). Undefined when nothing is recorded off, so such a unit is unchanged.
1714
+ */
1715
+ async recordedOffCarry(tenantId, scopeId, at) {
1716
+ const [modules, peers] = await Promise.all([
1717
+ this.recordedOff('system', tenantId, scopeId, at),
1718
+ this.recordedOff('peer', tenantId, scopeId, at),
1719
+ ]);
1720
+ if (!modules.keys.length && !peers.keys.length)
1721
+ return undefined;
1722
+ return {
1723
+ moduleIds: modules.keys,
1724
+ tenantHeld: modules.tenantHeld,
1725
+ verticals: peers.keys,
1726
+ tenantHeldVerticals: peers.tenantHeld,
1727
+ fences: switchFencesCarried([
1728
+ ['system', modules],
1729
+ ['peer', peers],
1730
+ ]),
1731
+ };
1732
+ }
1733
+ /** The subjects of one kind the directory records OFF on the scope, and of those the tenant-held ones. */
1734
+ async recordedOff(kind, tenantId, scopeId, at) {
1735
+ // #2045: the OFF rows and their fences in one read, so every key carried has its own row's fence.
1736
+ const fences = new Map((await this.cp.switchRecordStatesOf(kind, tenantId, scopeId))
1737
+ .filter(([, row]) => row.position === 'off')
1738
+ .map(([key, row]) => [key, row.fence]));
1739
+ const keys = [...fences.keys()];
1740
+ if (!keys.length)
1741
+ return { keys, tenantHeld: [], fences };
1742
+ return { keys, tenantHeld: await this.cp.tenantHeldOf(kind, tenantId, keys, at), fences };
1743
+ }
1627
1744
  async snapshotScope(actor, tenantId, scopeId, opts) {
1628
1745
  const source = await this.admin.getScopeRecord(actor, tenantId, scopeId);
1629
1746
  if (!source)
@@ -1654,7 +1771,7 @@ export class CloudflareScopeHost {
1654
1771
  if (!rec)
1655
1772
  throw substratError('not_found', `unknown scope ${scopeId} in tenant ${tenantId}`);
1656
1773
  if (!rec.forkedFrom && rec.kind !== 'preview') {
1657
- throw new Error(`scope ${scopeId} is not a fork or preview — only previews may be deleted; ` +
1774
+ throw substratError('forbidden', `scope ${scopeId} is not a fork or preview — only previews may be deleted; ` +
1658
1775
  `archive a primary scope instead`);
1659
1776
  }
1660
1777
  // Storage BEFORE the directory row: a crash between the two leaves a visible row
@@ -1815,34 +1932,55 @@ export class CloudflareScopeHost {
1815
1932
  /** #1834: the job pass this door is opened for; its "not now" refusals are tied to it. */
1816
1933
  pass) {
1817
1934
  if (!this.moduleIds.has(moduleId)) {
1818
- throw new Error(`module not registered on this host: ${moduleId}`);
1935
+ throw substratError('not_found', `module not registered on this host: ${moduleId}`);
1819
1936
  }
1820
1937
  await this.assertLive(tenantId, scopeId);
1821
1938
  await this.migrateAndRecord(scopeId);
1939
+ return this.openDoor('system', moduleId, scopeId, gated, pass);
1940
+ }
1941
+ /**
1942
+ * #1834: one door, for either kind of switched subject (#2029). The gate is read here unless the
1943
+ * caller read it (`runDueSchedules` reports a hold it could not read on its pass report); a held
1944
+ * subject is refused, and a call landing on another instance than the gate's re-gates, at most
1945
+ * `SYSTEM_DOOR_REGATES` times, then fails closed.
1946
+ *
1947
+ * The peer door (`kind: 'peer'`) is the system door's for a peer vertical. A PITR rewind to before
1948
+ * a peer was switched off brings its grants back live and its OFF marker gone, so the rewind holds
1949
+ * it outside the scope (`rewindHolding`, as for a module), and every peer entry — an invoke, a
1950
+ * delivery, a coverage read, an export read for it as consumer — opens this door after
1951
+ * `peerScopeGate`.
1952
+ */
1953
+ async openDoor(kind, key, scopeId, gated, pass) {
1954
+ const words = DOOR_WORDS[kind];
1955
+ const what = words.what(key);
1822
1956
  // A gate this door reads itself reports a hold it could not read; one handed in was the
1823
- // caller's to report (`runDueSchedules` puts it on its pass report).
1957
+ // caller's to report.
1824
1958
  const regate = async () => {
1825
- const fresh = await this.systemDoorGate(scopeId, moduleId);
1959
+ const fresh = await this.doorGate(kind, scopeId, key);
1826
1960
  if (fresh.error)
1827
- console.error(`substrat: system door ${moduleId} on ${scopeId}: ${fresh.error}`);
1961
+ console.error(`substrat: door of ${what} on ${scopeId}: ${fresh.error}`);
1828
1962
  return fresh;
1829
1963
  };
1830
1964
  let gate = gated ?? (await regate());
1831
1965
  return {
1832
- moduleId,
1966
+ kind,
1967
+ key,
1833
1968
  through: async (call) => {
1834
1969
  for (let regates = 0;; regates += 1) {
1835
1970
  if (gate.held) {
1836
1971
  // The hold ends when the switch is applied again: a job run waits it out without
1837
1972
  // spending its retries, because the driver defers on THIS refusal (`doorWait`).
1838
- throw this.doorWait(pass, substratError('forbidden', `module '${moduleId}' is held off on this scope: it was rewound to before its schedule ` +
1839
- 'switch was turned off, and it stays off until the switch is applied again', { reason: SYSTEM_DOOR_WAIT }));
1973
+ const held = substratError('forbidden', `${what} is held off on this scope: it was rewound to before its ` +
1974
+ `${words.switchName} switch was turned off, and it stays off until ` +
1975
+ 'the switch is applied again', { reason: SYSTEM_DOOR_WAIT });
1976
+ this.heldRefusals.add(held);
1977
+ throw this.doorWait(pass, held);
1840
1978
  }
1841
1979
  const answer = await call(gate.instance);
1842
1980
  if (!isSystemDoorMoved(answer))
1843
1981
  return answer;
1844
1982
  if (regates >= SYSTEM_DOOR_REGATES) {
1845
- throw this.doorWait(pass, substratError('unavailable', `the scope kept restarting under the system door of module '${moduleId}' ` +
1983
+ throw this.doorWait(pass, substratError('unavailable', `the scope kept restarting under the ${words.door} of ${what} ` +
1846
1984
  `(${regates + 1} attempts); nothing ran. Retry later`, { reason: SYSTEM_DOOR_WAIT }));
1847
1985
  }
1848
1986
  gate = await regate();
@@ -1859,6 +1997,16 @@ export class CloudflareScopeHost {
1859
1997
  * on a later pass is an ordinary failure there. Held weakly: an error nobody holds goes.
1860
1998
  */
1861
1999
  doorWaits = new WeakMap();
2000
+ /**
2001
+ * #2029: the hold refusals this host's doors threw, by identity — so a peer delivery pauses its
2002
+ * edge, and a coverage read answers "holds nothing", on exactly those and on no error that only
2003
+ * looks like one. Held weakly, as `doorWaits` is.
2004
+ */
2005
+ heldRefusals = new WeakSet();
2006
+ /** #2029: is `err` a hold refusal this host's door threw? */
2007
+ isHeldRefusal(err) {
2008
+ return typeof err === 'object' && err !== null && this.heldRefusals.has(err);
2009
+ }
1862
2010
  /** #1834: mark this door's own refusal as "not now" for the pass it was opened for, and hand it back. */
1863
2011
  doorWait(pass, err) {
1864
2012
  if (pass)
@@ -1873,13 +2021,23 @@ export class CloudflareScopeHost {
1873
2021
  return true;
1874
2022
  }
1875
2023
  /**
1876
- * #1834: the system door's gate — the scope's state read for the module, the instance that
1877
- * answered it, and then (only for a module the scope has on) the rewind hold (#1819). The
2024
+ * #1834: a door's gate — the scope's state read for the subject, the instance that answered it,
2025
+ * and then the rewind hold (#1819), for any subject the scope does not already have off. The
1878
2026
  * hold is read AFTER the state, which is the order `switchHeld` needs.
2027
+ *
2028
+ * `ungranted` reads the hold too (#2029): a subject held on the scope only by a TENANT-level
2029
+ * grant (#1823, #2030) comes back from a rewind with no scope grant and no marker, and the
2030
+ * tenant grant still authorizes it there. A rewind captured it off, so the hold must keep it so.
1879
2031
  */
1880
- async systemDoorGate(scopeId, moduleId) {
1881
- const { state, instance } = await this.scopeStub(scopeId).systemDoorState(moduleId);
1882
- const hold = state === 'on' ? await this.switchHeld(scopeId, moduleId) : { held: false };
2032
+ async doorGate(kind, scopeId, key) {
2033
+ const stub = this.scopeStub(scopeId);
2034
+ const { state, instance } = await (kind === 'system' ? stub.systemDoorState(key) : stub.peerDoorState(key));
2035
+ const cleared = `${scopeId} ${instance}`;
2036
+ if (state === 'off' || CLEAR_SCOPE_INSTANCES.has(cleared))
2037
+ return { state, instance, held: false };
2038
+ const hold = await this.switchHeld(scopeId, holdKeyOf(kind, key));
2039
+ if (hold.scopeClear)
2040
+ rememberClearInstance(cleared);
1883
2041
  return { state, instance, held: hold.held, ...(hold.error ? { error: hold.error } : {}) };
1884
2042
  }
1885
2043
  /**
@@ -1893,8 +2051,25 @@ export class CloudflareScopeHost {
1893
2051
  // The peer door's own gate (#1706): K-3's pair check, the refusal for a scope served
1894
2052
  // elsewhere, the lifecycle check and the migration. A delivery is a write into the
1895
2053
  // consumer's scope, so it takes the gate an invoke through that door takes.
1896
- await this.peerScopeGate(tenantId, scopeId, 'deliverToPeer');
1897
- const result = importResult.parse(await this.scopeStub(scopeId).importApply(batch, tenantId, scopeId));
2054
+ const record = await this.peerScopeGate(tenantId, scopeId, 'deliverToPeer');
2055
+ // #2004: a copy consumes no other vertical's events, at the door as in the sweep
2056
+ // (`scope-copy.ts`). A pause: nothing runs and the watermark stays.
2057
+ if (await this.isInertScope(tenantId, scopeId, record)) {
2058
+ return { ...emptyImportResult(batch), paused: { reason: INERT_SCOPE_REASON } };
2059
+ }
2060
+ // #2029: the producer is a peer, so the delivery goes through its door: a producer the rewind
2061
+ // hold keeps off pauses the edge, exactly as a switched-off one does inside the DO.
2062
+ const door = await this.openDoor('peer', batch.source.vertical, scopeId);
2063
+ let applied;
2064
+ try {
2065
+ applied = await door.through((instance) => this.scopeStub(scopeId).importApply(batch, tenantId, scopeId, instance));
2066
+ }
2067
+ catch (err) {
2068
+ if (!this.isHeldRefusal(err))
2069
+ throw err;
2070
+ return { ...emptyImportResult(batch), paused: { reason: err.message } };
2071
+ }
2072
+ const result = importResult.parse(applied);
1898
2073
  if (result.delivered > 0) {
1899
2074
  try {
1900
2075
  await this.drainExecutors(tenantId, scopeId, null);
@@ -1938,7 +2113,9 @@ export class CloudflareScopeHost {
1938
2113
  async getVerticalScope(caller, tenantId, scopeId, options) {
1939
2114
  const parsed = verticalCaller.parse(caller);
1940
2115
  await this.peerScopeGate(tenantId, scopeId, 'getVerticalScope');
1941
- return this.buildStub(tenantId, scopeId, undefined, undefined, undefined, options, undefined, undefined, parsed);
2116
+ // #2029: gated against the rewind hold and pinned, as the system door is.
2117
+ const door = await this.openDoor('peer', parsed.vertical, scopeId);
2118
+ return this.buildStub(tenantId, scopeId, undefined, undefined, door, options, undefined, undefined, parsed);
1942
2119
  }
1943
2120
  /**
1944
2121
  * Does peer `vertical` hold each key at this scope's node now (#1706) — the ScopeDO's own
@@ -1946,9 +2123,19 @@ export class CloudflareScopeHost {
1946
2123
  */
1947
2124
  async peerCovers(tenantId, scopeId, vertical, permissions) {
1948
2125
  await this.peerScopeGate(tenantId, scopeId, 'peerCovers');
1949
- return peerCoverage
1950
- .array()
1951
- .parse(await this.scopeStub(scopeId).peerCovers(tenantId, scopeId, verticalSlug.parse(vertical), [...permissions]));
2126
+ const slug = verticalSlug.parse(vertical);
2127
+ // #2029: a peer the rewind hold keeps off holds nothing here, as a switched-off one doesn't.
2128
+ const door = await this.openDoor('peer', slug, scopeId);
2129
+ try {
2130
+ return peerCoverage
2131
+ .array()
2132
+ .parse(await door.through((instance) => this.scopeStub(scopeId).peerCovers(tenantId, scopeId, slug, [...permissions], instance)));
2133
+ }
2134
+ catch (err) {
2135
+ if (this.isHeldRefusal(err))
2136
+ return permissions.map((permission) => ({ permission, held: false }));
2137
+ throw err;
2138
+ }
1952
2139
  }
1953
2140
  /**
1954
2141
  * `ctx.canAssign`'s bound for a principal the host names (#1931) — the ScopeDO's own role read
@@ -1994,8 +2181,9 @@ export class CloudflareScopeHost {
1994
2181
  // door's own tenant-confinement test proved. A RECORD crosses intact, so the coordinator
1995
2182
  // reads the record and types the refusal itself. `validateScopeAccess` still runs below
1996
2183
  // and still owns the lifecycle half (a suspended tenant or scope).
2184
+ let record;
1997
2185
  if (!this.cpLess) {
1998
- const record = await this.cp.getScopeRecord(tenantId, scopeId);
2186
+ record = await this.cp.getScopeRecord(tenantId, scopeId);
1999
2187
  if (!record) {
2000
2188
  throw unknownScopeForTenant(tenantId, scopeId);
2001
2189
  }
@@ -2003,6 +2191,23 @@ export class CloudflareScopeHost {
2003
2191
  }
2004
2192
  await this.assertLive(tenantId, scopeId);
2005
2193
  await this.migrateAndRecord(scopeId);
2194
+ return record;
2195
+ }
2196
+ /**
2197
+ * Whether this scope is held inert (#2005): a fork, a snapshot or a preview, which causes no
2198
+ * outbound effects and consumes no other vertical's events (#2004). The one primacy question
2199
+ * every such door asks. Where there is a directory its record answers (`record`, the one
2200
+ * `peerScopeGate` returns, when the caller has already read it). A CP-less host has none, so it asks the scope's own storage
2201
+ * whether it is classified a copy (`_substrat_copy_origin`'s `is_copy`, #2009, set on the
2202
+ * directory's word): a preview and a snapshot reach a hosted vertical as restores the
2203
+ * platform marks.
2204
+ */
2205
+ isInertScope(tenantId, scopeId, record) {
2206
+ if (this.cpLess)
2207
+ return this.scopeStub(scopeId).isCopy();
2208
+ if (record)
2209
+ return Promise.resolve(!isPrimaryScopeRow(record));
2210
+ return this.cp.getScopeRecord(tenantId, scopeId).then((row) => !isPrimaryScopeRow(row));
2006
2211
  }
2007
2212
  /**
2008
2213
  * The capability's attachment door (#1686) — `getCapabilityScope`'s gate, then the
@@ -2183,7 +2388,7 @@ export class CloudflareScopeHost {
2183
2388
  // #1819: a scope rewound to before its switch was pulled answers `on`, and the rewind held
2184
2389
  // the module outside the scope. Both are read by the system door's own gate (#1834), which
2185
2390
  // every fire below then goes through, pinned to the instance this read came from.
2186
- const gate = await this.systemDoorGate(scopeId, moduleId);
2391
+ const gate = await this.doorGate('system', scopeId, moduleId);
2187
2392
  if (gate.state === 'ungranted')
2188
2393
  return report;
2189
2394
  if (gate.error)
@@ -2258,10 +2463,11 @@ export class CloudflareScopeHost {
2258
2463
  /** The stub body, shared by the principal, connection, system and impersonation doors. */
2259
2464
  buildStub(tenantId, scopeId, principal, connectionId,
2260
2465
  /**
2261
- * #1834: the system door this stub acts through. The ONLY way to make a stub acting as
2466
+ * #1834: the door this stub acts through. The ONLY way to make a stub acting as
2262
2467
  * `system:<moduleId>`: the module is the door's, and every invoke is gated and pinned by it.
2468
+ * #2029: a peer stub's door too (`openDoor('peer', …)`), gating and pinning each invoke the same way.
2263
2469
  */
2264
- systemDoor, options,
2470
+ door, options,
2265
2471
  /**
2266
2472
  * K-42: the session this stub acts under. Re-read from the directory on
2267
2473
  * EVERY invoke rather than captured here — a stub is a capability and
@@ -2282,7 +2488,7 @@ export class CloudflareScopeHost {
2282
2488
  */
2283
2489
  verticalCaller) {
2284
2490
  const stub = this.scopeStub(scopeId);
2285
- const systemModuleId = systemDoor?.moduleId;
2491
+ const systemModuleId = door?.kind === 'system' ? door.key : undefined;
2286
2492
  const cp = this.cp;
2287
2493
  const operationEntitlement = this.operationEntitlement;
2288
2494
  // The DO needs SOME principal-shaped value for `ctx.principal`; for a
@@ -2338,10 +2544,10 @@ export class CloudflareScopeHost {
2338
2544
  // K-42: resolved per invoke, so expiry and `endImpersonation` both bite.
2339
2545
  const session = sessionId === undefined ? undefined : await this.resolveImpersonation(sessionId, tenantId, scopeId);
2340
2546
  const send = (systemDoorInstance) => stub.invoke(operation, input, asPrincipalId, tenantId, scopeId, connectionId, requiredKey, systemModuleId, true, invokeOptions, session, capabilitySession, verticalCaller, systemDoorInstance);
2341
- // #1834: through the system door, pinned to the instance its gate read. The DO's
2342
- // refusal comes back as a failure envelope, so it is rethrown for the door to re-gate.
2343
- const envelope = systemDoor
2344
- ? await systemDoor.through(async (instance) => {
2547
+ // #1834: through the door (a module's, or #2029 a peer's), pinned to the instance its gate
2548
+ // read. A missed pin comes back as an answer, which `through` re-gates on.
2549
+ const envelope = door
2550
+ ? await door.through(async (instance) => {
2345
2551
  const sent = await send(instance);
2346
2552
  return sent.systemDoorMoved === true ? SYSTEM_DOOR_MOVED : sent;
2347
2553
  })
@@ -2394,9 +2600,9 @@ export class CloudflareScopeHost {
2394
2600
  throw substratError('unavailable', `${operation} ran on a scope host that did not understand peer callers. ` +
2395
2601
  'Retry once the scope has been migrated');
2396
2602
  }
2397
- // #1834, the same shape: a DO too old to know the system door's pin ran the call
2398
- // without checking it met the storage the door's gate read.
2399
- if (systemDoor !== undefined && envelope.systemDoor?.honoured !== true) {
2603
+ // #1834, the same shape: a DO too old to know the door's pin ran the call without
2604
+ // checking it met the storage the door's gate read.
2605
+ if (door !== undefined && envelope.systemDoor?.honoured !== true) {
2400
2606
  throw substratError('unavailable', `${operation} ran on a scope host that did not understand the system door's pin. ` +
2401
2607
  'Retry once the scope has been migrated');
2402
2608
  }
@@ -2410,11 +2616,13 @@ export class CloudflareScopeHost {
2410
2616
  // this scope's outbox — and draining anyway would run executors and
2411
2617
  // connectors as a side effect of a session that may not have side effects.
2412
2618
  // Whatever the outbox already held is the drain sweep's own backstop.
2619
+ // #1184: what this call's executors did inline, for `onExecutorOutcomes`.
2620
+ const executorOutcomes = invokeOptions?.onExecutorOutcomes ? [] : undefined;
2413
2621
  const drained = session?.mode === 'read-only'
2414
2622
  ? { attempted: 0, delivered: 0, retrying: 0, deadLettered: 0, routedToPlatform: 0 }
2415
2623
  // #1525: this drain is part of the call that emitted the events — the
2416
2624
  // coordinator's half of the post-commit tail the DO ran the consumers in.
2417
- : await this.drainExecutors(tenantId, scopeId, invokeOptions?.invocationId ?? null);
2625
+ : await this.drainExecutors(tenantId, scopeId, invokeOptions?.invocationId ?? null, executorOutcomes);
2418
2626
  // #458: the operation committed having enqueued platform intents — tell the
2419
2627
  // caller's harness so it can flag the response for the router kick (#381).
2420
2628
  // Routed connector deliveries (#574 phase 3) count too: the inline drain just
@@ -2431,6 +2639,9 @@ export class CloudflareScopeHost {
2431
2639
  invokeOptions?.onIdempotentReplay?.();
2432
2640
  if (envelope.emitted)
2433
2641
  invokeOptions?.onEmitted?.(envelope.emitted);
2642
+ if (executorOutcomes && session?.mode !== 'read-only' && !envelope.idempotency?.replayed) {
2643
+ invokeOptions?.onExecutorOutcomes?.(executorOutcomes);
2644
+ }
2434
2645
  return envelope.result;
2435
2646
  },
2436
2647
  };
@@ -2523,7 +2734,7 @@ export class CloudflareScopeHost {
2523
2734
  */
2524
2735
  const requireOrg = async (tenant, id) => {
2525
2736
  if (!(await this.cp.readOrg(tenant, id))) {
2526
- throw new Error(`unknown org ${id} in tenant ${tenant}`);
2737
+ throw substratError('not_found', `unknown org ${id} in tenant ${tenant}`);
2527
2738
  }
2528
2739
  };
2529
2740
  const mapScope = (r) => scopeSchema.parse({
@@ -2610,24 +2821,20 @@ export class CloudflareScopeHost {
2610
2821
  }
2611
2822
  };
2612
2823
  /**
2613
- * #1666: move one module's schedule switch on one scope — see `HostAdmin.revokeFromSystem`.
2824
+ * Where one scope's kill switch of one kind lands — the ONE rule the operator's switch and
2825
+ * the #1674 re-assert share, so a re-assert moves exactly the switch an operator's OFF would
2826
+ * have. `kind` is a module's schedule switch (#1666) or a peer's (#1706, #2029); `key` is the
2827
+ * module id or the peer's slug.
2614
2828
  *
2615
2829
  * Where the write lands is the whole point of the delegation branch. The shared control
2616
- * plane's own `SCOPE` namespace is the module-less placeholder, so a hosted scope's
2617
- * switch has to be moved in the deployment serving it; writing it here would report a
2618
- * switch pulled while every schedule kept firing. The audit row is written HERE either
2619
- * way — the deployment's host is CP-less, and its `recordAdmin` is a no-op.
2830
+ * plane's own `SCOPE` namespace is the module-less placeholder, so a hosted scope's switch
2831
+ * has to be moved in the deployment serving it, over that kind's delegation; writing it here
2832
+ * would report a switch pulled while every schedule kept firing, or every peer call kept
2833
+ * being admitted. A scope bound to no vertical (#1666 review) has no deployment to delegate
2834
+ * to: its store is the DO here, so the switch moves (or answers `held: false`) here too.
2835
+ * Delegating it would throw "no deployment serving scope" instead.
2620
2836
  */
2621
- /**
2622
- * #1706: move one PEER's kill switch on one scope — see `HostAdmin.revokeFromPeer`. The
2623
- * schedule switch's audit discipline (intent first, outcome after, every attempt) and,
2624
- * since part 3, its delegation: a scope served by a vertical's own deployment is switched
2625
- * THERE, over `PeerSwitchDelegation`. Without one configured the scope is still refused
2626
- * `unavailable` (`peerScopeGate`) rather than switched in the placeholder namespace.
2627
- */
2628
- const switchPeerAt = async (actor, raw, to) => {
2629
- const input = peerSwitch.parse(raw);
2630
- const { tenantId, scopeId } = input.node;
2837
+ const switchTarget = async (kind, tenantId, scopeId) => {
2631
2838
  let vertical = null;
2632
2839
  if (!this.cpLess) {
2633
2840
  const rec = await this.cp.getScopeRecord(tenantId, scopeId);
@@ -2635,136 +2842,109 @@ export class CloudflareScopeHost {
2635
2842
  throw unknownScopeForTenant(tenantId, scopeId);
2636
2843
  vertical = rec.vertical;
2637
2844
  }
2638
- const action = to === 'off' ? 'revokeFromPeer' : 'restoreToPeer';
2639
- // Where the write lands is the whole point of this branch (#1706 part 3, mirroring
2640
- // #1666). A scope bound to a vertical is served by that vertical's own deployment, and
2641
- // its `vertical:<slug>` grants live there; switching it in the shared control plane's
2642
- // placeholder namespace would tombstone nothing and report a peer cut off while every
2643
- // call it makes kept being admitted. A scope bound to NO vertical has its store here,
2644
- // so it switches here — delegating it would throw "no deployment serving scope".
2645
- // With no delegation configured the old refusal stands, loudly, rather than a write
2646
- // into the wrong namespace.
2647
- const delegation = this.cpLess || vertical !== null ? this.peerSwitchDelegation : undefined;
2648
- if (delegation) {
2649
- // The gate's lifecycle half still runs (a suspended tenant or scope cannot be
2650
- // switched); its `scopeStub` half is what the delegation replaces.
2651
- await this.validateScopeAccess(tenantId, scopeId);
2652
- }
2653
- else {
2654
- await this.peerScopeGate(tenantId, scopeId, action);
2655
- }
2656
- const operationId = ulid();
2657
- const target = { tenantId, scopeId, vertical };
2658
- const base = { operationId, vertical: input.vertical, calls: to };
2659
- await this.recordAdmin(actor, action, target, null, { ...base, phase: 'intent', reason: input.reason });
2660
- let outcome;
2661
- try {
2662
- outcome = delegation
2663
- ? await delegation.switch({ tenantId, scopeId, vertical: input.vertical, to })
2664
- : await this.scopeStub(scopeId).switchPeer(input.vertical, scopeId, to, new Date().toISOString());
2665
- }
2666
- catch (err) {
2667
- await this.recordAdmin(actor, action, target, null, {
2668
- ...base,
2669
- phase: 'failed',
2670
- error: err instanceof Error ? err.message : String(err),
2671
- }).catch(() => undefined);
2672
- throw err;
2673
- }
2674
- await this.recordAdmin(actor, action, target, null, {
2675
- ...base,
2676
- phase: outcome.held ? 'applied' : 'refused',
2677
- changed: outcome.changed,
2678
- permissions: outcome.permissions,
2679
- });
2680
- if (!outcome.held) {
2681
- throw substratError('not_found', `scope ${scopeId} holds no grant for peer '${input.vertical}' — nothing to switch ${to} ` +
2682
- `(check the slug: it is the calling vertical's registry id, as the target's \`peers\` names it)`);
2683
- }
2684
- return {
2685
- operationId,
2686
- vertical: input.vertical,
2687
- calls: to,
2688
- changed: outcome.changed,
2689
- permissions: outcome.permissions,
2690
- };
2691
- };
2692
- /**
2693
- * Where one scope's schedule switch lands — the ONE rule `switchSystem` and the #1674
2694
- * re-assert share, so a re-assert moves exactly the switch an operator's OFF would have.
2695
- * A scope bound to no vertical (#1666 review) has no deployment to delegate to: its store
2696
- * is the DO here, so the switch moves (or answers `held: false`) here too. Delegating it
2697
- * would throw "no deployment serving scope" instead. A scope WITH a vertical still
2698
- * delegates, and still fails loudly when none serves it.
2699
- */
2700
- const systemSwitchTarget = async (tenantId, scopeId) => {
2701
- let vertical = null;
2702
- if (!this.cpLess) {
2703
- const rec = await this.cp.getScopeRecord(tenantId, scopeId);
2704
- if (!rec)
2705
- throw unknownScopeForTenant(tenantId, scopeId);
2706
- vertical = rec.vertical;
2707
- }
2708
- const delegation = this.cpLess || vertical !== null ? this.systemSwitchDelegation : undefined;
2845
+ const hosted = this.cpLess || vertical !== null;
2846
+ const systemDelegation = kind === 'system' && hosted ? this.systemSwitchDelegation : undefined;
2847
+ const peerDelegation = kind === 'peer' && hosted ? this.peerSwitchDelegation : undefined;
2848
+ const delegated = systemDelegation !== undefined || peerDelegation !== undefined;
2709
2849
  // #1819: a scope whose store is here was rewound here, so `switchInScope` releases its
2710
2850
  // hold here too. A delegated move releases in the deployment serving the scope.
2711
- // #1823: a module whose only authority on the scope is a TENANT-level grant has nothing in
2712
- // the scope's storage for the switch to find, so the directory says whether it is held.
2713
- // A caller moving several modules reads them in one call and passes each answer in.
2714
- // An OFF of a tenant-held module is refused, after the far end answers, unless that answer
2851
+ // #1823 (#2030 for a peer): a subject whose only authority on the scope is a TENANT-level
2852
+ // grant has nothing in the scope's storage for the switch to find, so the directory says
2853
+ // whether it is held. A caller moving several reads them in one call and passes each in.
2854
+ // An OFF of a tenant-held MODULE is refused, after the far end answers, unless that answer
2715
2855
  // attests the evaluator that denies a tenant-level grant (`deniesTenantGrants`). A
2716
2856
  // deployment built before #1823 drops `tenantHeld`, still answers `held: true` (the module's
2717
2857
  // scope-level grants were there to switch), and keeps authorizing the tenant-level grant —
2718
- // so its `held` alone would have the platform record a scope off that is not. Every OFF the
2719
- // platform drives moves through here: an operator's (`switchSystem`, which then puts the
2720
- // switch back) and a re-assert's, which every carry ends with.
2721
- const move = async (moduleId, to, at, tenantHeld) => {
2722
- const held = tenantHeld ?? (await this.cp.tenantHeldSystemModules(tenantId, [moduleId], at)).length > 0;
2723
- const outcome = await (delegation
2724
- ? delegation.switch({ tenantId, scopeId, moduleId, to, tenantHeld: held })
2725
- : this.switchInScope(scopeId, moduleId, to, at, held));
2726
- if (to === 'off' && held && outcome.held && outcome.deniesTenantGrants !== true) {
2727
- throw new UnattestedSwitchOff(scopeId, moduleId, outcome);
2858
+ // so its `held` alone would have the platform record a scope off that is not. A peer needs
2859
+ // no attestation: its door refuses on the marker in every build that has the peer switch at
2860
+ // all (`admitPeer` and `switchPeer` arrived together, #1714). Every OFF the platform drives
2861
+ // moves through here: an operator's (`switchSubjectAt`) and a re-assert's, which every carry
2862
+ // ends with.
2863
+ //
2864
+ // #2045 (Codex r3): whether a delegated deployment honours the switch fence is settled
2865
+ // BEFORE anything moves (`attestFence`, once per target), never after the fact by putting a
2866
+ // move back. Without the fence an ON is refused and an OFF goes through unfenced (see
2867
+ // `UnattestedSwitch`). A move a fenced deployment answers without the attestation still
2868
+ // throws, as defence in depth against a rollback between the preflight and the move.
2869
+ const delegation = systemDelegation ?? peerDelegation;
2870
+ let attested;
2871
+ const attestFence = () => (attested ??= delegation ? delegation.fenceSupported({ tenantId, scopeId }) : Promise.resolve(true));
2872
+ const move = async (key, to, at, tenantHeld,
2873
+ /** #2045: the switch call's fence — its operation id, or for a re-assert the record's. */
2874
+ fence) => {
2875
+ const fenced = await attestFence();
2876
+ if (!fenced && to === 'on')
2877
+ throw new UnattestedSwitch(unfencedMessage(scopeId));
2878
+ const held = tenantHeld ?? (await this.cp.tenantHeldOf(kind, tenantId, [key], at)).length > 0;
2879
+ const outcome = await (systemDelegation
2880
+ ? systemDelegation.switch({ tenantId, scopeId, moduleId: key, to, tenantHeld: held, fence })
2881
+ : peerDelegation
2882
+ ? peerDelegation.switch({ tenantId, scopeId, vertical: key, to, tenantHeld: held, fence })
2883
+ : this.switchInScope(kind, scopeId, key, to, at, held, fence));
2884
+ // An unfenced OFF answers without `fenced`, and that is what the callers read it by.
2885
+ if (fenced && outcome.fenced !== true)
2886
+ throw new UnattestedSwitch(unfencedMessage(scopeId));
2887
+ if (outcome.superseded)
2888
+ return outcome;
2889
+ if (kind === 'system' && to === 'off' && held && outcome.held && outcome.deniesTenantGrants !== true) {
2890
+ throw new UnattestedSwitch(undeniedMessage(scopeId, key));
2728
2891
  }
2729
2892
  return outcome;
2730
2893
  };
2731
- /**
2732
- * Is the module switched off on the scope right now? Read from wherever `move` writes — the
2733
- * deployment's status read or this scope's own — for a move that threw (#1823): the switch
2734
- * may already have moved before the throw (`switchInScope` moves it, then works the rewind
2735
- * hold), and then the directory's record must stay.
2736
- */
2737
- const isOff = async (moduleId) => {
2738
- const rows = delegation
2739
- ? await delegation.status({ tenantId, scopeId })
2740
- : systemScheduleEntry.array().parse(await this.scopeStub(scopeId).systemGrantsStatus());
2741
- return rows.some((r) => r.moduleId === moduleId && r.schedules === 'off');
2742
- };
2743
- return { vertical, move, isOff };
2894
+ return { vertical, delegated, move, attestFence };
2744
2895
  };
2745
- const switchSystem = async (actor, raw, to) => {
2746
- const input = systemSwitch.parse(raw);
2747
- const { tenantId, scopeId } = input.node;
2748
- const { vertical, move, isOff } = await systemSwitchTarget(tenantId, scopeId);
2896
+ /**
2897
+ * One kill switch moved on one scope — the ONE body `revokeFromSystem` / `restoreToSystem`
2898
+ * (#1666) and `revokeFromPeer` / `restoreToPeer` (#1706) share, so the two levers cannot
2899
+ * disagree about the record, the audit, the readback or the undo (#2029). The audit row is
2900
+ * written HERE either way — a deployment's host is CP-less, and its `recordAdmin` is a no-op.
2901
+ */
2902
+ const switchSubjectAt = async (actor, kind, node, key, reason, to) => {
2903
+ const { tenantId, scopeId } = node;
2904
+ const { vertical, delegated, move, attestFence } = await switchTarget(kind, tenantId, scopeId);
2905
+ const action = switchActionOf(kind, to);
2906
+ if (kind === 'peer') {
2907
+ // The peer door's gate (#1706): with a delegation, its lifecycle half (a suspended tenant
2908
+ // or scope cannot be switched), the `scopeStub` half being what the delegation replaces.
2909
+ // Without one, a hosted scope is refused `unavailable` rather than switched in the
2910
+ // placeholder namespace, where it would tombstone nothing a peer call reads.
2911
+ if (delegated)
2912
+ await this.validateScopeAccess(tenantId, scopeId);
2913
+ else
2914
+ await this.peerScopeGate(tenantId, scopeId, action);
2915
+ }
2749
2916
  // AUDIT FIRST (#1666 review): the intent row lands before anything moves, and the
2750
2917
  // outcome row after — every attempt, a repeat included. The scope's store and the
2751
2918
  // admin log are separate, so no order makes the pair atomic; this one fails toward
2752
2919
  // "an intent with no recorded outcome" and never toward "a switch that moved with no
2753
2920
  // audit row". A retry after a crash re-audits even though it answers `changed: false`.
2754
2921
  const operationId = ulid();
2755
- const action = to === 'off' ? 'revokeFromSystem' : 'restoreToSystem';
2756
2922
  const target = { tenantId, scopeId, vertical };
2757
- const base = { operationId, moduleId: input.moduleId, schedules: to };
2758
- await this.recordAdmin(actor, action, target, null, { ...base, phase: 'intent', reason: input.reason });
2759
- // The directory's record (#1674), written BEFORE the scope moves, both ways, and undone
2760
- // if the move throws or holds nothing — see `recordSystemSwitchedOn` and (#1823)
2761
- // `recordSystemSwitchedOff` for why that order is the safe one.
2762
- const at = new Date().toISOString();
2763
- const record = { tenantId, scopeId, moduleId: input.moduleId, actor, reason: input.reason, operationId, at };
2923
+ const base = { operationId, ...switchAuditSubject(kind, key, to) };
2924
+ await this.recordAdmin(actor, action, target, null, { ...base, phase: 'intent', reason });
2764
2925
  const errorOf = (err) => (err instanceof Error ? err.message : String(err));
2926
+ // #2045 (Codex r3): the deployment's fence is attested BEFORE anything is recorded or moved. An
2927
+ // ON on a deployment built before it is refused here, with nothing written anywhere but the
2928
+ // audit. An OFF goes on: the kill switch must work on every deployment (see `UnattestedSwitch`).
2929
+ try {
2930
+ if (!(await attestFence()) && to === 'on') {
2931
+ throw substratError('precondition_failed', `${unfencedMessage(scopeId)} Nothing was switched.`);
2932
+ }
2933
+ }
2934
+ catch (err) {
2935
+ await this.recordAdmin(actor, action, target, null, { ...base, phase: 'refused', error: errorOf(err) }).catch(() => undefined);
2936
+ throw err;
2937
+ }
2938
+ // The directory's record (#1674; #2029 for a peer), written BEFORE the scope moves, both
2939
+ // ways — see `recordSwitchedOn` and (#1823) `recordSwitchedOff` for why that order is the
2940
+ // safe one. #2045 (Codex r3): the same write marks the subject owed and clears the scope's
2941
+ // reconcile receipt, so from here on the scope converges to this record whatever happens to
2942
+ // the move. The mark goes only when the scope confirms the move under this call's fence.
2943
+ const at = new Date().toISOString();
2944
+ const record = { kind, tenantId, scopeId, key, actor, reason, operationId, at };
2765
2945
  let prior;
2766
2946
  try {
2767
- prior = to === 'on' ? await this.cp.recordSystemSwitchedOn(record) : await this.cp.recordSystemSwitchedOff(record);
2947
+ prior = to === 'on' ? await this.cp.recordSwitchedOn(record) : await this.cp.recordSwitchedOff(record);
2768
2948
  }
2769
2949
  catch (err) {
2770
2950
  // Nothing has moved: fail the call here, audited. An ON must not switch a scope on
@@ -2773,92 +2953,92 @@ export class CloudflareScopeHost {
2773
2953
  await this.recordAdmin(actor, action, target, null, { ...base, phase: 'failed', error: errorOf(err) }).catch(() => undefined);
2774
2954
  throw err;
2775
2955
  }
2776
- /**
2777
- * The record's undo, after the scope's move threw or held nothing: retried once, and a
2778
- * failure of both is answered rather than swallowed — it lands on the outcome row as
2779
- * `recordError`, and the call says so.
2780
- */
2781
- const undoRecord = async () => {
2782
- const undo = () => this.cp.restoreSystemSwitchRecord(record, prior);
2956
+ // #2045: a newer call on this subject has recorded its position already: this one writes
2957
+ // nothing, here or in the scope, and says so.
2958
+ if (recordWriteSuperseded(prior, record)) {
2959
+ await this.recordAdmin(actor, action, target, null, { ...base, phase: 'refused', superseded: true });
2960
+ throw substratError('conflict', switchSupersededMessage(kind, scopeId, key, to));
2961
+ }
2962
+ let outcome;
2963
+ try {
2783
2964
  try {
2784
- await undo();
2785
- return null;
2965
+ outcome = await move(key, to, at, undefined, operationId);
2786
2966
  }
2787
- catch {
2788
- try {
2789
- await undo();
2790
- return null;
2791
- }
2792
- catch (err) {
2793
- return errorOf(err);
2794
- }
2967
+ catch (first) {
2968
+ if (first instanceof UnattestedSwitch)
2969
+ throw first;
2970
+ // #2045 (Codex r2): one retry under this call's own fence — idempotent, and refused by
2971
+ // the scope if a newer call has moved it meanwhile.
2972
+ outcome = await move(key, to, new Date().toISOString(), undefined, operationId);
2795
2973
  }
2796
- };
2797
- let outcome;
2798
- try {
2799
- outcome = await move(input.moduleId, to, at);
2800
2974
  }
2801
2975
  catch (err) {
2802
- if (err instanceof UnattestedSwitchOff) {
2803
- // The far end switched, and cannot hold it against the tenant-level grant. Refused:
2804
- // ON gives back exactly what this OFF took (nothing, when it changed nothing), and
2805
- // then the record goes back too — neither the scope nor the directory says off. An ON
2806
- // that fails here leaves the scope off, and the readback rule below keeps its record.
2807
- const putBack = err.outcome.changed
2808
- ? await move(input.moduleId, 'on', new Date().toISOString(), true).then(() => true, () => false)
2809
- : true;
2810
- if (putBack) {
2811
- const recordError = await undoRecord();
2812
- await this.recordAdmin(actor, action, target, null, {
2813
- ...base,
2814
- phase: 'refused',
2815
- error: err.message,
2816
- ...(recordError ? { recordError } : {}),
2817
- }).catch(() => undefined);
2818
- throw substratError('precondition_failed', `${err.message} Nothing was switched` +
2819
- (recordError ? `, but its directory record could not be put back (${recordError}).` : '.'));
2820
- }
2821
- }
2822
- // A throw does not say the scope did not move: the move crosses the scope's DO and then
2823
- // the rewind hold, and a throw from the second leaves the scope's switch moved. So the
2824
- // record is undone only when the scope reads back in the OTHER position — an OFF that
2825
- // reads not off, an ON that still reads off — and kept when it reads back as asked.
2826
- // Undoing an ON that landed would put the record back to `off` beside a scope that is
2827
- // on, and the next re-assert would switch the operator's ON back off. A read that fails
2828
- // too counts as off, both ways, so what is left fails toward off: an OFF keeps its
2829
- // record (never a scope switched off with no record — a wipe or restore would have
2830
- // nothing to re-assert, and #1743's refusal nothing to read), and an ON is undone back
2831
- // to `off`, which the next re-assert completes if the scope did move.
2832
- const off = await isOff(input.moduleId).catch(() => true);
2833
- const recordKept = off === (to === 'off');
2834
- const recordError = recordKept ? null : await undoRecord();
2835
- // Best effort: the original error is what the caller must see, and the intent row
2836
- // already says an attempt was made.
2976
+ // The move threw twice, or was answered by a deployment that could not attest it. Either
2977
+ // way it may have landed, and an older call's move may still be in flight behind it, so no
2978
+ // readback can settle the scope and nothing is put back: the record is this call's position
2979
+ // under the newest fence, and its write-ahead mark makes the next re-assert move the scope
2980
+ // to it, under that fence, whichever way it lies now.
2837
2981
  await this.recordAdmin(actor, action, target, null, {
2838
2982
  ...base,
2839
- phase: 'failed',
2983
+ phase: err instanceof UnattestedSwitch ? 'refused' : 'failed',
2840
2984
  error: errorOf(err),
2841
- ...(recordKept ? { recordKept: true } : {}),
2842
- ...(recordError ? { recordError } : {}),
2985
+ recordKept: true,
2986
+ reassertOwed: true,
2843
2987
  }).catch(() => undefined);
2988
+ if (err instanceof UnattestedSwitch)
2989
+ throw substratError('precondition_failed', err.message);
2844
2990
  throw err;
2845
2991
  }
2992
+ // #2045: the scope has applied a newer call on this subject, so this move wrote nothing. The
2993
+ // record is that newer call's too (its write overwrote this one's, mark included), so nothing
2994
+ // is undone and no mark is this call's to clear.
2995
+ if (outcome.superseded) {
2996
+ await this.recordAdmin(actor, action, target, null, { ...base, phase: 'refused', superseded: true });
2997
+ throw substratError('conflict', switchSupersededMessage(kind, scopeId, key, to));
2998
+ }
2846
2999
  // A call that held nothing moved nothing, so its record write is undone: left `off`, it
2847
- // would switch the module off the day it is installed; left `on`, the next reconcile of
2848
- // a wiped scope would leave the module running.
2849
- const recordError = outcome.held ? null : await undoRecord();
3000
+ // would switch the subject off the day it is installed; left `on`, the next reconcile of
3001
+ // a wiped scope would leave it running. Retried once; a failure of both lands on the
3002
+ // outcome row as `recordError`, and the call says so.
3003
+ //
3004
+ // The scope confirmed this call's move (or that there was nothing to move), so its mark goes.
3005
+ // Held nothing, it goes WITH the undo, in one directory transaction: the row put back carries
3006
+ // the prior call's older id, and no later re-assert confirms under a fence older than this
3007
+ // call's mark. An undo that fails twice leaves the record AND the mark this call's, and the
3008
+ // next re-assert confirms both under it. Held, a failed clear is not the call's: the mark only
3009
+ // costs the next re-assert one idempotent move under this same record.
3010
+ //
3011
+ // An unfenced OFF that held something (a deployment built before the fence) confirmed nothing
3012
+ // under its fence, so its mark stays for the first re-assert after the vertical is redeployed.
3013
+ const unfenced = outcome.held && outcome.fenced !== true;
3014
+ let recordError = null;
3015
+ let owedError = null;
3016
+ if (!outcome.held) {
3017
+ const undo = () => this.cp.restoreSwitchRecord(record, prior, true);
3018
+ recordError = await undo().then(() => null, () => undo().then(() => null, errorOf));
3019
+ }
3020
+ else if (!unfenced) {
3021
+ owedError = await this.cp.clearSwitchOwed(kind, tenantId, scopeId, key, operationId).then(() => null, errorOf);
3022
+ }
2850
3023
  await this.recordAdmin(actor, action, target, null, {
2851
3024
  ...base,
2852
3025
  phase: outcome.held ? 'applied' : 'refused',
2853
3026
  changed: outcome.changed,
2854
3027
  permissions: outcome.permissions,
3028
+ ...(unfenced ? { unfenced: true, reassertOwed: true } : {}),
2855
3029
  ...(recordError ? { recordError } : {}),
3030
+ ...(owedError ? { owedError } : {}),
2856
3031
  });
2857
3032
  if (!outcome.held) {
2858
- throw substratError('not_found', `scope ${scopeId} holds no system grant for module '${input.moduleId}' — nothing to switch ${to} ` +
2859
- `(check the module id: it is the module's manifest id, e.g. '@substrat-run/engine-absence')` +
3033
+ throw substratError('not_found', switchNotFoundMessage(kind, scopeId, key, to) +
2860
3034
  (recordError ? `; and its directory record could not be put back (${recordError})` : ''));
2861
3035
  }
3036
+ return { operationId, outcome };
3037
+ };
3038
+ /** #1666: move one module's schedule switch on one scope — see `HostAdmin.revokeFromSystem`. */
3039
+ const switchSystem = async (actor, raw, to) => {
3040
+ const input = systemSwitch.parse(raw);
3041
+ const { operationId, outcome } = await switchSubjectAt(actor, 'system', input.node, input.moduleId, input.reason, to);
2862
3042
  return {
2863
3043
  operationId,
2864
3044
  moduleId: input.moduleId,
@@ -2867,6 +3047,18 @@ export class CloudflareScopeHost {
2867
3047
  permissions: outcome.permissions,
2868
3048
  };
2869
3049
  };
3050
+ /** #1706: move one PEER's kill switch on one scope — see `HostAdmin.revokeFromPeer`. */
3051
+ const switchPeerAt = async (actor, raw, to) => {
3052
+ const input = peerSwitch.parse(raw);
3053
+ const { operationId, outcome } = await switchSubjectAt(actor, 'peer', input.node, input.vertical, input.reason, to);
3054
+ return {
3055
+ operationId,
3056
+ vertical: input.vertical,
3057
+ calls: to,
3058
+ changed: outcome.changed,
3059
+ permissions: outcome.permissions,
3060
+ };
3061
+ };
2870
3062
  /**
2871
3063
  * The status read's admin-log join (#1674): the `intent` row of the `revokeFromSystem`
2872
3064
  * still in force for each OFF module — actor, reason, when. Paired to its `applied` row
@@ -2939,7 +3131,7 @@ export class CloudflareScopeHost {
2939
3131
  delegation
2940
3132
  ? delegation.status({ tenantId, scopeId })
2941
3133
  : this.scopeStub(scopeId).systemGrantsStatus().then((rows) => systemScheduleEntry.array().parse(rows)),
2942
- this.cp.systemSwitchRecordsOf(tenantId, scopeId),
3134
+ this.cp.switchRecordsOf('system', tenantId, scopeId),
2943
3135
  ]);
2944
3136
  const offModules = new Set(states.filter((s) => s.schedules === 'off').map((s) => s.moduleId));
2945
3137
  const explanations = offModules.size > 0
@@ -2958,71 +3150,137 @@ export class CloudflareScopeHost {
2958
3150
  };
2959
3151
  /**
2960
3152
  * #1674: put the directory's OFF positions back into one scope — see
2961
- * `HostAdmin.reassertSystemSwitches`. It moves the switch through `systemSwitchTarget`,
2962
- * so it lands wherever an operator's OFF would have — for a hosted scope, over the
2963
- * `/internal/system-switch` route every deployment since #1666 answers.
3153
+ * `HostAdmin.reassertSystemSwitches`. Every kind (#2029): the recorded-off modules, then the
3154
+ * recorded-off peers. Each moves through `switchTarget`, so it lands wherever an operator's
3155
+ * OFF would have — for a hosted scope, over that kind's `/internal/*` switch route.
2964
3156
  */
2965
3157
  const reassertSystemSwitchesOf = async (actor, node, opts) => {
3158
+ const results = [];
3159
+ for (const kind of SWITCH_KINDS)
3160
+ results.push(...(await reassertKind(actor, kind, node, opts)));
3161
+ return results;
3162
+ };
3163
+ /** One kind's half of `reassertSystemSwitchesOf`. */
3164
+ const reassertKind = async (actor, kind, node, opts) => {
2966
3165
  const { tenantId, scopeId } = node;
2967
- const { vertical, move } = await systemSwitchTarget(tenantId, scopeId);
2968
- const recorded = new Map(await this.cp.systemSwitchRecordsOf(tenantId, scopeId));
3166
+ const { vertical, delegated, move, attestFence } = await switchTarget(kind, tenantId, scopeId);
3167
+ const action = reassertActionOf(kind);
3168
+ const target = { tenantId, scopeId, vertical };
3169
+ const recorded = new Map(await this.cp.switchRecordsOf(kind, tenantId, scopeId));
3170
+ // #2045: every switch call's write-ahead mark, with the newest call that wrote it.
3171
+ const owed = new Map(await this.cp.switchesOwedOf(kind, tenantId, scopeId));
2969
3172
  // #1742 review: moves the deployment made from a stale list (restored ON after the list
2970
3173
  // was read), to undo before the OFF pass below.
2971
- const reverts = staleCarryReverts(recorded, opts?.appliedInUnit);
3174
+ const reverts = staleCarryReverts(kind, recorded, opts?.appliedInUnit);
2972
3175
  // A hosted scope with no delegation configured (Copilot review): this host's own
2973
3176
  // namespace is the module-less placeholder, where the switch would answer `held: false`
2974
3177
  // quietly — a re-assert reported done, and a receipt written, while the deployment
2975
- // serving the scope keeps its schedules running. Refused loudly instead, as the status
2976
- // read is, and only when a re-assert is owed: nothing recorded, nothing to refuse.
2977
- const owed = reverts.length > 0 || [...recorded.values()].includes('off');
2978
- if (owed && !this.cpLess && vertical !== null && !this.systemSwitchDelegation) {
3178
+ // serving the scope keeps its schedules running or its peer admitted. Refused loudly
3179
+ // instead, as the status read is, and only when a re-assert is owed: nothing recorded,
3180
+ // nothing to refuse.
3181
+ const reassertOwed = reverts.length > 0 || owed.size > 0 || [...recorded.values()].includes('off');
3182
+ if (reassertOwed && !this.cpLess && vertical !== null && !delegated) {
2979
3183
  throw substratError('unavailable', `no delegation configured for hosted scope ${scopeId} (vertical '${vertical}') — cannot re-assert ` +
2980
- `its switched-off schedules in the deployment serving it`);
3184
+ `its switched-off ${DOOR_WORDS[kind].switched} in the deployment serving it`);
2981
3185
  }
3186
+ // #2045 (Codex r3): a subject's mark goes only once the scope has confirmed a move under the
3187
+ // record's fence — a fenced answer that was not superseded. A throw anywhere below leaves
3188
+ // every mark not yet confirmed for the next pass.
3189
+ // An unfenced OFF (a deployment built before the fence) confirms nothing, so its mark stays.
3190
+ const confirm = async (key, fence, outcome) => {
3191
+ if (owed.has(key) && !outcome.superseded && outcome.fenced === true) {
3192
+ await this.cp.clearSwitchOwed(kind, tenantId, scopeId, key, fence);
3193
+ }
3194
+ };
3195
+ // #2045 follow-up: on a deployment built before the fence no ON is sent (see
3196
+ // `UnattestedSwitch`). Every OFF below still moves, and the pass refuses at its end, so a
3197
+ // sweep records no receipt for a scope whose owed ON was not sent. Asked only when an ON is due.
3198
+ const onWithheld = [];
3199
+ const withholdOn = async (key) => {
3200
+ if (await attestFence())
3201
+ return false;
3202
+ onWithheld.push(key);
3203
+ return true;
3204
+ };
2982
3205
  const at = new Date().toISOString();
2983
3206
  const reverted = new Set();
2984
- for (const moduleId of reverts) {
3207
+ for (const key of reverts) {
2985
3208
  // Re-read immediately before the move: a staff OFF that completed since the read above
2986
- // (record `off`) must not get a transient ON a due schedule could run in.
2987
- const current = new Map(await this.cp.systemSwitchRecordsOf(tenantId, scopeId));
2988
- if (current.get(moduleId) !== 'on')
3209
+ // (record `off`) must not get a transient ON a due schedule, or a peer call, could use.
3210
+ // #2045: under the fence of the ON the record holds, which is the operator's own call — read
3211
+ // with the position, so it is that ON's fence and no other call's.
3212
+ const current = new Map(await this.cp.switchRecordStatesOf(kind, tenantId, scopeId)).get(key);
3213
+ if (current?.position !== 'on')
2989
3214
  continue;
2990
- const outcome = await move(moduleId, 'on', at);
2991
- reverted.add(moduleId);
3215
+ const fence = current.fence;
3216
+ if (await withholdOn(key))
3217
+ continue;
3218
+ const outcome = await move(key, 'on', at, undefined, fence);
3219
+ reverted.add(key);
2992
3220
  if (outcome.changed) {
2993
- await this.recordAdmin(actor, 'reassertSystemSwitch', { tenantId, scopeId, vertical }, null, {
2994
- operationId: ulid(),
2995
- ...staleCarryRevertRow(moduleId, outcome),
2996
- });
3221
+ await this.recordAdmin(actor, action, target, null, { operationId: ulid(), ...staleCarryRevertRow(kind, key, outcome) });
2997
3222
  }
3223
+ await confirm(key, fence, outcome);
2998
3224
  }
2999
3225
  // Read AFTER the reverts: an OFF that landed meanwhile is switched off below.
3000
- const modules = await this.cp.switchedOffModulesOf(tenantId, scopeId);
3226
+ const { keys, tenantHeld, fences } = await this.recordedOff(kind, tenantId, scopeId, at);
3001
3227
  // #1742: what the deployment already switched off inside its own unit, audited here —
3002
- // the move below answers `changed: false` for it and would write no row.
3003
- // A move the revert above undid is not credited as an in-unit OFF.
3004
- const applied = opts?.appliedInUnit?.filter((a) => !reverted.has(a.moduleId));
3005
- for (const row of inUnitMovesToAudit(modules, applied)) {
3006
- await this.recordAdmin(actor, 'reassertSystemSwitch', { tenantId, scopeId, vertical }, null, {
3007
- operationId: ulid(),
3008
- ...row,
3009
- });
3228
+ // the move below answers `changed: false` for it and would write no row. A move the
3229
+ // revert above undid is not credited as an in-unit OFF.
3230
+ for (const row of inUnitMovesToAudit(kind, keys, opts?.appliedInUnit, reverted)) {
3231
+ await this.recordAdmin(actor, action, target, null, { operationId: ulid(), ...row });
3010
3232
  }
3233
+ // #2045: every move here carries the fence of the call the record holds, so a scope a newer call
3234
+ // has moved since refuses it. A deployment from before the fence takes the OFF unfenced.
3011
3235
  const results = [];
3012
- const tenantHeld = new Set(modules.length ? await this.cp.tenantHeldSystemModules(tenantId, modules, at) : []);
3013
- for (const moduleId of modules) {
3014
- const outcome = await move(moduleId, 'off', at, tenantHeld.has(moduleId));
3236
+ const held = new Set(tenantHeld);
3237
+ for (const key of keys) {
3238
+ const fence = fences.get(key); // `recordedOff` read each key with its own row's fence
3239
+ const outcome = await move(key, 'off', at, held.has(key), fence);
3015
3240
  if (outcome.changed) {
3016
- await this.recordAdmin(actor, 'reassertSystemSwitch', { tenantId, scopeId, vertical }, null, {
3241
+ await this.recordAdmin(actor, action, target, null, {
3017
3242
  operationId: ulid(),
3018
- moduleId,
3019
- schedules: 'off',
3020
- phase: 'applied',
3021
- changed: true,
3022
- permissions: outcome.permissions,
3243
+ ...reassertOffRow(kind, key, outcome.permissions),
3244
+ ...(outcome.fenced === true ? {} : { unfenced: true }),
3023
3245
  });
3024
3246
  }
3025
- results.push({ moduleId, held: outcome.held, changed: outcome.changed });
3247
+ results.push(reassertEntry(kind, key, outcome));
3248
+ await confirm(key, fence, outcome);
3249
+ }
3250
+ // A subject marked owed converges to its record in either direction: OFF above, and here a
3251
+ // record of ON — sent whatever the scope reads now, because a scope that reads ON may still
3252
+ // be behind the record's fence, with an older call's OFF in flight that only the fence can
3253
+ // refuse (Codex r3, finding 1). Idempotent: an ON already in place changes nothing and moves
3254
+ // the fence. Every other record of ON keeps #1674's rule and turns nothing on.
3255
+ const settled = new Set([...keys, ...reverted]);
3256
+ const pending = [...owed.keys()].filter((key) => !settled.has(key));
3257
+ if (pending.length) {
3258
+ // Position and fence in one read: an ON is sent only under the fence of the ON it read.
3259
+ const now = new Map(await this.cp.switchRecordStatesOf(kind, tenantId, scopeId));
3260
+ for (const key of pending) {
3261
+ const row = now.get(key);
3262
+ if (row?.position !== 'on') {
3263
+ // No record (a call that held nothing, whose undo removed its row), or one that turned
3264
+ // OFF since the read above, which the next pass moves: nothing to send on this one. A
3265
+ // missing row's mark goes under its own id, so a newer call's mark stays.
3266
+ if (row === undefined)
3267
+ await this.cp.clearSwitchOwed(kind, tenantId, scopeId, key, owed.get(key));
3268
+ continue;
3269
+ }
3270
+ const fence = row.fence;
3271
+ if (await withholdOn(key))
3272
+ continue;
3273
+ const outcome = await move(key, 'on', at, undefined, fence);
3274
+ if (outcome.changed) {
3275
+ await this.recordAdmin(actor, action, target, null, { operationId: ulid(), ...reassertOnRow(kind, key, outcome.permissions) });
3276
+ results.push(reassertEntry(kind, key, outcome));
3277
+ }
3278
+ await confirm(key, fence, outcome);
3279
+ }
3280
+ }
3281
+ if (onWithheld.length) {
3282
+ throw substratError('precondition_failed', `${unfencedMessage(scopeId)} Its recorded OFF positions were applied; the owed ON of ` +
3283
+ `${onWithheld.map((k) => `'${k}'`).join(', ')} was not sent.`);
3026
3284
  }
3027
3285
  return results;
3028
3286
  };
@@ -3109,18 +3367,21 @@ export class CloudflareScopeHost {
3109
3367
  unassignRole: async (actor, assignment) => {
3110
3368
  // Tombstone (K-21) — the checker skips revoked rows. A no-op returns false so
3111
3369
  // a repeat unassign stays silent (no second audit row, no needless fan-out).
3112
- const subject = `principal:${assignment.principalId}`;
3113
3370
  const relation = `role:${assignment.roleKey}`;
3114
- const now = new Date().toISOString();
3115
- const changed = assignment.node.scopeId
3116
- ? await this.scopeStub(assignment.node.scopeId).revokeTuple(subject, relation, `scope:${assignment.node.scopeId}`, now)
3117
- : await this.cp.revokeTenantTuple(assignment.node.tenantId, subject, relation, `tenant:${assignment.node.tenantId}`, now);
3371
+ const { tenantId, scopeId } = assignment.node;
3372
+ if (!scopeId) {
3373
+ // One ControlPlaneDO method (#1184): the revoke, the removal fence (raised even when
3374
+ // nothing was held) and the audit row, so no add can land between any two of them.
3375
+ const changed = await this.cp.revokeAndFence(tenantId, assignment.principalId, relation, `tenant:${tenantId}`, this.adminEntry(actor, 'unassignRole', { tenantId, scopeId: null }, assignment, null));
3376
+ // A tenant-level revoke changes the projected set — the tombstone must reach scopes.
3377
+ if (changed)
3378
+ await this.fanOut(tenantId);
3379
+ return;
3380
+ }
3381
+ const changed = await this.scopeStub(scopeId).revokeTuple(`principal:${assignment.principalId}`, relation, `scope:${scopeId}`, new Date().toISOString());
3118
3382
  if (!changed)
3119
3383
  return;
3120
- await this.recordAdmin(actor, 'unassignRole', { tenantId: assignment.node.tenantId, scopeId: assignment.node.scopeId }, assignment, null);
3121
- // A tenant-level revoke changes the projected set — the tombstone must reach scopes.
3122
- if (!assignment.node.scopeId)
3123
- await this.fanOut(assignment.node.tenantId);
3384
+ await this.recordAdmin(actor, 'unassignRole', { tenantId, scopeId }, assignment, null);
3124
3385
  },
3125
3386
  grant: async (actor, raw) => {
3126
3387
  // Parsed like its `grantToConnection`/`grantToSystem` siblings, not taken on
@@ -3242,11 +3503,17 @@ export class CloudflareScopeHost {
3242
3503
  },
3243
3504
  tenantHeldSystemModules: async (actor, tenantId, moduleIds) => {
3244
3505
  const held = moduleIds.length
3245
- ? await this.cp.tenantHeldSystemModules(tenantId, moduleIds, new Date().toISOString())
3506
+ ? await this.cp.tenantHeldOf('system', tenantId, moduleIds, new Date().toISOString())
3246
3507
  : [];
3247
3508
  await this.recordAccess(actor, 'tenantHeldSystemModules', { tenantId }, { moduleIds: [...moduleIds] }, held.length);
3248
3509
  return held;
3249
3510
  },
3511
+ peerSwitchCarry: async (actor, node) => {
3512
+ const { keys: switchedOffPeers, tenantHeld: tenantHeldPeers, fences: all, } = await this.recordedOff('peer', node.tenantId, node.scopeId, new Date().toISOString());
3513
+ const fences = Object.fromEntries(switchedOffPeers.map((v) => [v, all.get(v)]));
3514
+ await this.recordAccess(actor, 'peerSwitchCarry', node, null, switchedOffPeers.length);
3515
+ return { switchedOffPeers, tenantHeldPeers, fences };
3516
+ },
3250
3517
  reassertSystemSwitches: reassertSystemSwitchesOf,
3251
3518
  // #1672 — the platform's two capability verbs. Audited AFTER the write, on both, and
3252
3519
  // the failure each leaves is the safe one: a mint whose audit row did not land never
@@ -3312,7 +3579,7 @@ export class CloudflareScopeHost {
3312
3579
  // another tenant's traffic. Exception: the holder is ARCHIVED or REAPED (a
3313
3580
  // deleted app, storage since wiped) — it has released the name, so the rebind
3314
3581
  // reclaims it.
3315
- throw new Error(`hostname '${parsed.hostname}' is already bound to another scope`);
3582
+ throw substratError('conflict', `hostname '${parsed.hostname}' is already bound to another scope`);
3316
3583
  }
3317
3584
  // Exactly one canonical per (scope, surface).
3318
3585
  if (parsed.canonical)
@@ -3333,7 +3600,7 @@ export class CloudflareScopeHost {
3333
3600
  const hostname = raw.toLowerCase(); // DNS is case-insensitive; the map is normalized
3334
3601
  const row = await this.cp.readHostname(hostname);
3335
3602
  if (!row)
3336
- throw new Error(`unknown hostname '${hostname}'`);
3603
+ throw substratError('not_found', `unknown hostname '${hostname}'`);
3337
3604
  if (row.status === status)
3338
3605
  return; // idempotent, unaudited
3339
3606
  await this.cp.setHostnameStatus(hostname, status, note ?? null);
@@ -3343,7 +3610,7 @@ export class CloudflareScopeHost {
3343
3610
  const hostname = raw.toLowerCase(); // DNS is case-insensitive; the map is normalized
3344
3611
  const row = await this.cp.readHostname(hostname);
3345
3612
  if (!row)
3346
- throw new Error(`unknown hostname '${hostname}'`);
3613
+ throw substratError('not_found', `unknown hostname '${hostname}'`);
3347
3614
  // A poll that finds nothing changed (same status, same records, id already set)
3348
3615
  // is not an event — skip the write and the audit entry, so the reconcile sweep
3349
3616
  // does not flood the admin log with no-op rows every interval.
@@ -3429,7 +3696,7 @@ export class CloudflareScopeHost {
3429
3696
  if (existing.owner_tenant !== parsed.ownerTenant) {
3430
3697
  throw substratError('conflict', `vertical '${parsed.slug}' is owned by ${existing.owner_tenant ?? 'the platform'}, not ${parsed.ownerTenant ?? 'the platform'}`);
3431
3698
  }
3432
- throw new Error(`vertical '${parsed.slug}' is already registered as ${existing.source}`);
3699
+ throw substratError('conflict', `vertical '${parsed.slug}' is already registered as ${existing.source}`);
3433
3700
  }
3434
3701
  await this.cp.insertVertical(parsed.slug, parsed.name, parsed.source, parsed.ownerTenant, envSpecJson, installSpecJson, parsed.listed ? 1 : 0, new Date().toISOString());
3435
3702
  await this.recordAdmin(actor, 'registerVertical', { tenantId: null }, null, parsed);
@@ -3610,11 +3877,11 @@ export class CloudflareScopeHost {
3610
3877
  // nothing to diff against — the gate is about change, not existence.
3611
3878
  if (outgoing) {
3612
3879
  if (outgoing.permission_digest !== incoming.permission_digest && !ack.permissionChange) {
3613
- throw new Error(`promotion changes the permission surface (${outgoing.permission_digest} → ` +
3880
+ throw substratError('conflict', `promotion changes the permission surface (${outgoing.permission_digest} → ` +
3614
3881
  `${incoming.permission_digest}) — acknowledge it explicitly to promote`);
3615
3882
  }
3616
3883
  if (outgoing.migration_digest !== incoming.migration_digest && !ack.migrationChange) {
3617
- throw new Error(`promotion changes migrations (${outgoing.migration_digest} → ` +
3884
+ throw substratError('conflict', `promotion changes migrations (${outgoing.migration_digest} → ` +
3618
3885
  `${incoming.migration_digest}) — acknowledge it explicitly to promote`);
3619
3886
  }
3620
3887
  // #1705 PR 3: an export an installed consumer imports, dropped or re-versioned.
@@ -3874,7 +4141,7 @@ export class CloudflareScopeHost {
3874
4141
  },
3875
4142
  createOrg: async (actor, input) => {
3876
4143
  const parsed = createOrgInput.parse(input);
3877
- const created = await this.cp.createOrg(parsed.id, parsed.tenantId, parsed.slug, parsed.name, new Date().toISOString());
4144
+ const created = await this.directory('createOrg', parsed.id, parsed.tenantId, parsed.slug, parsed.name, new Date().toISOString());
3878
4145
  if (!created)
3879
4146
  return; // idempotent, and a no-op is not audited
3880
4147
  await this.recordAdmin(actor, 'createOrg', { tenantId: parsed.tenantId }, null, parsed);
@@ -3895,15 +4162,27 @@ export class CloudflareScopeHost {
3895
4162
  await this.recordAdmin(actor, 'addMember', { tenantId }, null, { principal, orgId });
3896
4163
  await this.fanOut(tenantId); // membership is a tenant-level tuple
3897
4164
  },
4165
+ applyMembership: async (actor, change) => {
4166
+ // The row is minted here, where attribution and `causedBy` live, and written by the
4167
+ // ControlPlaneDO in the same synchronous method as the fence, the bound and the tuple
4168
+ // (#1184): one DO unit, so nothing lands between the check and the write.
4169
+ const { tenantId, principal, roleKey, op } = change;
4170
+ const assignment = { principalId: principal, roleKey, node: { tenantId, scopeId: null } };
4171
+ const result = await this.cp.applyMembership(change, op === 'add'
4172
+ ? this.adminEntry(actor, 'assignRole', { tenantId, scopeId: null }, null, assignment)
4173
+ : this.adminEntry(actor, 'unassignRole', { tenantId, scopeId: null }, assignment, null));
4174
+ // The tenant-level tuple, or its tombstone, reaches the projections.
4175
+ if (result.applied && (op === 'add' || result.changed))
4176
+ await this.fanOut(tenantId);
4177
+ return result;
4178
+ },
3898
4179
  removeMember: async (actor, tenantId, principal, orgId) => {
3899
4180
  await requireOrg(tenantId, orgId);
3900
- // Tombstone (K-21), never DELETE. The DO reports whether anything changed
3901
- // so a repeat revoke stays a silent no-op rather than a second audit row.
3902
- const changed = await this.cp.revokeMember(tenantId, `principal:${principal}`, `org:${orgId}`, new Date().toISOString());
3903
- if (!changed)
3904
- return;
3905
- await this.recordAdmin(actor, 'removeMember', { tenantId }, { principal, orgId }, null);
3906
- await this.fanOut(tenantId); // the tombstone must reach the projections
4181
+ // Tombstone (K-21), never DELETE, with the removal fence in the same DO method
4182
+ // (#1184). A repeat revoke stays a silent no-op rather than a second audit row.
4183
+ const changed = await this.cp.revokeAndFence(tenantId, principal, 'member', `org:${orgId}`, this.adminEntry(actor, 'removeMember', { tenantId }, { principal, orgId }, null));
4184
+ if (changed)
4185
+ await this.fanOut(tenantId); // the tombstone must reach the projections
3907
4186
  },
3908
4187
  listMembers: async (actor, tenantId, orgId, options) => {
3909
4188
  await requireOrg(tenantId, orgId);
@@ -3917,20 +4196,20 @@ export class CloudflareScopeHost {
3917
4196
  },
3918
4197
  createTenant: async (actor, input) => {
3919
4198
  const parsed = createTenantInput.parse(input);
3920
- const created = await this.cp.createTenant(parsed.id, parsed.slug, parsed.name, new Date().toISOString(), parsed.provisionedByTenant ?? null);
4199
+ const created = await this.directory('createTenant', parsed.id, parsed.slug, parsed.name, new Date().toISOString(), parsed.provisionedByTenant ?? null);
3921
4200
  // Idempotent: re-creating an existing tenant is a no-op, not audited.
3922
4201
  if (!created)
3923
4202
  return;
3924
4203
  await this.recordAdmin(actor, 'createTenant', { tenantId: parsed.id }, null, created);
3925
4204
  },
3926
4205
  setTenantStatus: async (actor, tenantId, status) => {
3927
- const before = await this.cp.setTenantStatus(tenantId, status);
4206
+ const before = await this.directory('setTenantStatus', tenantId, status);
3928
4207
  await this.recordAdmin(actor, 'setTenantStatus', { tenantId }, { status: before }, { status });
3929
4208
  // #1713: every hosted scope under the tenant holds or resumes its work by this.
3930
4209
  await this.deliverLifecycles(actor, { tenantId });
3931
4210
  },
3932
4211
  setTenantName: async (actor, tenantId, name) => {
3933
- const before = await this.cp.setTenantName(tenantId, name);
4212
+ const before = await this.directory('setTenantName', tenantId, name);
3934
4213
  if (before === name)
3935
4214
  return; // no-op is not audited — nothing changed
3936
4215
  await this.recordAdmin(actor, 'setTenantName', { tenantId }, { name: before }, { name });
@@ -3940,7 +4219,7 @@ export class CloudflareScopeHost {
3940
4219
  // first (archive-if-needed → reapScope in the vertical deployment); the DO clears
3941
4220
  // the tenant's PII/config rows and flips the row to a `reaped` tombstone, keeping
3942
4221
  // the row + admin log. Only a `deleting` tenant may be reaped (checked in the DO).
3943
- const before = await this.cp.reapTenant(tenantId);
4222
+ const before = await this.directory('reapTenant', tenantId);
3944
4223
  await this.recordAdmin(actor, 'reapTenant', { tenantId }, { status: before }, { status: 'reaped' });
3945
4224
  },
3946
4225
  listTenants: async (actor, page) => {
@@ -4024,7 +4303,7 @@ export class CloudflareScopeHost {
4024
4303
  // Not `peerScopeGate`: a read must not migrate, and the phase makes this call once
4025
4304
  // per edge per pass, where an extra round trip is a fleet-wide cost.
4026
4305
  this.assertServedHere(await this.scopeRecordForRead(tenantId, scopeId), scopeId, 'readExportedEvents');
4027
- const batch = exportedBatch.parse(await this.scopeStub(scopeId).exportedEventsRead(input, tenantId, scopeId));
4306
+ const batch = await this.readExportsThroughDoor(input, tenantId, scopeId);
4028
4307
  await this.recordAccess(actor, 'readExportedEvents', { tenantId, scopeId }, { consumer: input.consumer, after: input.after, limit: input.limit }, batch.events.length);
4029
4308
  return batch;
4030
4309
  },
@@ -4204,12 +4483,18 @@ export class CloudflareScopeHost {
4204
4483
  await this.recordAccess(actor, 'lifecycleFlow', { tenantId, scopeId }, delegatedReadParams.lifecycleFlow(input), flow.observation.events);
4205
4484
  return flow;
4206
4485
  },
4486
+ operationSeries: async (actor, tenantId, scopeId, input) => {
4487
+ await this.scopeRecordForRead(tenantId, scopeId);
4488
+ const series = await this.scopeStub(scopeId).operationSeries(input);
4489
+ await this.recordAccess(actor, 'operationSeries', { tenantId, scopeId }, delegatedReadParams.operationSeries(input), operationSeriesCount(series));
4490
+ return series;
4491
+ },
4207
4492
  readScopeTable: async (actor, tenantId, scopeId, input) => {
4208
4493
  await this.scopeRecordForRead(tenantId, scopeId);
4209
4494
  // Checked here as well as in the ScopeDO, so the refusal keeps its code across the hop.
4210
4495
  assertRowLimit('limit', input.limit);
4211
4496
  assertRowOffset('offset', input.offset);
4212
- const page = await this.scopeStub(scopeId).introspectTable(input.table, input.limit, input.offset);
4497
+ const page = unwrapReply(await this.scopeStub(scopeId).introspectTableReply(input.table, input.limit, input.offset));
4213
4498
  await this.recordAccess(actor, 'readScopeTable', { tenantId, scopeId }, { table: input.table, limit: page.limit, offset: page.offset }, page.rows.length);
4214
4499
  return page;
4215
4500
  },
@@ -4250,7 +4535,7 @@ export class CloudflareScopeHost {
4250
4535
  },
4251
4536
  queryScope: async (actor, tenantId, scopeId, input) => {
4252
4537
  await this.scopeRecordForRead(tenantId, scopeId);
4253
- const result = await this.scopeStub(scopeId).introspectQuery(input.sql);
4538
+ const result = unwrapReply(await this.scopeStub(scopeId).introspectQueryReply(input.sql));
4254
4539
  // The statement is the logged argument: the access log is the evidence trail,
4255
4540
  // and for a console read the SQL is the whole story.
4256
4541
  await this.recordAccess(actor, 'queryScope', { tenantId, scopeId }, { sql: input.sql }, result.rows.length);
@@ -4343,7 +4628,7 @@ export class CloudflareScopeHost {
4343
4628
  if (!rec)
4344
4629
  throw substratError('not_found', `unknown scope ${scopeId} in tenant ${tenantId}`);
4345
4630
  if (rec.status !== 'archived') {
4346
- throw new Error(`scope ${scopeId} is ${rec.status}, not archived — only an archived scope may be reaped`);
4631
+ throw substratError('conflict', `scope ${scopeId} is ${rec.status}, not archived — only an archived scope may be reaped`);
4347
4632
  }
4348
4633
  // A serving scope always holds ≥1 bound hostname; a truly-dead one has been
4349
4634
  // unbound. The dashboard delete path unbinds AT archive, but a bare console
@@ -4355,7 +4640,7 @@ export class CloudflareScopeHost {
4355
4640
  // being released anyway; the interactive per-scope reap never sets it.
4356
4641
  const bound = opts?.force ? [] : await this.cp.listHostnames({ scopeId, limit: 1 });
4357
4642
  if (bound.length > 0) {
4358
- throw new Error(`scope ${scopeId} still resolves hostname '${bound[0].hostname}' — ` +
4643
+ throw substratError('conflict', `scope ${scopeId} still resolves hostname '${bound[0].hostname}' — ` +
4359
4644
  `unbind it before reaping (reap wipes storage and cannot be undone)`);
4360
4645
  }
4361
4646
  await this.scopeStub(scopeId).destroyStorage();
@@ -4553,7 +4838,7 @@ export class CloudflareScopeHost {
4553
4838
  },
4554
4839
  registerIdentityPool: async (actor, input) => {
4555
4840
  const parsed = identityPool.parse(input);
4556
- const created = await this.cp.registerIdentityPool(parsed.provider, parsed.topology, parsed.tenantId, new Date().toISOString());
4841
+ const created = await this.directory('registerIdentityPool', parsed.provider, parsed.topology, parsed.tenantId, new Date().toISOString());
4557
4842
  if (!created)
4558
4843
  return; // identical registration is idempotent, unaudited
4559
4844
  // Null tenant for a central pool: it belongs to no single tenant, which is
@@ -4570,9 +4855,9 @@ export class CloudflareScopeHost {
4570
4855
  listIdentityTenants: async (actor, provider, externalId) => {
4571
4856
  const r = await this.cp.readPool(provider);
4572
4857
  if (!r)
4573
- throw new Error(`identity pool '${provider}' is not registered`);
4858
+ throw substratError('not_found', `identity pool '${provider}' is not registered`);
4574
4859
  if (r.topology !== 'central') {
4575
- throw new Error(`identity pool '${provider}' is tenant-bound — enumerating tenants is only ` +
4860
+ throw substratError('forbidden', `identity pool '${provider}' is tenant-bound — enumerating tenants is only ` +
4576
4861
  `meaningful on a central pool, where the same externalId is the same person`);
4577
4862
  }
4578
4863
  const tenants = (await this.cp.identityTenants(provider, externalId));
@@ -4589,9 +4874,9 @@ export class CloudflareScopeHost {
4589
4874
  at: new Date().toISOString(),
4590
4875
  });
4591
4876
  if (topology === null)
4592
- throw new Error(`identity pool '${provider}' is not registered`);
4877
+ throw substratError('not_found', `identity pool '${provider}' is not registered`);
4593
4878
  if (topology !== 'central') {
4594
- throw new Error(`identity pool '${provider}' is tenant-bound — enumerating tenants is only ` +
4879
+ throw substratError('forbidden', `identity pool '${provider}' is tenant-bound — enumerating tenants is only ` +
4595
4880
  `meaningful on a central pool, where the same externalId is the same person`);
4596
4881
  }
4597
4882
  return memberships.map((m) => identityMembership.parse(m));
@@ -4744,14 +5029,14 @@ export class CloudflareScopeHost {
4744
5029
  const parsed = identityLink.parse(input);
4745
5030
  const pool = await this.cp.readPool(parsed.provider);
4746
5031
  if (!pool) {
4747
- throw new Error(`identity pool '${parsed.provider}' is not registered — a pool must declare ` +
5032
+ throw substratError('conflict', `identity pool '${parsed.provider}' is not registered — a pool must declare ` +
4748
5033
  `its topology before it may link (central vs tenant-bound decides whether ` +
4749
5034
  `the same externalId in two tenants is one person or two)`);
4750
5035
  }
4751
5036
  if (pool.topology === 'tenant-bound' && pool.tenant_id !== parsed.tenantId) {
4752
- throw new Error(`identity pool '${parsed.provider}' is bound to tenant ${pool.tenant_id} and cannot link into ${parsed.tenantId}`);
5037
+ throw substratError('conflict', `identity pool '${parsed.provider}' is bound to tenant ${pool.tenant_id} and cannot link into ${parsed.tenantId}`);
4753
5038
  }
4754
- const changed = await this.cp.linkIdentity(parsed.provider, parsed.externalId, parsed.principal, parsed.tenantId, parsed.scopeId ?? null, new Date().toISOString());
5039
+ const changed = await this.directory('linkIdentity', parsed.provider, parsed.externalId, parsed.principal, parsed.tenantId, parsed.scopeId ?? null, new Date().toISOString());
4755
5040
  // Idempotent: an identity already bound is a no-op, not audited.
4756
5041
  if (!changed)
4757
5042
  return;
@@ -5013,6 +5298,10 @@ export class CloudflareScopeHost {
5013
5298
  };
5014
5299
  }
5015
5300
  // -- helpers --------------------------------------------------------------
5301
+ /** A directory write whose refusal must keep its code across the DO hop (#113). */
5302
+ async directory(method, ...args) {
5303
+ return unwrapReply(await this.cp.reply(method, args));
5304
+ }
5016
5305
  /**
5017
5306
  * The getScope gate (control-plane.md §4.1/§4.2) — the pair check, then the tenant's and
5018
5307
  * the scope's lifecycle — thrown on THIS side of the RPC (#1718). The ControlPlaneDO
@@ -5032,8 +5321,11 @@ export class CloudflareScopeHost {
5032
5321
  */
5033
5322
  async validateScopeAccess(tenantId, scopeId) {
5034
5323
  const refusal = await this.cp.scopeAccessRefusal(tenantId, scopeId);
5035
- if (refusal)
5036
- throw refusal.code ? substratError(refusal.code, refusal.message) : new Error(refusal.message);
5324
+ if (refusal) {
5325
+ throw refusal.code
5326
+ ? substratError(refusal.code, refusal.message, refusal.reason ? { reason: refusal.reason } : undefined)
5327
+ : new Error(refusal.message);
5328
+ }
5037
5329
  if (!this.cpLess)
5038
5330
  return null;
5039
5331
  const { verdict, lifecycle } = await this.scopeStub(scopeId).admission(tenantId);
@@ -5065,7 +5357,7 @@ export class CloudflareScopeHost {
5065
5357
  async assertLive(tenantId, scopeId) {
5066
5358
  const refusal = lifecycleRefusal(await this.validateScopeAccess(tenantId, scopeId), { tenantId, scopeId });
5067
5359
  if (refusal)
5068
- throw new Error(refusal);
5360
+ throw substratError('conflict', refusal, { reason: SCOPE_GATE_REASONS.notActive });
5069
5361
  }
5070
5362
  /**
5071
5363
  * Deliver the directory's lifecycle to the deployments serving the matching scopes (#1713):
@@ -5262,7 +5554,7 @@ export class CloudflareScopeHost {
5262
5554
  if (!row)
5263
5555
  throw unknownScopeForTenant(tenantId, scopeId);
5264
5556
  if (row.status === 'reaped') {
5265
- throw new Error(`scope ${scopeId} is reaped — its storage is gone and cannot be read`);
5557
+ throw substratError('conflict', `scope ${scopeId} is reaped — its storage is gone and cannot be read`);
5266
5558
  }
5267
5559
  return row;
5268
5560
  }
@@ -5295,7 +5587,14 @@ export class CloudflareScopeHost {
5295
5587
  });
5296
5588
  }
5297
5589
  async recordAdmin(actor, action, target, before, after) {
5298
- await this.cp.recordAdmin({
5590
+ await this.cp.recordAdmin(this.adminEntry(actor, action, target, before, after));
5591
+ }
5592
+ /**
5593
+ * One admin-log row, minted here where attribution and `causedBy` live — for `recordAdmin`,
5594
+ * and for a ControlPlaneDO method that writes its row in the same unit as its effect (#1184).
5595
+ */
5596
+ adminEntry(actor, action, target, before, after) {
5597
+ return {
5299
5598
  id: ulid(),
5300
5599
  actor,
5301
5600
  action,
@@ -5307,7 +5606,7 @@ export class CloudflareScopeHost {
5307
5606
  before: before ?? null,
5308
5607
  after: after ?? null,
5309
5608
  at: new Date().toISOString(),
5310
- });
5609
+ };
5311
5610
  }
5312
5611
  /**
5313
5612
  * The scope's own answer to "what may this connection do here" (#726 gap 1).
@@ -5432,6 +5731,11 @@ export class CloudflareScopeHost {
5432
5731
  // rewind CLAIMS them on `SWITCH_HOLDS_NAME`, which the rewind cannot reach. A module is held
5433
5732
  // while any claim on it exists, and `runDueSchedules` skips a held module.
5434
5733
  //
5734
+ // #2029: the same for a PEER switched off (`vertical:<slug>`): the rewind claims the scope's
5735
+ // off peers beside its off modules, each under its hold key (`holdKeyOf`), and the peer door
5736
+ // refuses a held peer as the system door refuses a held module. Everything below holds for
5737
+ // both kinds; "module" reads as "subject".
5738
+ //
5435
5739
  // A claim is released only by a switch move KNOWN to survive its rewind (`switchInScope`): a
5436
5740
  // move on the restored storage, after the rewind armed. A move during the settle, or one the
5437
5741
  // doomed instance applied, lands in storage the restart discards, so it releases nothing. A
@@ -5450,7 +5754,7 @@ export class CloudflareScopeHost {
5450
5754
  * reopens only the gap this closes, for a rewind that lands during the outage, and the next
5451
5755
  * reconcile still switches that module off. The error is answered, for the pass to report.
5452
5756
  */
5453
- async switchHeld(scopeId, moduleId) {
5757
+ async switchHeld(scopeId, holdKey) {
5454
5758
  const at = Date.now();
5455
5759
  let error;
5456
5760
  if (!this.holdSnapshot || at - this.holdSnapshot.at > SWITCH_HOLD_SNAPSHOT_MS) {
@@ -5473,8 +5777,11 @@ export class CloudflareScopeHost {
5473
5777
  }
5474
5778
  }
5475
5779
  }
5476
- const held = this.holdSnapshot?.held.has(`${scopeId} ${moduleId}`) ?? false;
5477
- return { held, ...(error ? { error } : {}) };
5780
+ const snapshot = this.holdSnapshot;
5781
+ const held = snapshot?.held.has(`${scopeId} ${holdKey}`) ?? false;
5782
+ // #2029: a good read naming no row for this scope at all, for `doorGate`'s instance cache.
5783
+ const scopeClear = !error && snapshot?.good === true && !snapshot.scopes.has(scopeId);
5784
+ return { held, ...(error ? { error } : {}), ...(scopeClear ? { scopeClear: true } : {}) };
5478
5785
  }
5479
5786
  /** One read of the whole hold into the snapshot, stamped with when it went out. Never throws. */
5480
5787
  async refreshHolds(at) {
@@ -5482,7 +5789,12 @@ export class CloudflareScopeHost {
5482
5789
  const rows = await this.switchHoldsStub().switchHoldsAll();
5483
5790
  // A slow read that lands after a newer one never replaces it.
5484
5791
  if (!this.holdSnapshot || at >= this.holdSnapshot.at) {
5485
- this.holdSnapshot = { at, held: new Set(rows.map((r) => `${r.scopeId} ${r.moduleId}`)) };
5792
+ this.holdSnapshot = {
5793
+ at,
5794
+ held: new Set(rows.map((r) => `${r.scopeId} ${r.moduleId}`)),
5795
+ scopes: new Set(rows.map((r) => r.scopeId)),
5796
+ good: true,
5797
+ };
5486
5798
  }
5487
5799
  return {};
5488
5800
  }
@@ -5493,8 +5805,10 @@ export class CloudflareScopeHost {
5493
5805
  // than hammered per scope. For that long, other scopes reuse this possibly pre-hold set
5494
5806
  // silently, and only the consult that sent this read reports the error: the declared
5495
5807
  // fail-open, attributed once.
5496
- if (!this.holdSnapshot || at >= this.holdSnapshot.at)
5497
- this.holdSnapshot = { at, held: known ?? new Set() };
5808
+ if (!this.holdSnapshot || at >= this.holdSnapshot.at) {
5809
+ const scopes = new Set([...(known ?? [])].map((k) => k.slice(0, k.indexOf(' '))));
5810
+ this.holdSnapshot = { at, held: known ?? new Set(), scopes, good: false };
5811
+ }
5498
5812
  return {
5499
5813
  error: known
5500
5814
  ? `switch hold unreadable (${why}); the ${known.size} hold(s) from this pass's last good read ` +
@@ -5541,7 +5855,11 @@ export class CloudflareScopeHost {
5541
5855
  * its claims before the OFF's join lands. The joined row then holds the module the operator just
5542
5856
  * turned ON, until the next surviving move of that switch releases it (another ON does).
5543
5857
  */
5544
- async switchInScope(scopeId, moduleId, to, at, tenantHeld = false) {
5858
+ async switchInScope(kind, scopeId, key, to, at, tenantHeld = false,
5859
+ /** #2045: the switch call's fence; see `SWITCH_FENCES_DDL`. */
5860
+ fence) {
5861
+ // #2029: a peer's claims live beside the modules', under its own hold key.
5862
+ const holdKey = holdKeyOf(kind, key);
5545
5863
  let scopeClaims;
5546
5864
  let readError;
5547
5865
  try {
@@ -5550,23 +5868,30 @@ export class CloudflareScopeHost {
5550
5868
  catch (err) {
5551
5869
  readError = err;
5552
5870
  }
5553
- const { instance, ...outcome } = await this.scopeStub(scopeId).switchSystemSchedules(moduleId, scopeId, to, at, tenantHeld);
5871
+ const stub = this.scopeStub(scopeId);
5872
+ const { instance, ...outcome } = await (kind === 'system'
5873
+ ? stub.switchSystemSchedules(key, scopeId, to, at, tenantHeld, fence)
5874
+ : stub.switchPeer(key, scopeId, to, at, tenantHeld, fence));
5554
5875
  if (!scopeClaims)
5555
5876
  throw readError;
5877
+ // #2045: a move the fence refused wrote nothing, so it is known to survive nothing and to
5878
+ // release nothing; the newer call that superseded it did whatever the hold needed.
5879
+ if (outcome.superseded)
5880
+ return outcome;
5556
5881
  const now = Date.now();
5557
5882
  const pendingLive = (c) => c.state === 'pending' && now - Date.parse(c.heldAt) <= SWITCH_HOLD_PENDING_MAX_MS;
5558
- const before = scopeClaims.filter((c) => c.moduleId === moduleId);
5883
+ const before = scopeClaims.filter((c) => c.moduleId === holdKey);
5559
5884
  if (to === 'on') {
5560
5885
  // Every ON, claims or not: its tombstone is what keeps a rewind's stale read from claiming
5561
5886
  // the module after this (#1839).
5562
5887
  this.holdSnapshot = null;
5563
- await this.switchHoldsStub().switchHoldOn(scopeId, moduleId, before.map((c) => c.claimId));
5888
+ await this.switchHoldsStub().switchHoldOn(scopeId, holdKey, before.map((c) => c.claimId));
5564
5889
  }
5565
5890
  else if (outcome.held && before.length > 0) {
5566
5891
  const survived = before.filter((c) => !pendingLive(c) && c.doomed !== instance);
5567
5892
  if (survived.length > 0) {
5568
5893
  this.holdSnapshot = null;
5569
- await this.switchHoldsStub().switchHoldRelease(scopeId, moduleId, survived.map((c) => c.claimId));
5894
+ await this.switchHoldsStub().switchHoldRelease(scopeId, holdKey, survived.map((c) => c.claimId));
5570
5895
  }
5571
5896
  }
5572
5897
  if (to === 'off' && outcome.changed) {
@@ -5577,7 +5902,7 @@ export class CloudflareScopeHost {
5577
5902
  const joins = new Set([...scopeClaims, ...after].filter((c) => pendingLive(c) || c.doomed === instance).map((c) => c.claimId));
5578
5903
  if (joins.size > 0) {
5579
5904
  this.holdSnapshot = null;
5580
- await this.switchHoldsStub().switchHoldJoin(scopeId, moduleId, [...joins]);
5905
+ await this.switchHoldsStub().switchHoldJoin(scopeId, holdKey, [...joins]);
5581
5906
  }
5582
5907
  }
5583
5908
  return outcome;
@@ -5641,11 +5966,16 @@ export class CloudflareScopeHost {
5641
5966
  await holds.switchHoldArm(scopeId, claimId, result.instance ?? null).catch(() => undefined);
5642
5967
  return { rewindingTo: result.rewindingTo };
5643
5968
  }
5644
- /** The modules this scope's own storage has switched off, as a rewind reads them before it arms. */
5969
+ /**
5970
+ * The subjects this scope's own storage has switched off, as a rewind reads them before it arms:
5971
+ * its modules, and (#2029) its peers, each by its hold key (`holdKeyOf`).
5972
+ */
5645
5973
  async offInStorage(scopeId) {
5646
- return (await this.systemGrantsStatusLocal(scopeId))
5647
- .filter((e) => e.schedules === 'off')
5648
- .map((e) => e.moduleId);
5974
+ const [modules, peers] = await Promise.all([this.systemGrantsStatusLocal(scopeId), this.peerGrantsStatusLocal(scopeId)]);
5975
+ return [
5976
+ ...modules.filter((e) => e.schedules === 'off').map((e) => holdKeyOf('system', e.moduleId)),
5977
+ ...peers.filter((e) => e.calls === 'off').map((e) => holdKeyOf('peer', e.vertical)),
5978
+ ];
5649
5979
  }
5650
5980
  /**
5651
5981
  * #1839: wait until every row of this rewind's claim is `SWITCH_HOLD_SETTLE_MS` old. The settle
@@ -5714,7 +6044,11 @@ export class CloudflareScopeHost {
5714
6044
  * arrive as a different class of answer just because the caller asked for a socket.
5715
6045
  */
5716
6046
  liveReads = {
5717
- subscribe: async ({ tenantId, scopeId, principal, request }) => {
6047
+ subscribe: async ({ tenantId, scopeId, principal, request, within }) => {
6048
+ // Decided before anything else, and thrown: a `within` that is neither a plain
6049
+ // EntityRef nor a value `vouchedWithin` built is a caller bug, and the two ways of
6050
+ // guessing at it both open a feed wider than was asked for (#1853).
6051
+ const narrowed = liveWithinOf(within);
5718
6052
  if (!isUpgradeRequest(request)) {
5719
6053
  return new Response('live reads are a WebSocket surface', {
5720
6054
  status: 426,
@@ -5772,6 +6106,10 @@ export class CloudflareScopeHost {
5772
6106
  headers.set(LIVE_PRINCIPAL_HEADER, principal);
5773
6107
  headers.set(LIVE_TENANT_HEADER, tenantId);
5774
6108
  headers.set(LIVE_SCOPE_HEADER, scopeId);
6109
+ if (narrowed)
6110
+ headers.set(LIVE_WITHIN_HEADER, encodeLiveWithin(narrowed));
6111
+ else
6112
+ headers.delete(LIVE_WITHIN_HEADER);
5775
6113
  const forwarded = new Request(new URL(LIVE_SUBSCRIBE_PATH, 'https://scope.substrat.internal'),
5776
6114
  // `new Request(url, { …, headers })` rather than `new Request(request, …)`:
5777
6115
  // the DO is addressed by its own path, not the client's, and the upgrade
@@ -5853,7 +6191,7 @@ export class CloudflareScopeHost {
5853
6191
  throw unknownScopeForTenant(tenantId, scopeId);
5854
6192
  const { roles, tuples, entitlements, identities } = await this.tenantProjection(tenantId);
5855
6193
  const connectionKeys = await this.connectionKeyRows(tenantId, scope.vertical);
5856
- await this.scopeStub(scopeId).applyProjection(tenantId, roles, tuples, entitlements, undefined, identities, connectionKeys);
6194
+ unwrapReply(await this.scopeStub(scopeId).applyProjectionReply(tenantId, roles, tuples, entitlements, undefined, identities, connectionKeys));
5857
6195
  }
5858
6196
  /**
5859
6197
  * Fan the tenant's current state out into ALL its scopes — called after any
@@ -5902,7 +6240,7 @@ export class CloudflareScopeHost {
5902
6240
  if (!keysByVertical.has(vertical)) {
5903
6241
  keysByVertical.set(vertical, this.connectionKeyRows(tenantId, s.vertical));
5904
6242
  }
5905
- await this.scopeStub(s.scope_id).applyProjection(tenantId, roles, tuples, entitlements, undefined, identities, await keysByVertical.get(vertical));
6243
+ unwrapReply(await this.scopeStub(s.scope_id).applyProjectionReply(tenantId, roles, tuples, entitlements, undefined, identities, await keysByVertical.get(vertical)));
5906
6244
  }));
5907
6245
  const failed = settled.flatMap((r, i) => (r.status === 'rejected' ? [{ scope: scopes[i].scope_id, reason: r.reason }] : []));
5908
6246
  if (failed.length === 0)
@@ -5932,10 +6270,11 @@ export class CloudflareScopeHost {
5932
6270
  * from its own storage. No tenant-level tuples, no control plane.
5933
6271
  */
5934
6272
  async provisionScopeLocal(input) {
6273
+ const carry = recordedOffFromWire(input);
5935
6274
  const services = await this.servicePrincipals?.(input.tenantId, input.scopeId);
5936
6275
  const stub = this.scopeStub(input.scopeId);
5937
6276
  await this.migrateAndRecord(input.scopeId); // create the module tables (setMigrationState no-ops on a null CP)
5938
- const switchedOff = await stub.applyProjection(input.tenantId, input.roles.map((r) => ({ role_key: r.key, permissions: JSON.stringify(r.permissions), source: r.source })), [], // no tenant-level tuples — a CP-less vertical grants at scope level only
6277
+ const switchedOff = unwrapReply(await stub.applyProjectionReply(input.tenantId, input.roles.map((r) => ({ role_key: r.key, permissions: JSON.stringify(r.permissions), source: r.source })), [], // no tenant-level tuples — a CP-less vertical grants at scope level only
5939
6278
  // Only PROJECT (and thereby switch on strict enforcement) when the platform actually
5940
6279
  // supplies entitlements. Omitting them leaves the scope un-projected and trusting-
5941
6280
  // upstream, so a vertical whose provision path predates #304 is never denied — it
@@ -6025,12 +6364,10 @@ export class CloudflareScopeHost {
6025
6364
  public_key: k.publicKey,
6026
6365
  }))
6027
6366
  : undefined,
6028
- // #1742: the recorded-off modules, switched off after the seat in the same unit, on the
6029
- // scope this call provisions — the one the seat's tuples name.
6030
- input.switchedOff
6031
- ? { scopeId: input.scopeId, moduleIds: input.switchedOff, at: new Date().toISOString(), tenantHeld: input.tenantHeld }
6032
- : undefined, services?.map((id) => `principal:${id}`));
6033
- return input.switchedOff ? { switchedOff } : {};
6367
+ // #1742: the recorded-off modules and (#2029) peers, switched off after the seat in the
6368
+ // same unit, on the scope this call provisions — the one the seat's tuples name.
6369
+ carry ? { scopeId: input.scopeId, at: new Date().toISOString(), ...carry } : undefined, services?.map((id) => `principal:${id}`)));
6370
+ return carry ? { switchedOff } : {};
6034
6371
  }
6035
6372
  /**
6036
6373
  * Resolve an external identity from a scope's PROJECTED links (#406) — the CP-less
@@ -6169,6 +6506,8 @@ export class CloudflareScopeHost {
6169
6506
  async connectorGrantLocal(connectionId, scopeId, permission, expiresAt) {
6170
6507
  await this.writeScopeTuple(scopeId, subjectRef({ kind: 'connection', id: connectionId }), `granted:${permission}`, `scope:${scopeId}`, expiresAt ?? null);
6171
6508
  }
6509
+ /** #2045 (Codex r3): every switch mover here honours the fence — what `/internal/switch-fence` reports. */
6510
+ switchFenced = true;
6172
6511
  /**
6173
6512
  * The far end of the schedule kill switch for a scope served HERE (#1666):
6174
6513
  * `/internal/system-switch` lands on this, from the shared control plane's
@@ -6181,7 +6520,7 @@ export class CloudflareScopeHost {
6181
6520
  // Parsed on the way out: this is the wire answer the platform reads, and the DO's
6182
6521
  // plain strings become the published shape here rather than on trust.
6183
6522
  // #1819: `switchInScope` releases a rewind's hold on the module once the switch lands.
6184
- return systemSwitchOutcome.parse(await this.switchInScope(scopeId, moduleId, to, new Date().toISOString(), opts?.tenantHeld ?? false));
6523
+ return systemSwitchOutcome.parse(await this.switchInScope('system', scopeId, moduleId, to, new Date().toISOString(), opts?.tenantHeld ?? false, opts?.fence));
6185
6524
  }
6186
6525
  /**
6187
6526
  * The far end of the status read for a scope served HERE (#1674): `/internal/system-grants`
@@ -6216,9 +6555,12 @@ export class CloudflareScopeHost {
6216
6555
  * Move one peer's kill switch on a scope served HERE (#1706) — the far end of
6217
6556
  * `revokeFromPeer` / `restoreToPeer` for a hosted scope. Nothing is audited here; the
6218
6557
  * platform writes the admin rows around this call, as for the schedule switch.
6558
+ * #2029: through `switchInScope`, so the move releases a rewind's hold on the peer once it
6559
+ * lands, as a module's does. #2030: `tenantHeld`, the platform's word that the peer holds a
6560
+ * live tenant-level grant, which this deployment has no directory to read.
6219
6561
  */
6220
- async peerSwitchLocal(scopeId, vertical, to) {
6221
- return peerSwitchOutcome.parse(await this.scopeStub(scopeId).switchPeer(verticalSlug.parse(vertical), scopeId, to, new Date().toISOString()));
6562
+ async peerSwitchLocal(scopeId, vertical, to, opts) {
6563
+ return peerSwitchOutcome.parse(await this.switchInScope('peer', scopeId, verticalSlug.parse(vertical), to, new Date().toISOString(), opts?.tenantHeld ?? false, opts?.fence));
6222
6564
  }
6223
6565
  // -- the cross-vertical far ends (#1705 PR 2) --------------------------------
6224
6566
  // The shared control plane runs the cross-vertical phase for every hosted edge and reaches the
@@ -6232,7 +6574,25 @@ export class CloudflareScopeHost {
6232
6574
  async exportedEventsLocal(tenantId, scopeId, raw) {
6233
6575
  const input = exportReadInput.parse(raw);
6234
6576
  await this.assertServesLocally(tenantId, scopeId, 'readExportedEvents');
6235
- return exportedBatch.parse(await this.scopeStub(scopeId).exportedEventsRead(input, tenantId, scopeId));
6577
+ return this.readExportsThroughDoor(input, tenantId, scopeId);
6578
+ }
6579
+ /**
6580
+ * #2029: a producer's export read, through the CONSUMER's peer door: the consumer's grants on
6581
+ * this scope decide what leaves it, so a consumer the rewind hold keeps off must hold nothing
6582
+ * here, exactly as a switched-off one does. Gated and pinned like every door call; held, the
6583
+ * producer answers every key missing, so the edge pauses rather than reading past the hold.
6584
+ */
6585
+ async readExportsThroughDoor(input, tenantId, scopeId) {
6586
+ const stub = this.scopeStub(scopeId);
6587
+ const door = await this.openDoor('peer', input.consumer, scopeId);
6588
+ try {
6589
+ return exportedBatch.parse(await door.through((instance) => stub.exportedEventsRead(input, tenantId, scopeId, { instance })));
6590
+ }
6591
+ catch (err) {
6592
+ if (!this.isHeldRefusal(err))
6593
+ throw err;
6594
+ return exportedBatch.parse(await stub.exportedEventsRead(input, tenantId, scopeId, { held: true }));
6595
+ }
6236
6596
  }
6237
6597
  /**
6238
6598
  * The consumer's imports and watermarks (#1705), for a scope served HERE. The served-here