@substrat-run/adapter-cloudflare 0.122.1 → 0.125.0

package/dist/host.js

@@ -1,6 +1,7 @@
-import { fromWireFailure, exportReadInput, exportedBatch, importBatch, importResult, importState, importsOfManifestJson, exportsOfManifestJson, importCursorMove, importCursorMoveAt, importCursorMoved, accessLogEntry, adminLogEntry, opsFailureEntry, opsFailureFingerprint, issueEntry, sweepRunEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunsPayload, modelUsageEntry, attachmentRecord, createTenantInput, identityLink, identityPool, createOrgInput, promotionAcknowledgement, bindAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, registerVerticalInput, vertical as verticalSchema, verticalServingState, verticalChannel, verticalVersion, connection, capabilityExchange, capabilityGrant, principalId, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, systemSwitch, systemSwitchOutcome, systemScheduleEntry, systemSwitchRecord, entitlementGrant, entitlementGrantInput, instant, meterReading, subjectRef, createConnectionInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, org as orgSchema, orgMembership, resolvedIdentity, identityMembership, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, delegatedReadRecord, subjectShredReceipt, connectorDispatchKind, callsOfManifestJson, outboundOfManifestJson, substratError, redrainEventsInput, peerCoverage, peerGrantsEntry, peerSwitch, peerSwitchOutcome, verticalCaller, verticalResolution, verticalSlug, } from '@substrat-run/contracts';
+import { isRewindRefusal, REWIND_REFUSED } from './rewind-refusal.js';
+import { delegatedReadParams, fromWireFailure, exportReadInput, exportedBatch, importBatch, importResult, importState, importsOfManifestJson, exportsOfManifestJson, importCursorMove, importCursorMoveAt, importCursorMoved, accessLogEntry, adminLogEntry, opsFailureEntry, opsFailureFingerprint, issueEntry, sweepRunEntry, FRESHNESS_HEARTBEAT_MINUTES, sweepRunsPayload, modelUsageEntry, attachmentRecord, createTenantInput, identityLink, identityPool, createOrgInput, promotionAcknowledgement, bindAcknowledgement, bindHostnameInput, channelHistoryEntry, hostnameBinding, publishVersionInput, AUTO_ADMISSION_NOTE, registerVerticalInput, vertical as verticalSchema, verticalServingState, verticalChannel, verticalVersion, connection, capabilityExchange, capabilityGrant, principalId, connectionGrant, connectionGrantRecord, connectionSecret, systemGrant, systemSwitch, systemSwitchOutcome, systemScheduleEntry, systemSwitchRecord, entitlementGrant, entitlementGrantInput, instant, meterReading, subjectRef, createConnectionInput, projectedConnectionGrant, projectedConnectionKey, moduleManifest, org as orgSchema, orgMembership, resolvedIdentity, identityMembership, roleDefinition, scope as scopeSchema, tenant as tenantSchema, tenantRole, delegatedReadRecord, subjectShredReceipt, connectorDispatchKind, callsOfManifestJson, outboundOfManifestJson, substratError, redrainEventsInput, peerCoverage, peerGrantsEntry, peerSwitch, peerSwitchOutcome, verticalCaller, verticalResolution, verticalSlug, entityObjectRef, } from '@substrat-run/contracts';
 import { normalizeHostname, toRouteTarget } from './route-resolver.js';
-import { attachmentBlobKey, entitlementDenial, foldMeterReading, parseValidationRecords, resolveScopeRecord, ulid, capabilityTokenHash, checkBecomeInput, plausibleSessionToken, foldModelUsage, backoffAt, resolveRetryPolicy, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, openSealed, ImpersonationRefused, assertSessionUsable, impersonationRowValues, mapImpersonationRow, newImpersonationSession, jobRunOf, runDueJobRuns, startJobRun, globalFetch, assertRedrainWindow, platformRequestOf, undrainedEventsOf, withRecorded, systemSwitchedOffMessage, CrossVerticalRegistry, importCursorSourceOf, exportBreaksOf, exportBreakRefusal, bindExportBreaksOf, bindExportBreakRefusal, collectPeers, peerSeats, connectorCallRecord, noopConnectorCallRecorder, recordConnectorCall, settleConnectionUse, splitManifestMigrations, } from '@substrat-run/kernel';
+import { attachmentBlobKey, entitlementDenial, foldMeterReading, parseValidationRecords, resolveScopeRecord, ulid, capabilityTokenHash, checkBecomeInput, plausibleSessionToken, foldModelUsage, backoffAt, resolveRetryPolicy, isSecretBoxConfigured, unconfiguredSecretBox, createSubjectKeys, generateSealingKeyPair, openSealed, ImpersonationRefused, assertSessionUsable, impersonationRowValues, mapImpersonationRow, newImpersonationSession, jobRunOf, runDueJobRuns, startJobRun, globalFetch, assertRedrainWindow, platformRequestOf, undrainedEventsOf, inUnitMovesToAudit, staleCarryRevertRow, staleCarryReverts, withRecorded, systemSwitchedOffMessage, tenantSystemSwitchedOffMessage, CrossVerticalRegistry, importCursorSourceOf, exportBreaksOf, exportBreakRefusal, bindExportBreaksOf, bindExportBreakRefusal, collectPeers, peerSeats, connectorCallRecord, noopConnectorCallRecorder, recordConnectorCall, settleConnectionUse, splitManifestMigrations, } from '@substrat-run/kernel';
 import { isOrangeToOrange, isUpgradeRequest, LIVE_MODE_HEADER, LIVE_PRINCIPAL_HEADER, LIVE_SCOPE_HEADER, LIVE_SUBSCRIBE_PATH, LIVE_TENANT_HEADER, } from './live-reads.js';
 import { tenantStoreDatabaseName } from './d1.js';
 import { blobStoreBucketName, r2TenantBlobStore } from './r2.js';
@@ -67,6 +68,43 @@ function nullControlPlane() {
             },
     });
 }
+/**
+ * #1819: the ONE object in a deployment's scope namespace that carries the rewind hold (see
+ * "the rewind hold" on `CloudflareScopeHost`). A name no `ScopeId` can parse to, so it is
+ * never a scope.
+ */
+export const SWITCH_HOLDS_NAME = 'substrat:switch-holds';
+/**
+ * #1819: how long one read of the hold serves a sweep pass. The pass reads the whole hold once
+ * (it is normally empty) rather than once per scope, so a scope never rewound costs no RPC.
+ */
+export const SWITCH_HOLD_SNAPSHOT_MS = 2_000;
+/**
+ * #1819: how long a rewind waits between writing its hold and rewinding. This is what makes the
+ * snapshot safe. The pass reads its snapshot only AFTER the scope's own state read, and re-reads
+ * one older than `SWITCH_HOLD_SNAPSHOT_MS`. A state read that met rewound storage therefore
+ * finished at least this long after the hold was written, so a snapshot taken before the hold
+ * is past its age by then and is re-read. Each side measures a duration on its own clock;
+ * nothing compares two clocks.
+ */
+export const SWITCH_HOLD_SETTLE_MS = SWITCH_HOLD_SNAPSHOT_MS + 1_000;
+/**
+ * #1839: how many times a rewind waits again, after its settle, for a row a late OFF added to its
+ * claim to be `SWITCH_HOLD_SETTLE_MS` old. The settle argument above holds per ROW, so a row
+ * added during the settle needs its own full settle before the rewind arms. Each late OFF can
+ * cost up to one more settle; past this many, the rewind is REFUSED rather than armed with a young
+ * row, so a switch flapping through the wait cannot hold a rewind forever. See `settleClaim`.
+ */
+export const SWITCH_HOLD_EXTRA_WAITS = 3;
+/**
+ * #1819: how long a claim may stay `pending` before a switch move treats it as armed with no
+ * doomed instance. A claim is pending from capture until its rewind armed or refused, which is
+ * `SWITCH_HOLD_SETTLE_MS` times at most `1 + SWITCH_HOLD_EXTRA_WAITS`, plus a few calls. One still
+ * pending long after that belongs to a rewinding request that died in between, and without this
+ * bound its hold could never be released. The age is `held_at`, stamped on the hold object's
+ * clock, against the releasing host's clock: minutes, not seconds.
+ */
+export const SWITCH_HOLD_PENDING_MAX_MS = 5 * 60_000;
 export class CloudflareScopeHost {
     admin;
     scopeNs;
@@ -135,6 +173,17 @@ export class CloudflareScopeHost {
     peerSwitchDelegation;
     /** #1705 PR 3: the replay lever's reach into the deployment serving a consumer scope. */
     importCursorDelegation;
+    /**
+     * #1819: the last read of the rewind hold, and when it was taken (before the read went out).
+     * Per instance, so per pass: the host is built per request, and never outlives one.
+     */
+    holdSnapshot = null;
+    /**
+     * #1819: the hold read in flight, and when it went out. The sweepers run up to eight scopes
+     * at once on one host, and each can find the snapshot missing or old at the same moment; they
+     * join this one read rather than each sending their own.
+     */
+    holdRefresh = null;
     /**
      * MUST be constructed per request. Never cache an instance across requests.
      *
@@ -568,9 +617,12 @@ export class CloudflareScopeHost {
      * `/internal/restore`. The control plane is the gate and the auditor; this end
      * just replaces its own bytes with the dump's, migration frontier included.
      */
-    async restoreScopeLocal(scopeId, tables) {
-        await this.scopeStub(scopeId).importDump(tables, scopeId);
-        return { tables: tables.length };
+    async restoreScopeLocal(scopeId, tables, 
+    /** #1742: the directory's recorded-off modules, switched off on THIS scope in the replay's
+     *  own event — a dump from before a switch was pulled brings its grants back live. */
+    opts) {
+        const switchedOff = await this.scopeStub(scopeId).importDump(tables, scopeId, opts?.switchedOff ? { moduleIds: opts.switchedOff, at: new Date().toISOString() } : undefined);
+        return { tables: tables.length, ...(opts?.switchedOff ? { switchedOff } : {}) };
     }
     /**
      * Re-apply the vertical's OWN role definitions to one scope, CP-lessly — the
@@ -592,6 +644,10 @@ export class CloudflareScopeHost {
      */
     async deleteScopeLocal(scopeId) {
         await this.scopeStub(scopeId).destroyStorage();
+        // #1819: a deleted scope runs nothing; its claims go with it. Best effort, never a failed reap.
+        await this.switchHoldsStub()
+            .switchHoldRelease(scopeId, null, null)
+            .catch(() => undefined);
     }
     /**
      * Dump one scope's tables from THIS deployment — the data half of a governed
@@ -681,9 +737,12 @@ export class CloudflareScopeHost {
      * Rewind one scope to a pre-migration bookmark (#286's backout) — schema AND data,
      * discarding every write since; the DO enforces the freshness window and restarts
      * itself to complete the restore. Behind the vertical's `/internal/rewind`.
+     *
+     * #1819: what the scope has switched off is held outside it first, so the rewound grants
+     * run nothing until the switch is back in the scope. See `rewindHolding`.
      */
     async rewindScopeLocal(scopeId, bookmark, opts) {
-        return this.scopeStub(scopeId).rewindToBookmark(bookmark, opts);
+        return this.rewindHolding(scopeId, () => this.scopeStub(scopeId).rewindToBookmark(bookmark, opts));
     }
     registerModule(registration) {
         const manifest = moduleManifest.parse(registration.manifest);
@@ -779,13 +838,14 @@ export class CloudflareScopeHost {
         // per-scope schedule kill switch is `revokeFromSystem` (#1666), and its OFF marker is
         // not a grant, so no reconcile can seat it away; `restoreToSystem` is the only way back.
         // (`grantToSystem` still clears a tuple's tombstone, but it does not move the switch.)
+        const seats = [];
         for (const [moduleId, schedules] of this.moduleSchedules) {
             const perms = new Set();
             for (const s of schedules)
                 for (const p of s.permissions)
                     perms.add(p);
             for (const perm of perms) {
-                await this.scopeStub(input.scopeId).seatTuple(`system:${moduleId}`, `granted:${perm}`, `scope:${input.scopeId}`, null);
+                seats.push({ subject: `system:${moduleId}`, relation: `granted:${perm}`, object: `scope:${input.scopeId}`, expires_at: null });
             }
         }
         // #1706: every declared PEER holds its keys on the scope from provisioning on — the one
@@ -793,14 +853,22 @@ export class CloudflareScopeHost {
         // schedule grants above, so a grant a switch tombstoned stays tombstoned and a
         // switched-off peer gets nothing seated.
         for (const seat of peerSeats(collectPeers(this.peerSources), input.scopeId)) {
-            await this.scopeStub(input.scopeId).seatTuple(seat.subject, seat.relation, seat.object, null);
+            seats.push({ subject: seat.subject, relation: seat.relation, object: seat.object, expires_at: null });
         }
         // #1674: re-assert the recorded OFF positions after the seat (`system-switch-record.ts`).
         // Only where this seat landed in the scope's real store — a CP-less host, or a scope
         // bound to no vertical. A scope a vertical's deployment serves is seated there, later,
         // and re-asserted after that deployment's reconcile instead.
-        if (this.cpLess || record.vertical === null) {
-            await this.admin.reassertSystemSwitches(actor, { tenantId: input.tenantId, scopeId: input.scopeId });
+        //
+        // #1742: in the SAME unit as the seat, so no sweep can land between them; the re-assert
+        // after it then finds them off, and audits what the unit moved.
+        const ownStore = this.cpLess || record.vertical === null;
+        const recordedOff = ownStore ? await this.cp.switchedOffModulesOf(input.tenantId, input.scopeId) : [];
+        const switchedOff = await this.scopeStub(input.scopeId).seatTuples(seats, recordedOff.length
+            ? { scopeId: input.scopeId, moduleIds: recordedOff, at: new Date().toISOString() }
+            : undefined);
+        if (ownStore) {
+            await this.admin.reassertSystemSwitches(actor, { tenantId: input.tenantId, scopeId: input.scopeId }, { appliedInUnit: switchedOff });
         }
         // Audit a real provision only; an idempotent re-provision changed nothing.
         if (created) {
@@ -1120,8 +1188,17 @@ export class CloudflareScopeHost {
         const existing = await this.admin.getScopeRecord(actor, tenantId, scopeId);
         if (!existing)
             throw substratError('not_found', `unknown scope ${scopeId} in tenant ${tenantId}`);
-        await this.scopeStub(scopeId).importDump(dump.tables, scopeId);
+        // #1742: a dump from before a switch was pulled brings the module's grants back live, so
+        // the directory's recorded-off modules go back off in the replay's own event. Only where
+        // this is the scope's real store, as for the provision's seat above; a hosted scope is
+        // restored in its deployment, which the platform carries the same list to.
+        const ownStore = this.cpLess || existing.vertical === null;
+        const recordedOff = ownStore ? await this.cp.switchedOffModulesOf(tenantId, scopeId) : [];
+        const switchedOff = await this.scopeStub(scopeId).importDump(dump.tables, scopeId, recordedOff.length ? { moduleIds: recordedOff, at: new Date().toISOString() } : undefined);
         await this.recordAdmin(actor, 'restoreScope', { tenantId, scopeId }, null, { sourceScopeId: dump.scopeId, tables: dump.tables.length, capturedAt: dump.capturedAt });
+        if (recordedOff.length) {
+            await this.admin.reassertSystemSwitches(actor, { tenantId, scopeId }, { appliedInUnit: switchedOff });
+        }
     }
     async snapshotScope(actor, tenantId, scopeId, opts) {
         const source = await this.admin.getScopeRecord(actor, tenantId, scopeId);
@@ -1554,7 +1631,12 @@ export class CloudflareScopeHost {
         const state = await stub.systemScheduleState(moduleId);
         if (state === 'ungranted')
             return report;
-        if (state === 'off') {
+        // #1819: a scope rewound to before its switch was pulled answers `on`. The rewind held the
+        // module outside the scope; read AFTER the state, which is the order `switchHeld` needs.
+        const hold = state === 'on' ? await this.switchHeld(scopeId, moduleId) : { held: false };
+        if (hold.error)
+            report.errors.push({ operation: 'switch-hold', error: hold.error });
+        if (state === 'off' || hold.held) {
             for (const schedule of schedules) {
                 report.skipped += 1;
                 report.runs.push({ operation: schedule.operation, outcome: 'skipped' });
@@ -1572,10 +1654,14 @@ export class CloudflareScopeHost {
                 report.runs.push({ operation: schedule.operation, outcome: 'skipped' });
                 continue;
             }
+            // #1525: one id for THIS call, minted before the invoke so it is the same id
+            // whether the operation succeeds or throws — an event or a denial the call
+            // produces either way carries it, and so does the row below.
+            const invocationId = ulid();
             let status = 'ok';
             try {
                 const scope = await this.getSystemScope(moduleId, tenantId, scopeId);
-                await scope.invoke(schedule.operation, schedule.input);
+                await scope.invoke(schedule.operation, schedule.input, { invocationId });
                 report.fired += 1;
             }
             catch (err) {
@@ -1588,7 +1674,7 @@ export class CloudflareScopeHost {
             }
             // #1288: 'schedule', whatever this operation happens to be called — including
             // `freshness:<something>`, which is exactly the row the evaluator no longer eats.
-            await stub.recordScheduleRun(schedule.operation, new Date(now).toISOString(), status, 'schedule');
+            await stub.recordScheduleRun(schedule.operation, new Date(now).toISOString(), status, 'schedule', invocationId);
             report.runs.push({ operation: schedule.operation, outcome: status === 'ok' ? 'ok' : 'failed' });
         }
         return report;
@@ -1633,6 +1719,18 @@ export class CloudflareScopeHost {
         return {
             tenantId,
             scopeId,
+            // #1746: the door decided it — the same precedence `asPrincipalId` uses below.
+            subjectKind: principal !== undefined
+                ? 'principal'
+                : connectionId !== undefined
+                    ? 'connection'
+                    : systemModuleId !== undefined
+                        ? 'system'
+                        : capabilitySession !== undefined
+                            ? 'capability'
+                            : verticalCaller !== undefined
+                                ? 'vertical'
+                                : undefined,
             invoke: async (operation, input, invokeOptions) => {
                 // Entitlement gate (§4.3): a module loads for a tenant only if the tenant holds its
                 // SKU flag. The COORDINATOR gates the console-managed path against the shared CP
@@ -1729,6 +1827,8 @@ export class CloudflareScopeHost {
                     invokeOptions?.onEntityVersion?.(envelope.concurrency.version);
                 if (envelope.idempotency?.replayed)
                     invokeOptions?.onIdempotentReplay?.();
+                if (envelope.emitted)
+                    invokeOptions?.onEmitted?.(envelope.emitted);
                 return envelope.result;
             },
         };
@@ -1891,7 +1991,8 @@ export class CloudflareScopeHost {
         };
         const writeGrant = async (subject, permission, node, entity, expiresAt) => {
             if (entity) {
-                await this.writeScopeTuple(node.scopeId, subject, `granted:${permission}`, `${entity.entityType}:${entity.entityId}`, expiresAt ?? null);
+                await this.writeScopeTuple(node.scopeId, subject, `granted:${permission}`, entityObjectRef(entity, 'HostAdmin'), // #1856: grant and grantToOrg both land here
+                expiresAt ?? null);
             }
             else if (node.scopeId) {
                 await this.writeScopeTuple(node.scopeId, subject, `granted:${permission}`, `scope:${node.scopeId}`, expiresAt ?? null);
@@ -1997,9 +2098,11 @@ export class CloudflareScopeHost {
                 vertical = rec.vertical;
             }
             const delegation = this.cpLess || vertical !== null ? this.systemSwitchDelegation : undefined;
+            // #1819: a scope whose store is here was rewound here, so `switchInScope` releases its
+            // hold here too. A delegated move releases in the deployment serving the scope.
             const move = (moduleId, to, at) => delegation
                 ? delegation.switch({ tenantId, scopeId, moduleId, to })
-                : this.scopeStub(scopeId).switchSystemSchedules(moduleId, scopeId, to, at);
+                : this.switchInScope(scopeId, moduleId, to, at);
             return { vertical, move };
         };
         const switchSystem = async (actor, raw, to) => {
@@ -2198,20 +2301,52 @@ export class CloudflareScopeHost {
          * so it lands wherever an operator's OFF would have — for a hosted scope, over the
          * `/internal/system-switch` route every deployment since #1666 answers.
          */
-        const reassertSystemSwitchesOf = async (actor, node) => {
+        const reassertSystemSwitchesOf = async (actor, node, opts) => {
             const { tenantId, scopeId } = node;
             const { vertical, move } = await systemSwitchTarget(tenantId, scopeId);
-            const modules = await this.cp.switchedOffModulesOf(tenantId, scopeId);
+            const recorded = new Map(await this.cp.systemSwitchRecordsOf(tenantId, scopeId));
+            // #1742 review: moves the deployment made from a stale list (restored ON after the list
+            // was read), to undo before the OFF pass below.
+            const reverts = staleCarryReverts(recorded, opts?.appliedInUnit);
             // A hosted scope with no delegation configured (Copilot review): this host's own
             // namespace is the module-less placeholder, where the switch would answer `held: false`
             // quietly — a re-assert reported done, and a receipt written, while the deployment
             // serving the scope keeps its schedules running. Refused loudly instead, as the status
             // read is, and only when a re-assert is owed: nothing recorded, nothing to refuse.
-            if (modules.length > 0 && !this.cpLess && vertical !== null && !this.systemSwitchDelegation) {
+            const owed = reverts.length > 0 || [...recorded.values()].includes('off');
+            if (owed && !this.cpLess && vertical !== null && !this.systemSwitchDelegation) {
                 throw substratError('unavailable', `no delegation configured for hosted scope ${scopeId} (vertical '${vertical}') — cannot re-assert ` +
                     `its switched-off schedules in the deployment serving it`);
             }
             const at = new Date().toISOString();
+            const reverted = new Set();
+            for (const moduleId of reverts) {
+                // Re-read immediately before the move: a staff OFF that completed since the read above
+                // (record `off`) must not get a transient ON a due schedule could run in.
+                const current = new Map(await this.cp.systemSwitchRecordsOf(tenantId, scopeId));
+                if (current.get(moduleId) !== 'on')
+                    continue;
+                const outcome = await move(moduleId, 'on', at);
+                reverted.add(moduleId);
+                if (outcome.changed) {
+                    await this.recordAdmin(actor, 'reassertSystemSwitch', { tenantId, scopeId, vertical }, null, {
+                        operationId: ulid(),
+                        ...staleCarryRevertRow(moduleId, outcome),
+                    });
+                }
+            }
+            // Read AFTER the reverts: an OFF that landed meanwhile is switched off below.
+            const modules = await this.cp.switchedOffModulesOf(tenantId, scopeId);
+            // #1742: what the deployment already switched off inside its own unit, audited here —
+            // the move below answers `changed: false` for it and would write no row.
+            // A move the revert above undid is not credited as an in-unit OFF.
+            const applied = opts?.appliedInUnit?.filter((a) => !reverted.has(a.moduleId));
+            for (const row of inUnitMovesToAudit(modules, applied)) {
+                await this.recordAdmin(actor, 'reassertSystemSwitch', { tenantId, scopeId, vertical }, null, {
+                    operationId: ulid(),
+                    ...row,
+                });
+            }
             const results = [];
             for (const moduleId of modules) {
                 const outcome = await move(moduleId, 'off', at);
@@ -2414,7 +2549,12 @@ export class CloudflareScopeHost {
                     }
                 }
                 else {
-                    await writeGrant(subjectRef({ kind: 'system', id: grant.moduleId }), grant.permission, grant.node, undefined, grant.expiresAt);
+                    // #1743: refused while the record holds the module off on any scope — checked and
+                    // written in one control-plane call, where the record and the tenant tuple both live.
+                    const off = await this.cp.writeTenantSystemGrant(grant.node.tenantId, grant.moduleId, `granted:${grant.permission}`, grant.expiresAt ?? null);
+                    if (off.length > 0) {
+                        throw substratError('conflict', tenantSystemSwitchedOffMessage(grant.moduleId, off));
+                    }
                 }
                 await this.recordAdmin(actor, 'grantToSystem', { tenantId: grant.node.tenantId, scopeId: grant.node.scopeId }, null, { moduleId: grant.moduleId, permission: grant.permission, node: grant.node });
                 if (!grant.node.scopeId)
@@ -3050,7 +3190,8 @@ export class CloudflareScopeHost {
                     // host's namespace here would PITR an unrelated, unused DO.
                     return { rewindingTo: bookmark };
                 }
-                return this.scopeStub(scopeId).rewindToBookmark(bookmark, { force: opts?.force });
+                // #1819: the same rewind a dispatched vertical runs, holding what the scope switched off.
+                return this.rewindScopeLocal(scopeId, bookmark, { force: opts?.force });
             },
             createOrg: async (actor, input) => {
                 const parsed = createOrgInput.parse(input);
@@ -3338,7 +3479,7 @@ export class CloudflareScopeHost {
             facetEvents: async (actor, tenantId, scopeId, input) => {
                 await this.scopeRecordForRead(tenantId, scopeId);
                 const result = await this.scopeStub(scopeId).facetEvents(input);
-                await this.recordAccess(actor, 'facetEvents', { tenantId, scopeId }, input, result.buckets.length);
+                await this.recordAccess(actor, 'facetEvents', { tenantId, scopeId }, delegatedReadParams.facetEvents(input, result), result.buckets.length);
                 return result;
             },
             entityHistory: async (actor, tenantId, scopeId, input) => {
@@ -4052,6 +4193,7 @@ export class CloudflareScopeHost {
                     request_id: entry.requestId ?? null,
                     event_type: entry.eventType ?? null,
                     observed_at: entry.observedAt ?? null,
+                    platform_requests: entry.platformRequests == null ? null : JSON.stringify(entry.platformRequests),
                     at: entry.at ?? new Date().toISOString(),
                 });
             },
@@ -4331,6 +4473,274 @@ export class CloudflareScopeHost {
         // via newUniqueId (K-7) and stores the mapping in the directory — deferred.
         return this.scopeNs.get(this.scopeNs.idFromName(scopeId));
     }
+    // -- the rewind hold (#1819) ------------------------------------------------
+    // A PITR rewind to a bookmark from before a schedule kill switch was pulled brings the
+    // module's `system:` grants back live, and nothing in the rewound storage says otherwise.
+    // The scope's OFF modules are read while the pre-rewind storage still holds them, and each
+    // rewind CLAIMS them on `SWITCH_HOLDS_NAME`, which the rewind cannot reach. A module is held
+    // while any claim on it exists, and `runDueSchedules` skips a held module.
+    //
+    // A claim is released only by a switch move KNOWN to survive its rewind (`switchInScope`): a
+    // move on the restored storage, after the rewind armed. A move during the settle, or one the
+    // doomed instance applied, lands in storage the restart discards, so it releases nothing. A
+    // definite refusal drops only that rewind's own claim, so a concurrent rewind keeps its own.
+    // An OFF landing in that same discarded storage joins the claims that would discard it (#1839).
+    switchHoldsStub() {
+        return this.scopeNs.get(this.scopeNs.idFromName(SWITCH_HOLDS_NAME));
+    }
+    /**
+     * Is this module held off on this scope? Read from one snapshot of the whole hold, re-read
+     * once it is older than `SWITCH_HOLD_SNAPSHOT_MS`. Called only AFTER the scope's own state
+     * read, which is the order `SWITCH_HOLD_SETTLE_MS` relies on.
+     *
+     * A failed read fails OPEN except for what this pass already knows is held. Failing closed
+     * would turn one object's outage into every schedule of the deployment stopping; failing open
+     * reopens only the gap this closes, for a rewind that lands during the outage, and the next
+     * reconcile still switches that module off. The error is answered, for the pass to report.
+     */
+    async switchHeld(scopeId, moduleId) {
+        const at = Date.now();
+        let error;
+        if (!this.holdSnapshot || at - this.holdSnapshot.at > SWITCH_HOLD_SNAPSHOT_MS) {
+            const inflight = this.holdRefresh;
+            // Join a read that went out within the snapshot age of this consult: its result is then as
+            // fresh as one this consult would take itself, which is what the settle argument needs.
+            // Only the consult that sent the read reports its failure.
+            if (inflight && at - inflight.at <= SWITCH_HOLD_SNAPSHOT_MS) {
+                await inflight.done;
+            }
+            else {
+                const refresh = { at, done: this.refreshHolds(at) };
+                this.holdRefresh = refresh;
+                try {
+                    ({ error } = await refresh.done);
+                }
+                finally {
+                    if (this.holdRefresh === refresh)
+                        this.holdRefresh = null;
+                }
+            }
+        }
+        const held = this.holdSnapshot?.held.has(`${scopeId} ${moduleId}`) ?? false;
+        return { held, ...(error ? { error } : {}) };
+    }
+    /** One read of the whole hold into the snapshot, stamped with when it went out. Never throws. */
+    async refreshHolds(at) {
+        try {
+            const rows = await this.switchHoldsStub().switchHoldsAll();
+            // A slow read that lands after a newer one never replaces it.
+            if (!this.holdSnapshot || at >= this.holdSnapshot.at) {
+                this.holdSnapshot = { at, held: new Set(rows.map((r) => `${r.scopeId} ${r.moduleId}`)) };
+            }
+            return {};
+        }
+        catch (err) {
+            const known = this.holdSnapshot?.held;
+            const why = err instanceof Error ? err.message : String(err);
+            // Deliberately re-stamped, so a failing hold object is retried once per snapshot age rather
+            // than hammered per scope. For that long, other scopes reuse this possibly pre-hold set
+            // silently, and only the consult that sent this read reports the error: the declared
+            // fail-open, attributed once.
+            if (!this.holdSnapshot || at >= this.holdSnapshot.at)
+                this.holdSnapshot = { at, held: known ?? new Set() };
+            return {
+                error: known
+                    ? `switch hold unreadable (${why}); the ${known.size} hold(s) from this pass's last good read ` +
+                        `still applied, any newer ones did not`
+                    : `switch hold unreadable (${why}); no earlier read in this pass, so no hold applied`,
+            };
+        }
+    }
+    /**
+     * Move one module's switch in this host's own scope DO, and release the claims the move is
+     * known to survive. The claims are read BEFORE the move (S0). A claim in S0 that was already
+     * `armed` had its rewind armed before the move started, and a Durable Object has one live
+     * instance at a time. So the instance that applied the move is either the doomed one (its
+     * write is discarded: keep the claim) or one started after it, on the restored storage (the
+     * write persists: release). A claim still `pending` in S0 may belong to a rewind that has not
+     * armed yet, which would discard this move too: keep it, unless it is older than
+     * `SWITCH_HOLD_PENDING_MAX_MS`. A claim created after S0 is never touched.
+     *
+     * An ON releases every claim in S0, surviving or not. A claim exists to keep off a module the
+     * operator had off, and ON is their newer word: the directory records it before the move. If
+     * the ON itself is discarded by a rewind, the scope comes back as the bookmark had it, and a
+     * claim would then keep the module off while the directory and the status read both say ON.
+     * The same call tombstones the module on the hold object (`switchHoldOn`), after the move: a
+     * rewind that read the switch before this ON cannot claim the module from that read afterwards
+     * (#1839). The release still covers only S0: a row a stale read inserted between S0 and the
+     * tombstone, into a claim that did not hold the module at S0, stays. That window is a few calls,
+     * and it errs toward OFF, until the next surviving move of the switch.
+     *
+     * An OFF that changed the switch JOINS the claims it would not survive (#1839): each claim in S0
+     * or S1 (below) that is still pending (inside the bound), or armed with this move's instance as
+     * its doomed one. Those are exactly the rewinds whose restart discards this OFF, and each of them captured
+     * the module ON. The row takes the claim's state (`switchHoldJoin`, which never recreates a claim
+     * with no rows left) and is stamped on the hold object's clock, and
+     * a pending claim's rewind then waits for it to be a full settle old (`rewindHolding`). The
+     * candidates are read twice, in S0 and again after the move (S1), so a claim written while
+     * this move was queued is joined too. Both reads are scope-wide; S1 is taken only by an OFF
+     * that changed the switch.
+     *
+     * A failed claims read does not stop the move. It is thrown after it, and the caller treats
+     * that as a failed move, with the claims kept, which errs toward OFF. A failed join is thrown
+     * the same way.
+     *
+     * An OFF then an ON of one module, milliseconds apart, can interleave so the ON's release reads
+     * its claims before the OFF's join lands. The joined row then holds the module the operator just
+     * turned ON, until the next surviving move of that switch releases it (another ON does).
+     */
+    async switchInScope(scopeId, moduleId, to, at) {
+        let scopeClaims;
+        let readError;
+        try {
+            scopeClaims = await this.switchHoldsStub().switchHoldClaims(scopeId);
+        }
+        catch (err) {
+            readError = err;
+        }
+        const { instance, ...outcome } = await this.scopeStub(scopeId).switchSystemSchedules(moduleId, scopeId, to, at);
+        if (!scopeClaims)
+            throw readError;
+        const now = Date.now();
+        const pendingLive = (c) => c.state === 'pending' && now - Date.parse(c.heldAt) <= SWITCH_HOLD_PENDING_MAX_MS;
+        const before = scopeClaims.filter((c) => c.moduleId === moduleId);
+        if (to === 'on') {
+            // Every ON, claims or not: its tombstone is what keeps a rewind's stale read from claiming
+            // the module after this (#1839).
+            this.holdSnapshot = null;
+            await this.switchHoldsStub().switchHoldOn(scopeId, moduleId, before.map((c) => c.claimId));
+        }
+        else if (outcome.held && before.length > 0) {
+            const survived = before.filter((c) => !pendingLive(c) && c.doomed !== instance);
+            if (survived.length > 0) {
+                this.holdSnapshot = null;
+                await this.switchHoldsStub().switchHoldRelease(scopeId, moduleId, survived.map((c) => c.claimId));
+            }
+        }
+        if (to === 'off' && outcome.changed) {
+            // S1, read after the move: a claim written while this move was queued behind the capture
+            // is in S1 only. S0 still counts: a claim pending in S0 may have armed with another
+            // instance by S1, and this move may have landed before that arm.
+            const after = await this.switchHoldsStub().switchHoldClaims(scopeId);
+            const joins = new Set([...scopeClaims, ...after].filter((c) => pendingLive(c) || c.doomed === instance).map((c) => c.claimId));
+            if (joins.size > 0) {
+                this.holdSnapshot = null;
+                await this.switchHoldsStub().switchHoldJoin(scopeId, moduleId, [...joins]);
+            }
+        }
+        return outcome;
+    }
+    /**
+     * Rewind one scope, claiming what it has switched off, and what an earlier rewind still holds on
+     * it. The OFF modules are read and claimed BEFORE the rewind, while this storage still holds
+     * them, and the rewind then waits until every row of its claim is `SWITCH_HOLD_SETTLE_MS` old
+     * (`settleClaim`). Its claim is `armed` once the rewind arms, naming the doomed
+     * instance. Only a DEFINITE refusal drops the claim, and only this rewind's own. Any other throw
+     * may come after the DO armed the bookmark, so the claim is armed on what a probe finds: the
+     * serving instance if it armed, else none. A probe that fails leaves the claim pending, and
+     * `SWITCH_HOLD_PENDING_MAX_MS` bounds that.
+     *
+     * With nothing to claim, the rewind neither claims nor settles. An OFF that lands between that
+     * status read and the arm (a few calls) is then NOT held at all: the rewound scope runs that
+     * module until the next reconcile re-asserts the switch, #1819's gap for those milliseconds.
+     * Closing it would make every rewind claim and settle, 3 s on each, to cover a few calls.
+     */
+    async rewindHolding(scopeId, rewind) {
+        const holds = this.switchHoldsStub();
+        // What this rewind must keep off: what the storage has off, AND what an earlier rewind still
+        // holds here. A module held by an earlier rewind is ON in this storage (that is why it is
+        // held), so the storage alone would give this rewind no claim on it. Then a re-assert that
+        // lands on the instance THIS rewind dooms would release the earlier claim, and this
+        // rewind's restart would discard the re-assert's write.
+        // The token first: an ON that moves after these reads tombstones its module past it (#1839).
+        const token = await holds.switchHoldToken();
+        const [offInStorage, held] = await Promise.all([this.offInStorage(scopeId), holds.switchHoldsAll()]);
+        const off = [...new Set([...offInStorage, ...held.filter((h) => h.scopeId === scopeId).map((h) => h.moduleId)])];
+        if (off.length === 0)
+            return { rewindingTo: (await rewind()).rewindingTo };
+        const claimId = ulid();
+        await holds.switchHoldClaim(scopeId, off, claimId, token);
+        this.holdSnapshot = null;
+        try {
+            await this.settleClaim(holds, scopeId, claimId);
+        }
+        catch (err) {
+            // Nothing was asked to arm: this rewind never happens, and what it claimed stays in storage.
+            await holds.switchHoldDrop(scopeId, claimId).catch(() => undefined);
+            throw err;
+        }
+        let result;
+        try {
+            result = await rewind();
+        }
+        catch (err) {
+            if (isRewindRefusal(err)) {
+                await holds.switchHoldDrop(scopeId, claimId).catch(() => undefined);
+            }
+            else {
+                const probe = await this.scopeStub(scopeId)
+                    .rewindProbe()
+                    .catch(() => undefined);
+                if (probe)
+                    await holds.switchHoldArm(scopeId, claimId, probe.armed ? probe.instance : null).catch(() => undefined);
+            }
+            throw err;
+        }
+        await holds.switchHoldArm(scopeId, claimId, result.instance ?? null).catch(() => undefined);
+        return { rewindingTo: result.rewindingTo };
+    }
+    /** The modules this scope's own storage has switched off, as a rewind reads them before it arms. */
+    async offInStorage(scopeId) {
+        return (await this.systemGrantsStatusLocal(scopeId))
+            .filter((e) => e.schedules === 'off')
+            .map((e) => e.moduleId);
+    }
+    /**
+     * #1839: wait until every row of this rewind's claim is `SWITCH_HOLD_SETTLE_MS` old. The settle
+     * argument holds per row: a row younger than that when the rewound storage exists can be missed
+     * by a pass whose hold snapshot was read just before it. So after the settle, each round reads
+     * the scope's OFF modules again and claims them (an OFF that landed after the capture, in storage
+     * this rewind discards), then asks the hold object how old the claim's youngest row is, joined
+     * rows (`switchInScope`) included. The age is measured on the hold object's clock, the same one
+     * that stamped the row; this host only sleeps the difference. Each status read is preceded by a
+     * token read, so a module an operator turned ON after that read is not claimed from it
+     * (`switchHoldClaim`).
+     *
+     * Bounded at `SWITCH_HOLD_EXTRA_WAITS` more waits: past that, the rewind is REFUSED, with the
+     * refusal prefix. Nothing was asked to arm, so `rewindHolding` drops only this claim, and every
+     * OFF pulled meanwhile stays in the scope's own storage. The owner retries, and the retry's
+     * capture reads them all. Arming instead would leave a row younger than the settle, whose module
+     * a pass with a snapshot read shortly before that row could run once on the rewound storage.
+     *
+     * What remains: an OFF whose join lands after the last age read and before the arm (a few calls),
+     * or on the doomed instance in the moment between the arm and its restart. It is claimed, but by
+     * a row younger than the settle: that module can run once, in a pass whose hold snapshot was read
+     * in the `SWITCH_HOLD_SNAPSHOT_MS` before the row, until that snapshot expires. If the claim has
+     * no rows by then, the OFF has nothing to join (`switchHoldJoin`), and in that same moment it is
+     * not held at all. That happens when an ON emptied the claim, and also when a tombstone refused
+     * every module the capture read, so the capture wrote no rows. An OFF before the last re-read is
+     * claimed by the re-read.
+     */
+    async settleClaim(holds, scopeId, claimId) {
+        await new Promise((resolve) => setTimeout(resolve, SWITCH_HOLD_SETTLE_MS));
+        for (let waits = 0;; waits++) {
+            const token = await holds.switchHoldToken();
+            const off = await this.offInStorage(scopeId);
+            if (off.length > 0) {
+                await holds.switchHoldClaim(scopeId, off, claimId, token);
+                this.holdSnapshot = null;
+            }
+            const youngest = await holds.switchHoldYoungestMs(scopeId, claimId);
+            if (youngest === null || youngest >= SWITCH_HOLD_SETTLE_MS)
+                return;
+            if (waits === SWITCH_HOLD_EXTRA_WAITS) {
+                throw new Error(`${REWIND_REFUSED}schedule switches kept being pulled off while this rewind waited for them to ` +
+                    `settle (${waits} extra waits); nothing was rewound, and those switches stay off. Retry the rewind`);
+            }
+            // Never longer than one settle, even if the hold object's clock moved back under a restart.
+            await new Promise((resolve) => setTimeout(resolve, SWITCH_HOLD_SETTLE_MS - Math.max(0, youngest)));
+        }
+    }
     // -- live reads (#938): the door ------------------------------------------
     // The coordinator's half of the live-read path. It decides two things and no more:
     // whether this CONNECTION can carry a push at all, and who is asking. What a
@@ -4482,9 +4892,15 @@ export class CloudflareScopeHost {
     async projectScope(tenantId, scopeId) {
         if (!this.scopeLocalPermissions)
             return;
-        const { roles, tuples, entitlements, identities } = await this.tenantProjection(tenantId);
+        // The directory decides which tenant this scope belongs to (#1738): the scope's first
+        // projection pins its `provisioned_for` receipt, so a tenant taken from a request rather
+        // than from the record must not reach `applyProjection` — it would pin the wrong one for
+        // good. Read first, so a pair the directory does not hold writes nothing.
         const scope = await this.cp.getScopeRecord(tenantId, scopeId);
-        const connectionKeys = await this.connectionKeyRows(tenantId, scope?.vertical);
+        if (!scope)
+            throw substratError('not_found', `unknown scope for tenant: (${tenantId}, ${scopeId})`);
+        const { roles, tuples, entitlements, identities } = await this.tenantProjection(tenantId);
+        const connectionKeys = await this.connectionKeyRows(tenantId, scope.vertical);
         await this.scopeStub(scopeId).applyProjection(tenantId, roles, tuples, entitlements, undefined, identities, connectionKeys);
     }
     /**
@@ -4526,13 +4942,23 @@ export class CloudflareScopeHost {
         // rather than once for the tenant — memoized, because a fan-out over twenty scopes
         // of one vertical must not mint or re-read twenty times.
         const keysByVertical = new Map();
-        await Promise.all(scopes.map(async (s) => {
+        // allSettled, not all (#1738): a scope that refuses (its `provisioned_for` receipt names
+        // another tenant) must not stop its siblings converging, or one bad scope would freeze every
+        // revoke for the tenant's healthy ones. The failure is still loud, after the rest have landed.
+        const settled = await Promise.allSettled(scopes.map(async (s) => {
             const vertical = s.vertical ?? '';
             if (!keysByVertical.has(vertical)) {
                 keysByVertical.set(vertical, this.connectionKeyRows(tenantId, s.vertical));
             }
             await this.scopeStub(s.scope_id).applyProjection(tenantId, roles, tuples, entitlements, undefined, identities, await keysByVertical.get(vertical));
         }));
+        const failed = settled.flatMap((r, i) => (r.status === 'rejected' ? [{ scope: scopes[i].scope_id, reason: r.reason }] : []));
+        if (failed.length === 0)
+            return;
+        // One aggregated error, so the caller learns which scopes did not converge and that the
+        // rest did.
+        const named = failed.map((f) => `${f.scope}: ${f.reason instanceof Error ? f.reason.message : String(f.reason)}`).join('; ');
+        throw substratError('conflict', `projection reached ${scopes.length - failed.length} of ${scopes.length} scopes of tenant ${tenantId}; not converged — ${named}`);
     }
     /**
      * Re-project a tenant's full state into every one of its scopes — the
@@ -4556,7 +4982,7 @@ export class CloudflareScopeHost {
     async provisionScopeLocal(input) {
         const stub = this.scopeStub(input.scopeId);
         await this.migrateAndRecord(input.scopeId); // create the module tables (setMigrationState no-ops on a null CP)
-        await stub.applyProjection(input.tenantId, input.roles.map((r) => ({ role_key: r.key, permissions: JSON.stringify(r.permissions), source: r.source })), [], // no tenant-level tuples — a CP-less vertical grants at scope level only
+        const switchedOff = await stub.applyProjection(input.tenantId, input.roles.map((r) => ({ role_key: r.key, permissions: JSON.stringify(r.permissions), source: r.source })), [], // no tenant-level tuples — a CP-less vertical grants at scope level only
         // Only PROJECT (and thereby switch on strict enforcement) when the platform actually
         // supplies entitlements. Omitting them leaves the scope un-projected and trusting-
         // upstream, so a vertical whose provision path predates #304 is never denied — it
@@ -4645,7 +5071,13 @@ export class CloudflareScopeHost {
                 key_id: k.keyId,
                 public_key: k.publicKey,
             }))
+            : undefined, 
+        // #1742: the recorded-off modules, switched off after the seat in the same unit, on the
+        // scope this call provisions — the one the seat's tuples name.
+        input.switchedOff
+            ? { scopeId: input.scopeId, moduleIds: input.switchedOff, at: new Date().toISOString() }
             : undefined);
+        return input.switchedOff ? { switchedOff } : {};
     }
     /**
      * Resolve an external identity from a scope's PROJECTED links (#406) — the CP-less
@@ -4718,7 +5150,8 @@ export class CloudflareScopeHost {
      * (writeTuple is INSERT OR REPLACE), so it is safe to re-issue on every link.
      */
     async grantEntityLocal(scopeId, principal, permission, entity) {
-        await this.scopeStub(scopeId).writeTuple(`principal:${principal}`, `granted:${permission}`, `${entity.entityType}:${entity.entityId}`, null);
+        await this.scopeStub(scopeId).writeTuple(`principal:${principal}`, `granted:${permission}`, entityObjectRef(entity, 'grantEntityLocal'), // #1856
+        null);
     }
     // -- the connector write-back's far end (#574) -----------------------------
     // A CP-less dispatch vertical cannot run a connector, so the shared control
@@ -4776,7 +5209,8 @@ export class CloudflareScopeHost {
     async systemSwitchLocal(scopeId, moduleId, to) {
         // Parsed on the way out: this is the wire answer the platform reads, and the DO's
         // plain strings become the published shape here rather than on trust.
-        return systemSwitchOutcome.parse(await this.scopeStub(scopeId).switchSystemSchedules(moduleId, scopeId, to, new Date().toISOString()));
+        // #1819: `switchInScope` releases a rewind's hold on the module once the switch lands.
+        return systemSwitchOutcome.parse(await this.switchInScope(scopeId, moduleId, to, new Date().toISOString()));
     }
     /**
      * The far end of the status read for a scope served HERE (#1674): `/internal/system-grants`