@substrat-run/adapter-cloudflare 0.122.1 → 0.125.0

package/dist/scope-do.js

@@ -1,10 +1,11 @@
+import { REWIND_REFUSED } from './rewind-refusal.js';
 import { DurableObject } from 'cloudflare:workers';
-import { ATTACHMENT_ADDED, ATTACHMENT_REMOVED, attachmentRecord, domainEvent, domainEventInput, eventId, instant, objectRef, toWireFailure, grantRefFromProof, principalId, scopeId as scopeIdOf, tenantId as tenantIdOf, platformRequestInput, platformRequestId, MAX_PENDING_PLATFORM_REQUESTS, MAX_PENDING_SWEEP_RUNS, SWEEP_RUNS_KIND, SCOPE_TABLE_PAGE_MAX, SCOPE_QUERY_ROW_MAX, listLimitOf, requestFingerprint, substratError, assertReplayableDump, REDRAIN_BATCH, } from '@substrat-run/contracts';
-import { ulid, createUlid, assertAllowed, ConnectionSealingKeyUnavailableError, noSealingKeyMessage, sealTo, assertReadOnlyQuery, entitlementDenial, platformRequestHistoryQuery, platformRequestOf, PLATFORM_REQUEST_COLUMNS, PLATFORM_REQUEST_REDACTION_SQL, platformRequestRedactionParams, platformRequestRedactionQuery, intentPayloadCarriesSubject, redactSubjectJobRuns, JOB_RUN_PATCH_SQL, JOB_STEP_RECORD_SQL, DELIVERY_ERROR_REDACTION_SQL, REDACTED_DELIVERY_NOTE, seatScopeTuple, effectiveRoleGrantQuery, switchSystemSchedules, peerGrantsStatus, systemGrantsStatus, systemScheduleState, systemSwitchedOff, denialListQuery, denialSummaryQuery, denialTotalsQuery, DENIAL_WINDOW_QUERY, mapDenialRow, mapDenialSummaryBuckets, PermissionDenied, assertImpersonationWrites, assertModuleEnqueueableKind, impersonationStampOf, createAtomic, NotSearchable, isSearchIndexTable, searchIndexDdl, searchIndexPlans, NotListable, listIndexPlans, moduleMigrations, listQuery, cursorOf, searchLimit, searchMatchExpression, searchQuery, IDEMPOTENCY_DDL, assertIdempotencyKey, assertPermissionKey, idempotencyLookupQuery, idempotencyPruneStatement, idempotencyRecordStatement, idempotencyOptedOutMessage, replayFor, entityVersionQuery, entityVersionOf, assertIfMatch, OUTBOX_ENTITY_INDEX, SCHEDULE_STATE_DDL, SCHEDULE_STATE_REBUILD, scheduleStateHasKind, VERTICAL_EVENTS_DDL, EXPORT_HOPS_SQL, IMPORT_CURSORS_SQL, IMPORT_CURSOR_OF_SQL, OUTBOX_MARK_SQL, exportedSinceQuery, IMPORT_CURSOR_ADVANCE_SQL, IMPORT_RECORD_SQL, moveImportCursor, CrossVerticalRegistry, exportReadPlan, exportReadQuery, planExportBatch, withheldNote, JOB_RUN_DDL, jobRunListLimit, } from '@substrat-run/kernel';
+import { ATTACHMENT_ADDED, ATTACHMENT_REMOVED, attachmentRecord, domainEvent, domainEventInput, eventId, instant, objectRef, entityObjectRef, toWireFailure, grantRefFromProof, principalId, scopeId as scopeIdOf, tenantId as tenantIdOf, platformRequestInput, platformRequestId, MAX_PENDING_PLATFORM_REQUESTS, MAX_PENDING_SWEEP_RUNS, SWEEP_RUNS_KIND, SCOPE_TABLE_PAGE_MAX, SCOPE_QUERY_ROW_MAX, listLimitOf, requestFingerprint, substratError, assertReplayableDump, REDRAIN_BATCH, } from '@substrat-run/contracts';
+import { ulid, DO_SQL_LIMITS, createUlid, assertAllowed, ConnectionSealingKeyUnavailableError, noSealingKeyMessage, sealTo, assertReadOnlyQuery, entitlementDenial, platformRequestHistoryQuery, platformRequestOf, PLATFORM_REQUEST_COLUMNS, PLATFORM_REQUEST_REDACTION_SQL, platformRequestRedactionParams, platformRequestRedactionQuery, intentPayloadCarriesSubject, redactSubjectJobRuns, JOB_RUN_PATCH_SQL, JOB_STEP_RECORD_SQL, DELIVERY_ERROR_REDACTION_SQL, REDACTED_DELIVERY_NOTE, seatScopeTuple, effectiveRoleGrantQuery, switchRecordedOff, switchSystemSchedules, peerGrantsStatus, systemGrantsStatus, systemScheduleState, systemSwitchedOff, denialListQuery, denialSummaryQuery, denialTotalsQuery, DENIAL_WINDOW_QUERY, mapDenialRow, mapDenialSummaryBuckets, PermissionDenied, assertImpersonationWrites, assertModuleEnqueueableKind, impersonationStampOf, createAtomic, NotSearchable, isSearchIndexTable, searchIndexDdl, searchIndexPlans, NotListable, listIndexPlans, moduleMigrations, listQuery, cursorOf, searchLimit, searchMatchExpression, searchQuery, IDEMPOTENCY_DDL, assertIdempotencyKey, assertPermissionKey, idempotencyLookupQuery, idempotencyPruneStatement, idempotencyRecordStatement, idempotencyOptedOutMessage, replayFor, entityVersionQuery, entityVersionOf, assertIfMatch, OUTBOX_ENTITY_INDEX, SCHEDULE_STATE_DDL, SCHEDULE_STATE_REBUILD, scheduleStateHasKind, VERTICAL_EVENTS_DDL, EXPORT_HOPS_SQL, IMPORT_CURSORS_SQL, IMPORT_CURSOR_OF_SQL, OUTBOX_MARK_SQL, exportedSinceQuery, emittedSinceQuery, emittedReportOf, EMITTED_REPORT_CAP, IMPORT_CURSOR_ADVANCE_SQL, IMPORT_RECORD_SQL, moveImportCursor, CrossVerticalRegistry, exportReadPlan, exportReadQuery, planExportBatch, withheldNote, JOB_RUN_DDL, jobRunListLimit, } from '@substrat-run/kernel';
 import { isUpgradeRequest, readSubscription, LIVE_FANOUT_LIMIT, LIVE_MODE_HEADER, LIVE_PRINCIPAL_HEADER, LIVE_SCOPE_HEADER, LIVE_SUBSCRIBE_PATH, LIVE_TENANT_HEADER, } from './live-reads.js';
 import { OperationQueue } from './serialization.js';
 import { doScopedSql, doSpineSql } from './sql.js';
-import { actorOf, admitPeer, collectPeers, switchPeer, assertNoSecret, CAPABILITY_DDL, CAPABILITY_EXCHANGE_OPERATION, capabilityAttachmentWriteRefused, createCapabilityVerbs, exchangeCapability, guardSecrets, mintBecomeCapability, redactSecrets, resolveCapabilitySession, revokeCapabilityAsPlatform, domainEventOf, facetEvents, readDeadLetters, readHistory, readInvocation, readUndrainedOutbox, walkEventCause, walkEventEffects, } from '@substrat-run/kernel';
+import { actorOf, admitPeer, collectPeers, switchPeer, assertNoSecret, CAPABILITY_DDL, CAPABILITY_EXCHANGE_OPERATION, capabilityAttachmentWriteRefused, createCapabilityVerbs, exchangeCapability, guardSecrets, mintBecomeCapability, redactSecrets, redactSecretText, moduleLog, resolveCapabilitySession, revokeCapabilityAsPlatform, domainEventOf, facetEvents, readDeadLetters, readHistory, readInvocation, readUndrainedOutbox, walkEventCause, walkEventEffects, } from '@substrat-run/kernel';
 import { createDoTupleChecker, createLocalControlPlaneReader } from './checker.js';
 /**
  * The key marking a scope DO whose storage was destroyed (`destroyStorage`).
@@ -555,8 +556,20 @@ export function defineScopeDO(modules, bareOps) {
         lastFailure = null;
         /** Passes of `applyPendingMigrations` that had work to do — see `migrationAttemptsOnInstance`. */
         migrationRuns = 0;
+        /**
+         * TEST-ONLY (#1860): no production code reads this. An exact `'ping'` is answered
+         * by `setWebSocketAutoResponse` without reaching `webSocketMessage` at all, and
+         * this count is how a test proves that from outside the object — read through
+         * `runInDurableObject` — rather than assuming the runtime's documented behaviour.
+         */
+        webSocketMessagesHandled = 0;
         constructor(ctx, env) {
             super(ctx, env);
+            // #1860: the client's 45s keep-alive (demos/ticket0/app/src/feed.ts) answered at
+            // the runtime level, before this object wakes for it. Without this, every idle
+            // subscriber's ping still reaches `webSocketMessage` and pins the DO in memory —
+            // exactly the hibernation cost `acceptWebSocket` (below) exists to avoid.
+            ctx.setWebSocketAutoResponse(new WebSocketRequestResponsePair('ping', 'pong'));
             this.sql = ctx.storage.sql;
             for (const stmt of splitSqlStatements(KERNEL_DDL)) {
                 this.sql.exec(stmt);
@@ -900,22 +913,37 @@ export function defineScopeDO(modules, bareOps) {
             return { consumes, cursors };
         }
         /**
-         * #1705 PR 2: does this DO hold a scope PROVISIONED here for `tenantId`?
+         * #1705 PR 2 / #1738: does this DO hold a scope PROVISIONED here for `tenantId`?
          *
          * A CP-less deployment has no directory to ask whether it serves a scope. What it does
-         * have is what provisioning wrote: `provisionScopeLocal` projects the vertical's role
-         * definitions under the tenant (`_substrat_roles`), and a restore re-projects them. A scope
-         * this deployment never provisioned has none of that. Its DO is empty, and a cross-vertical
-         * verb answering from it is a wrong answer rather than a failure: a watermark of "never
-         * read", or a delivery journaled into a scope that is not the install. A scope provisioned
-         * for ANOTHER tenant has roles under that tenant's id only, so the same read is K-3's pair
-         * check too.
+         * have is what provisioning wrote. A scope this deployment never provisioned has none of
+         * it: its DO is empty, and a cross-vertical verb answering from it is a wrong answer rather
+         * than a failure: a watermark of "never read", or a delivery journaled into a scope that is
+         * not the install.
+         *
+         * The answer is the `provisioned_for` RECEIPT `applyProjection` writes (#1738): the tenant
+         * this scope was provisioned for, stated rather than inferred. A receipt decides alone, so
+         * a scope holding a receipt for ANOTHER tenant is refused whatever role rows sit in it (K-3's
+         * pair check), and a stray role row can no longer make a scope serve a tenant it was never
+         * provisioned for.
+         *
+         * A scope with NO receipt was provisioned before it existed, or has just been restored (a
+         * dump's receipt is dropped on import: it describes the scope the dump came from). Only
+         * then is the old inference used: `provisionScopeLocal` projected role definitions under
+         * the tenant, so a `_substrat_roles` row for it says the same thing. The next projection
+         * (reconcile, or the restore's repair) writes the receipt, after which the inference is
+         * never consulted again.
          *
          * Read without migrating, so asking about a foreign scope leaves its DO as empty as it found it.
          */
         async servesTenant(tenantId) {
-            const hasRoles = this.sql.exec(`SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '_substrat_roles'`).toArray().length > 0;
-            if (!hasRoles)
+            const has = (table) => this.sql.exec(`SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?`, table).toArray().length > 0;
+            if (has('_substrat_meta')) {
+                const receipt = this.sql.exec(`SELECT value FROM _substrat_meta WHERE key = 'provisioned_for'`).toArray()[0];
+                if (receipt)
+                    return receipt.value === tenantId;
+            }
+            if (!has('_substrat_roles'))
                 return false;
             return this.sql.exec('SELECT 1 FROM _substrat_roles WHERE tenant_id = ? LIMIT 1', tenantId).toArray().length > 0;
         }
@@ -1174,26 +1202,34 @@ export function defineScopeDO(modules, bareOps) {
         async rewindToBookmark(bookmark, opts) {
             const storage = this.ctx.storage;
             if (typeof storage.onNextSessionRestoreBookmark !== 'function') {
-                throw new Error('point-in-time rewind is not available on this host (PITR is production-plane only)');
+                throw new Error(`${REWIND_REFUSED}point-in-time rewind is not available on this host (PITR is production-plane only)`);
             }
             const row = this.sql
                 .exec('SELECT taken_at FROM _substrat_migration_bookmarks WHERE bookmark = ?', bookmark)
                 .toArray()[0];
             if (!opts?.force) {
                 if (!row) {
-                    throw new Error('unknown bookmark — not one this scope recorded before a migration (force admits any bookmark Cloudflare holds)');
+                    throw new Error(`${REWIND_REFUSED}unknown bookmark — not one this scope recorded before a migration ` +
+                        '(force admits any bookmark Cloudflare holds)');
                 }
                 const ageMs = Date.now() - Date.parse(row.taken_at);
                 if (ageMs > 24 * 60 * 60 * 1000) {
-                    throw new Error(`bookmark is ${Math.round(ageMs / 3_600_000)}h old — rewinding discards EVERY write since; ` +
+                    throw new Error(`${REWIND_REFUSED}bookmark is ${Math.round(ageMs / 3_600_000)}h old — rewinding discards EVERY write since; ` +
                         `use the backup restore path (#278), or force if the loss is intended`);
                 }
             }
+            // #1819: arming from BEFORE the call, so a probe that lands while it is in flight answers
+            // armed. Left set if the call throws: that is not a refusal, and it may have armed.
+            this.rewindArming = true;
             const confirmed = await storage.onNextSessionRestoreBookmark(bookmark);
+            // #1819: from here, everything this instance writes is discarded at the restart.
+            this.rewindArmed = true;
             // Answer first, then restart to complete the restore — an immediate abort
             // would take the RPC response down with it.
             setTimeout(() => this.ctx.abort(), 100);
-            return { rewindingTo: confirmed ?? bookmark };
+            // #1819: which instance is doomed, so the host can tell a switch move made here from one
+            // made on the restored storage after the restart.
+            return { rewindingTo: confirmed ?? bookmark, instance: this.instanceId };
         }
         /**
          * The scope's live `connection:<id>` grant tuples (#726 gap 1) — the read half of
@@ -1249,6 +1285,21 @@ export function defineScopeDO(modules, bareOps) {
                 this.sql.exec(seat.sql, ...seat.params);
             });
         }
+        /**
+         * Provisioning's seat of many tuples as ONE unit, then the directory's recorded-off
+         * modules switched off in it (#1742) — the CP-full mirror of what `applyProjection` does
+         * for a CP-less provision. One tuple per `seatTuple` call left a window between the seat
+         * and the re-assert in which a sweep could run the grants just seated.
+         */
+        async seatTuples(tuples, switchOff) {
+            return this.queue.enqueue(() => this.ctx.storage.transactionSync(() => {
+                for (const t of tuples) {
+                    const seat = seatScopeTuple(t.subject, t.relation, t.object, t.expires_at);
+                    this.sql.exec(seat.sql, ...seat.params);
+                }
+                return switchOff ? switchRecordedOff(this.switchSql(), switchOff) : [];
+            }));
+        }
         /**
          * The dispatch capability's admission (#726 remedy B): this delivery may read the
          * attachments of the entity its own spine row names, and no others.
@@ -1508,6 +1559,12 @@ export function defineScopeDO(modules, bareOps) {
                     // that exports something, and never for a read-only session, which commits nothing.
                     const exportTypes = impersonation?.mode === 'read-only' ? [] : this.crossVertical.exportTypes();
                     const exportMark = exportTypes.length > 0 ? this.outboxMark() : null;
+                    // #1746: the same mark, when the caller asked what this call emitted. The callback
+                    // itself never runs here — its presence is the request, and the report travels
+                    // back in the envelope for the coordinator to deliver.
+                    const emittedMark = invokeOptions?.onEmitted !== undefined && impersonation?.mode !== 'read-only'
+                        ? (exportMark ?? this.outboxMark())
+                        : null;
                     let result;
                     let committedVersion = null;
                     // #116: set when this invocation was answered from a recording rather
@@ -1641,6 +1698,13 @@ export function defineScopeDO(modules, bareOps) {
                     // own retry backstop, which is what that backstop is for.
                     // #1525: still inside the queued body that set it, so these deliveries are this
                     // call's own work — the same tail its consumers' emits are stamped in.
+                    // #1746: read BEFORE `settleCommitted` drains the consumers, so the rows above the
+                    // mark are this operation's own emits. A replay committed nothing.
+                    let emitted;
+                    if (emittedMark !== null && !replayed) {
+                        const q = emittedSinceQuery(emittedMark, EMITTED_REPORT_CAP);
+                        emitted = emittedReportOf(this.sql.exec(q.sql, ...q.params).toArray());
+                    }
                     if (!replayed)
                         await this.settleCommitted(tenantId, scopeId, liveSince, this.invocationId);
                     // After the tail, so an exported type a consumer emitted in it counts too.
@@ -1651,6 +1715,7 @@ export function defineScopeDO(modules, bareOps) {
                         // #1705 PR 2: the coordinator fires `onExportedEvents` from this. Omitted at 0, so
                         // the envelope of a deployment that exports nothing is byte-for-byte what it was.
                         ...(exported > 0 ? { exported } : {}),
+                        ...(emitted ? { emitted } : {}),
                         ...(impersonation ? { impersonation: { honoured: true } } : {}),
                         // #1672: the acknowledgement the coordinator's skew check reads — see `invoke`.
                         ...(capabilitySession !== undefined ? { capability: { honoured: true } } : {}),
@@ -1774,8 +1839,14 @@ export function defineScopeDO(modules, bareOps) {
          * Deliberately NOT a place to let a client narrow or widen what it receives: a
          * filter the client chooses is a filter the client can choose wrongly, and the
          * only filter that matters here is the one it does not control.
+         *
+         * #1860: an exact `'ping'` is now answered by `setWebSocketAutoResponse` in the
+         * constructor, at the runtime level, without waking this object — so this branch no
+         * longer runs for it. Kept as the documented fallback for a client on an older
+         * build, or a hop where the runtime's auto-response is unavailable.
          */
         webSocketMessage(ws, message) {
+            this.webSocketMessagesHandled++;
             if (message === 'ping')
                 ws.send('pong');
         }
@@ -2262,7 +2333,9 @@ export function defineScopeDO(modules, bareOps) {
          * as one `transactionSync`, so its reads and writes are one unit.
          */
         async switchSystemSchedules(moduleId, scopeId, to, at) {
-            return this.queue.enqueue(() => this.ctx.storage.transactionSync(() => switchSystemSchedules(this.switchSql(), { moduleId, scopeId, to, at })));
+            const outcome = await this.queue.enqueue(() => this.ctx.storage.transactionSync(() => switchSystemSchedules(this.switchSql(), { moduleId, scopeId, to, at })));
+            // #1819: the instance that applied it, which the rewind hold's release rule reads.
+            return { ...outcome, instance: this.instanceId };
         }
         /**
          * The replay lever on this scope (#1705 PR 3): the kernel's `moveImportCursor`, queued with
@@ -2307,6 +2380,185 @@ export function defineScopeDO(modules, bareOps) {
         async systemGrantsStatus() {
             return systemGrantsStatus(this.switchSql(), new Date().toISOString());
         }
+        // -- the rewind hold (#1819) ------------------------------------------------
+        // Two halves. On EVERY scope instance: its identity, and whether it has armed a rewind, so
+        // the host can tell storage the rewind will discard from storage it restored. On ONE object
+        // per deployment, the one named `SWITCH_HOLDS_NAME` and never a scope's own: the claims
+        // themselves, because a rewind replaces the rewound scope's whole storage. The claims table
+        // is created on first use, so no scope ever carries it.
+        /** This instance, and nothing before or after it: a restart is a new id. */
+        instanceId = crypto.randomUUID();
+        /** Whether this instance armed a rewind: then its writes are discarded at the restart. */
+        rewindArmed = false;
+        /** Whether this instance began arming one: set before the restore call, and never cleared. */
+        rewindArming = false;
+        /**
+         * Which instance is serving, and whether it armed a rewind or began to. For an ambiguous
+         * rewind throw: arming counts as armed, because the restore call may yet land.
+         */
+        rewindProbe() {
+            return { instance: this.instanceId, armed: this.rewindArmed || this.rewindArming };
+        }
+        /** Whether this instance has created the claims table; it never goes away once it exists. */
+        switchHoldsReady = false;
+        switchHoldsTable() {
+            if (this.switchHoldsReady)
+                return;
+            this.sql.exec(`CREATE TABLE IF NOT EXISTS _substrat_switch_holds (
+           scope_id TEXT NOT NULL,
+           module_id TEXT NOT NULL,
+           claim_id TEXT NOT NULL,
+           state TEXT NOT NULL,
+           doomed TEXT,
+           held_at TEXT NOT NULL,
+           PRIMARY KEY (scope_id, module_id, claim_id)
+         )`);
+            // #1839: the ON tombstones, and the one counter that orders them against a rewind's reads.
+            // This object's storage is never rewound, so the counter only ever goes up.
+            this.sql.exec(`CREATE TABLE IF NOT EXISTS _substrat_switch_hold_ons (
+           scope_id TEXT NOT NULL,
+           module_id TEXT NOT NULL,
+           seq INTEGER NOT NULL,
+           PRIMARY KEY (scope_id, module_id)
+         )`);
+            this.sql.exec(`CREATE TABLE IF NOT EXISTS _substrat_switch_hold_seq (
+           id INTEGER PRIMARY KEY CHECK (id = 1),
+           seq INTEGER NOT NULL
+         )`);
+            this.switchHoldsReady = true;
+        }
+        /**
+         * #1839: where the ON counter stands now. A rewind reads this BEFORE it reads the scope's status,
+         * and hands it to `switchHoldClaim`: an ON tombstoned after it may have moved after that read.
+         */
+        switchHoldToken() {
+            this.switchHoldsTable();
+            const row = this.sql.exec('SELECT seq FROM _substrat_switch_hold_seq WHERE id = 1').toArray()[0];
+            return row?.seq ?? 0;
+        }
+        /**
+         * #1839: an operator's ON has moved: release the claims it read before the move (S0) on this
+         * module, and tombstone the module with the next count, in one transaction. Called only AFTER the
+         * move, so a rewind whose token predates the tombstone may have read the switch before the ON;
+         * `switchHoldClaim` then refuses that stale read's row. A row it inserted before this ran is
+         * either one of the S0 claims (released here) or a claim created after S0, which is left alone.
+         */
+        switchHoldOn(scopeId, moduleId, claimIds) {
+            this.switchHoldsTable();
+            this.ctx.storage.transactionSync(() => {
+                const seq = this.sql
+                    .exec(`INSERT INTO _substrat_switch_hold_seq (id, seq) VALUES (1, 1)
+               ON CONFLICT (id) DO UPDATE SET seq = seq + 1 RETURNING seq`)
+                    .toArray()[0].seq;
+                this.sql.exec(`INSERT INTO _substrat_switch_hold_ons (scope_id, module_id, seq) VALUES (?, ?, ?)
+           ON CONFLICT (scope_id, module_id) DO UPDATE SET seq = excluded.seq`, scopeId, moduleId, seq);
+                for (const claimId of claimIds) {
+                    this.sql.exec('DELETE FROM _substrat_switch_holds WHERE scope_id = ? AND module_id = ? AND claim_id = ?', scopeId, moduleId, claimId);
+                }
+            });
+        }
+        /**
+         * One rewind's claim on these modules, `pending` until the rewind has armed. Only the rewind
+         * that owns the claim calls this, and only before it arms. Each new row is stamped with THIS
+         * object's clock, and `switchHoldYoungestMs` measures against the same clock (#1839). A row
+         * already there keeps its stamp.
+         *
+         * `token` is `switchHoldToken` as the rewind read it before the status read these modules came
+         * from. A module tombstoned by an ON after that is skipped: the read may predate the ON, and
+         * the ON is the operator's newer word (#1839).
+         */
+        switchHoldClaim(scopeId, moduleIds, claimId, token) {
+            this.switchHoldsTable();
+            const at = new Date().toISOString();
+            this.ctx.storage.transactionSync(() => {
+                for (const moduleId of moduleIds) {
+                    this.sql.exec(`INSERT OR IGNORE INTO _substrat_switch_holds (scope_id, module_id, claim_id, state, doomed, held_at)
+             SELECT ?, ?, ?, 'pending', NULL, ?
+              WHERE NOT EXISTS (
+                SELECT 1 FROM _substrat_switch_hold_ons WHERE scope_id = ? AND module_id = ? AND seq > ?
+              )`, scopeId, moduleId, claimId, at, scopeId, moduleId, token);
+                }
+            });
+        }
+        /**
+         * #1839: an OFF joins other rewinds' claims. The row takes its claim's state and doomed instance
+         * from the claim's own rows, in this transaction, so it can never be staler than the claim; it
+         * is stamped with this object's clock. A claim with no rows left is not joined: this cannot tell
+         * a claim its rewind dropped (a refusal) from one an ON emptied, and a row created for a dropped
+         * one would hold the module with no rewind behind it.
+         */
+        switchHoldJoin(scopeId, moduleId, claimIds) {
+            this.switchHoldsTable();
+            const at = new Date().toISOString();
+            this.ctx.storage.transactionSync(() => {
+                for (const claimId of claimIds) {
+                    this.sql.exec(`INSERT OR IGNORE INTO _substrat_switch_holds (scope_id, module_id, claim_id, state, doomed, held_at)
+             SELECT scope_id, ?, claim_id, state, doomed, ? FROM _substrat_switch_holds
+              WHERE scope_id = ? AND claim_id = ? LIMIT 1`, moduleId, at, scopeId, claimId);
+                }
+            });
+        }
+        /**
+         * #1839: how long ago, on this object's clock, the youngest row of one claim was stamped; null
+         * once the claim has no rows. The rewind waits on this, so its age and the stamp are read on
+         * one clock and no two clocks are ever compared.
+         */
+        switchHoldYoungestMs(scopeId, claimId) {
+            this.switchHoldsTable();
+            const row = this.sql
+                .exec('SELECT MAX(held_at) AS youngest FROM _substrat_switch_holds WHERE scope_id = ? AND claim_id = ?', scopeId, claimId)
+                .toArray()[0];
+            return row.youngest ? Date.now() - Date.parse(row.youngest) : null;
+        }
+        /** The rewind armed (or may have): its claim is `armed`, naming the instance it doomed. */
+        switchHoldArm(scopeId, claimId, doomed) {
+            this.switchHoldsTable();
+            this.sql.exec(`UPDATE _substrat_switch_holds SET state = 'armed', doomed = ? WHERE scope_id = ? AND claim_id = ?`, doomed, scopeId, claimId);
+        }
+        /** A definite refusal: this rewind's own claim goes, and no other. */
+        switchHoldDrop(scopeId, claimId) {
+            this.switchHoldsTable();
+            this.sql.exec('DELETE FROM _substrat_switch_holds WHERE scope_id = ? AND claim_id = ?', scopeId, claimId);
+        }
+        /** Every held (scope, module) in the deployment, one read for a whole sweep pass. */
+        switchHoldsAll() {
+            this.switchHoldsTable();
+            return this.sql
+                .exec('SELECT DISTINCT scope_id, module_id FROM _substrat_switch_holds')
+                .toArray()
+                .map((r) => ({ scopeId: r.scope_id, moduleId: r.module_id }));
+        }
+        /**
+         * Every claim row on one scope, for a switch move to read before it moves: the release rule
+         * reads the module's own rows, and an OFF (#1839) the scope's claims it would join.
+         */
+        switchHoldClaims(scopeId) {
+            this.switchHoldsTable();
+            return this.sql
+                .exec('SELECT claim_id, module_id, state, doomed, held_at FROM _substrat_switch_holds WHERE scope_id = ?', scopeId)
+                .toArray()
+                .map((r) => ({
+                claimId: r.claim_id,
+                moduleId: r.module_id,
+                state: r.state,
+                doomed: r.doomed ?? null,
+                heldAt: r.held_at,
+            }));
+        }
+        /** Release these claims on one module, or every claim on the scope when `claimIds` is null. */
+        switchHoldRelease(scopeId, moduleId, claimIds) {
+            this.switchHoldsTable();
+            this.ctx.storage.transactionSync(() => {
+                if (moduleId === null || claimIds === null) {
+                    this.sql.exec('DELETE FROM _substrat_switch_holds WHERE scope_id = ?', scopeId);
+                    this.sql.exec('DELETE FROM _substrat_switch_hold_ons WHERE scope_id = ?', scopeId);
+                    return;
+                }
+                for (const claimId of claimIds) {
+                    this.sql.exec('DELETE FROM _substrat_switch_holds WHERE scope_id = ? AND module_id = ? AND claim_id = ?', scopeId, moduleId, claimId);
+                }
+            });
+        }
         /**
          * Where every peer this scope holds or has held grants for stands (#1706) — the
          * kernel's `peerGrantsStatus`, over this DO's own storage, and the same plain
@@ -2429,12 +2681,17 @@ export function defineScopeDO(modules, bareOps) {
          * `kind` is LAST because this is a positional RPC — see the interface's own note
          * in `host.ts`. An old DO drops a trailing argument; a leading one it binds, and
          * every value after it lands one column to the left, silently.
+         *
+         * `invocationId` (#1525) is appended after `kind` for the same reason and defaulted
+         * to null, so a coordinator that sends none — every freshness verdict, and every
+         * caller predating this argument — records the honest "no call was carried".
          */
-        async recordScheduleRun(unit, at, status, kind) {
-            this.sql.exec(`INSERT INTO _substrat_schedule_state (kind, schedule_op, last_run_at, last_status)
-           VALUES (?, ?, ?, ?)
+        async recordScheduleRun(unit, at, status, kind, invocationId) {
+            this.sql.exec(`INSERT INTO _substrat_schedule_state (kind, schedule_op, last_run_at, last_status, invocation_id)
+           VALUES (?, ?, ?, ?, ?)
          ON CONFLICT(kind, schedule_op) DO UPDATE SET last_run_at = excluded.last_run_at,
-                                                      last_status = excluded.last_status`, kind, unit, at, status);
+                                                      last_status = excluded.last_status,
+                                                      invocation_id = excluded.invocation_id`, kind, unit, at, status, invocationId ?? null);
         }
         // -- the resumable-run driver's store (#1577) -----------------------------
         //
@@ -3092,6 +3349,11 @@ export function defineScopeDO(modules, bareOps) {
                 // #1237: the invocation column on a DO created before it. Nullable, and the
                 // null is honestly "none was carried" for every legacy row.
                 'ALTER TABLE _substrat_outbox ADD COLUMN invocation_id TEXT',
+                // #1525: the invocation a fired schedule ran in, on a scope DO created before
+                // the column. Nullable, and the null is honestly "no call was carried" for
+                // every legacy row — a past run's id cannot be recovered afterwards, exactly
+                // as the other #1525 columns above argued.
+                'ALTER TABLE _substrat_schedule_state ADD COLUMN invocation_id TEXT',
             ]) {
                 try {
                     this.sql.exec(alter);
@@ -3142,7 +3404,11 @@ export function defineScopeDO(modules, bareOps) {
                     this.sql.exec(stmt);
             });
         }
-        async importDump(tables, destScopeId) {
+        async importDump(tables, destScopeId, 
+        /** The directory's recorded-off modules (#1742), switched off on `destScopeId` right after
+         *  the replay re-points the grants, in the same event: a dump from before the switch was
+         *  pulled carries the grants live and no marker. Needs `destScopeId`, the scope restored. */
+        switchOff) {
             // The WHOLE drop-then-replay runs under deferred foreign keys, in one transaction.
             //
             // Two distinct FK hazards, and both need the deferral — the second is why it cannot
@@ -3161,7 +3427,17 @@ export function defineScopeDO(modules, bareOps) {
             // Deferral holds every check until commit, by which point the old rows are gone and
             // the new ones are all in. It also covers what a topological sort cannot express: FK
             // cycles, and self-referencing rows within a single table.
-            await this.ctx.storage.transaction(async () => {
+            // A dump written before indexes were excluded may still carry them; skipped
+            // rather than failing a restore over data about to be recomputed.
+            const replayable = tables.filter((t) => !isSearchIndexTable(t.name));
+            // The dump is untrusted input (#1143). `SqlStorage.exec` runs every statement
+            // in the string it is given, so a `ddl` with anything appended to its CREATE
+            // TABLE executed that too — with entirely plain identifiers, which is why no
+            // amount of name checking reaches it. There is no prepare step here to compile
+            // only the first statement, so the text itself has to be the one statement.
+            // Pure input validation, so it runs before the first DROP: a refused dump touches nothing.
+            assertReplayableDump(replayable, { maxColumns: DO_SQL_LIMITS.columns });
+            const switched = await this.ctx.storage.transaction(async () => {
                 this.sql.exec('PRAGMA defer_foreign_keys = ON');
                 // Real tables only; `sqlite_*` internals are auto-managed and un-droppable.
                 const existing = this.sql
@@ -3175,15 +3451,6 @@ export function defineScopeDO(modules, bareOps) {
                         continue;
                     this.sql.exec(`DROP TABLE IF EXISTS "${name}"`);
                 }
-                // A dump written before indexes were excluded may still carry them; skipped
-                // rather than failing a restore over data about to be recomputed.
-                const replayable = tables.filter((t) => !isSearchIndexTable(t.name));
-                // The dump is untrusted input (#1143). `SqlStorage.exec` runs every statement
-                // in the string it is given, so a `ddl` with anything appended to its CREATE
-                // TABLE executed that too — with entirely plain identifiers, which is why no
-                // amount of name checking reaches it. There is no prepare step here to compile
-                // only the first statement, so the text itself has to be the one statement.
-                assertReplayableDump(replayable);
                 for (const t of replayable)
                     this.sql.exec(t.ddl);
                 for (const t of replayable) {
@@ -3195,25 +3462,39 @@ export function defineScopeDO(modules, bareOps) {
                     for (const row of t.rows)
                         this.sql.exec(insert, ...row);
                 }
+                // Re-assert the kernel spine (#321). A dump captured from a WORLD that stores some
+                // `_substrat_*` tables ELSEWHERE carries only a subset — an `@substrat-run/adapter-
+                // sqlite` scope file, for instance, keeps `_substrat_roles`/`_substrat_tenant_tuples`
+                // in its DIRECTORY database, so its per-scope dump omits them. Replaying such a dump
+                // verbatim would leave this DO missing those spine tables, and the very next
+                // permission check would raise a bare `no such table: _substrat_roles`. KERNEL_DDL is
+                // all IF NOT EXISTS, so this recreates only what the dump did not carry (empty), and
+                // never disturbs a table the dump DID bring. Roles land empty here and are re-projected
+                // by the restore's repair leg (host.projectRolesLocal) — the spine's job is only to
+                // exist so the checker can read it.
+                for (const stmt of splitSqlStatements(KERNEL_DDL))
+                    this.sql.exec(stmt);
+                // …and the additive columns, for the same reason KERNEL_DDL is re-asserted: a
+                // dump captured before a column existed replays DDL WITHOUT it, and IF NOT
+                // EXISTS cannot widen a table the dump did bring. Without this, the next emit
+                // in this instance fails with `no such column` until a cold start re-runs the
+                // constructor's pass.
+                this.applySpineColumnAdditions();
+                // #1738: a dump's `provisioned_for` names the scope (and tenant) it was captured from,
+                // not this one. Dropped, so a restore never carries a receipt over; the repair
+                // projection that follows writes this scope's own.
+                this.sql.exec(`DELETE FROM _substrat_meta WHERE key = 'provisioned_for'`);
+                // Re-point the restored grants at THIS scope (after the spine exists, so a dump
+                // that carried no tuples table still finds one here).
+                if (destScopeId)
+                    this.rewriteScopeTuples(destScopeId);
+                // #1742: the recorded-off modules go back off INSIDE the replay's transaction, with
+                // the spine and the re-point. A switch that throws rolls the whole restore back, so
+                // the dump's grants never commit live without the switch that should cover them.
+                return destScopeId && switchOff
+                    ? switchRecordedOff(this.switchSql(), { scopeId: destScopeId, ...switchOff })
+                    : [];
             });
-            // Re-assert the kernel spine (#321). A dump captured from a WORLD that stores some
-            // `_substrat_*` tables ELSEWHERE carries only a subset — an `@substrat-run/adapter-
-            // sqlite` scope file, for instance, keeps `_substrat_roles`/`_substrat_tenant_tuples`
-            // in its DIRECTORY database, so its per-scope dump omits them. Replaying such a dump
-            // verbatim would leave this DO missing those spine tables, and the very next
-            // permission check would raise a bare `no such table: _substrat_roles`. KERNEL_DDL is
-            // all IF NOT EXISTS, so this recreates only what the dump did not carry (empty), and
-            // never disturbs a table the dump DID bring. Roles land empty here and are re-projected
-            // by the restore's repair leg (host.projectRolesLocal) — the spine's job is only to
-            // exist so the checker can read it.
-            for (const stmt of splitSqlStatements(KERNEL_DDL))
-                this.sql.exec(stmt);
-            // …and the additive columns, for the same reason KERNEL_DDL is re-asserted: a
-            // dump captured before a column existed replays DDL WITHOUT it, and IF NOT
-            // EXISTS cannot widen a table the dump did bring. Without this, the next emit
-            // in this instance fails with `no such column` until a cold start re-runs the
-            // constructor's pass.
-            this.applySpineColumnAdditions();
             // Rebuild the derived search indexes over the rows just loaded (#827). Drop-then-
             // create, so it also repairs an index a dump left stale, and the triggers it
             // recreates are what keep the restored scope in step from here. Skipped for a plan
@@ -3228,10 +3509,6 @@ export function defineScopeDO(modules, bareOps) {
                 for (const stmt of splitSqlStatements(searchIndexDdl(plan)))
                     this.sql.exec(stmt);
             }
-            // Re-point the restored grants at THIS scope (after the spine exists, so a dump
-            // that carried no tuples table still finds one here).
-            if (destScopeId)
-                this.rewriteScopeTuples(destScopeId);
             // The frontier arrived with the dump — refresh the in-memory applied set so a
             // later migrate() builds on the imported state, not the provisioning state.
             this.applied.clear();
@@ -3263,6 +3540,7 @@ export function defineScopeDO(modules, bareOps) {
             // is complete applies nothing, so there is nothing to report either way.
             this.migrationPromise = undefined;
             this.lastFailure = null;
+            return switched;
         }
         /**
          * Re-point scope-level tuples at the scope they now live in.
@@ -3666,6 +3944,19 @@ export function defineScopeDO(modules, bareOps) {
                 principal: capabilityId ? capabilityId : principal,
                 sql: guardSecrets(doScopedSql(sql), minted),
                 now: () => at,
+                // #1746/#1747: the host stamps who and where; module code supplies only the template
+                // and its fields. A consumer runs under the system override, so it logs as `system`.
+                // The invocation id is read per call, as the DO holds it for the queued body's length.
+                log: moduleLog({
+                    tenantId,
+                    scopeId,
+                    operation: operation ?? null,
+                    invocationId: () => this.invocationId,
+                    principalKind: systemActor ? 'system' : subject.kind,
+                    // The string-safe redaction: `redactSecrets` parses its serialization back, and a
+                    // log's text is not JSON.
+                    redact: (text) => redactSecretText(text, minted),
+                }),
                 emit: (event) => {
                     assertImpersonationWrites(impersonation, 'ctx.emit');
                     const parsed = domainEventInput.parse(event);
@@ -3797,6 +4088,7 @@ export function defineScopeDO(modules, bareOps) {
                  */
                 grant: async (principal, permission, entity) => {
                     assertImpersonationWrites(impersonation, 'ctx.grant');
+                    entityObjectRef(entity, 'ctx.grant'); // #1856: a tuple the walk can read back
                     const held = await runCheck(permission, entity);
                     if (!held.allowed) {
                         throw new PermissionDenied(`cannot grant '${permission}' on ${entity.entityType}:${entity.entityId} — ` +
@@ -3804,6 +4096,11 @@ export function defineScopeDO(modules, bareOps) {
                     }
                     sql.exec(`INSERT OR IGNORE INTO _substrat_tuples (subject, relation, object) VALUES (?, ?, ?)`, `principal:${principal}`, `granted:${permission}`, `${entity.entityType}:${entity.entityId}`);
                 },
+                /**
+                 * Deliberately NOT the #1856 grammar check `grant` and `link` make: a revoke writes
+                 * nothing the walk must read back, and a grant stored before that check existed must
+                 * stay removable.
+                 */
                 revoke: async (principal, permission, entity) => {
                     assertImpersonationWrites(impersonation, 'ctx.revoke');
                     const held = await runCheck(permission, entity);
@@ -3832,6 +4129,8 @@ export function defineScopeDO(modules, bareOps) {
                 }),
                 link: (child, parent) => {
                     assertImpersonationWrites(impersonation, 'ctx.link');
+                    entityObjectRef(child, 'ctx.link'); // #1856: an edge the walk can read back
+                    entityObjectRef(parent, 'ctx.link');
                     const allowed = relations.get(child.entityType);
                     if (!allowed?.has(parent.entityType)) {
                         throw new Error(`undeclared entity relation: ${child.entityType} → ${parent.entityType} ` +
@@ -3989,32 +4288,50 @@ export function defineScopeDO(modules, bareOps) {
          *  (a role-only re-projection like the restore repair) leaves projected keys
          *  untouched; passing a list — even `[]` — full-replaces them, which is how a
          *  revoked connection stops being sealable to. */
-        connectionKeys) {
+        connectionKeys, 
+        /** The directory's recorded-off modules (#1742), switched off in THIS unit right after the
+         *  seat, so no sweep can run the grants the seat just re-created. `scopeId` is the scope
+         *  this projection provisions (the object of `scopeTuples`), never another. */
+        switchOff) {
             if (await this.isReaped())
-                return;
+                return [];
             // A projection that arrives after this scope was reaped is dropped, not
             // applied: writing it would recreate storage the platform deliberately
             // destroyed. Silent rather than throwing — the fan-out is a best-effort
             // convergence over many scopes and one dead sibling must not fail the others,
             // and a reaped scope converging to "nothing" IS convergence.
-            await this.queue.enqueue(() => {
+            // The whole body is synchronous storage work, so it is ONE transaction (#1738): the queue
+            // serializes but does not roll back, and a role insert that throws part-way must not leave
+            // the receipt (or a half-replaced role set) behind for `servesTenant` to trust.
+            return this.queue.enqueue(() => this.ctx.storage.transactionSync(() => {
+                // #1738: the receipt `servesTenant` reads. First writer wins: absent or equal is written,
+                // a receipt for ANOTHER tenant refuses the whole projection (K-3), before a single row
+                // moves, so a misdirected projection can never re-point a scope or leave its roles behind.
+                // A restore does not trip this: `importDump` drops the dump's receipt, so the repair
+                // projection that follows finds none and writes the destination's own.
+                const held = this.sql.exec(`SELECT value FROM _substrat_meta WHERE key = 'provisioned_for'`).toArray()[0];
+                if (held && held.value !== tenantId) {
+                    throw substratError('conflict', `applyProjection refused: this scope was provisioned for tenant ${held.value}, and a projection for tenant ${tenantId} would re-point it`);
+                }
+                // Written before any guard below can return early, so every projection leaves it.
+                this.sql.exec(`INSERT OR REPLACE INTO _substrat_meta (key, value) VALUES ('provisioned_for', ?)`, tenantId);
                 this.sql.exec(`DELETE FROM _substrat_roles WHERE tenant_id = ?`, tenantId);
                 for (const r of roles) {
                     this.sql.exec(`INSERT OR REPLACE INTO _substrat_roles (tenant_id, role_key, permissions, source, revoked_at)
-             VALUES (?, ?, ?, ?, NULL)`, tenantId, r.role_key, r.permissions, r.source);
+               VALUES (?, ?, ?, ?, NULL)`, tenantId, r.role_key, r.permissions, r.source);
                 }
                 this.sql.exec(`DELETE FROM _substrat_tenant_tuples WHERE tenant_id = ?`, tenantId);
                 for (const t of tuples) {
                     this.sql.exec(`INSERT OR REPLACE INTO _substrat_tenant_tuples
-               (tenant_id, subject, relation, object, expires_at, revoked_at)
-             VALUES (?, ?, ?, ?, ?, ?)`, tenantId, t.subject, t.relation, t.object, t.expires_at, t.revoked_at);
+                 (tenant_id, subject, relation, object, expires_at, revoked_at)
+               VALUES (?, ?, ?, ?, ?, ?)`, tenantId, t.subject, t.relation, t.object, t.expires_at, t.revoked_at);
                 }
                 if (entitlements !== undefined) {
                     this.sql.exec(`DELETE FROM _substrat_entitlements WHERE tenant_id = ?`, tenantId);
                     for (const e of entitlements) {
                         this.sql.exec(`INSERT OR REPLACE INTO _substrat_entitlements
-                 (tenant_id, entitlement_key, expires_at, quota, plan)
-               VALUES (?, ?, ?, ?, ?)`, tenantId, e.entitlement_key, e.expires_at, e.quota, e.plan);
+                   (tenant_id, entitlement_key, expires_at, quota, plan)
+                 VALUES (?, ?, ?, ?, ?)`, tenantId, e.entitlement_key, e.expires_at, e.quota, e.plan);
                     }
                     // #304: once a scope has been projected WITH entitlements even once, its gate
                     // switches from trust-upstream to strict fail-closed (a missing/expired key
@@ -4030,8 +4347,8 @@ export function defineScopeDO(modules, bareOps) {
                     this.sql.exec(`DELETE FROM _substrat_identity_links WHERE tenant_id = ?`, tenantId);
                     for (const i of identities) {
                         this.sql.exec(`INSERT OR REPLACE INTO _substrat_identity_links
-                 (tenant_id, provider, external_id, principal_id, scope_id)
-               VALUES (?, ?, ?, ?, ?)`, tenantId, i.provider, i.external_id, i.principal_id, i.scope_id);
+                   (tenant_id, provider, external_id, principal_id, scope_id)
+                 VALUES (?, ?, ?, ?, ?)`, tenantId, i.provider, i.external_id, i.principal_id, i.scope_id);
                     }
                 }
                 // #687: connection sealing keys ride the same snapshot. Full replace, so a
@@ -4041,8 +4358,8 @@ export function defineScopeDO(modules, bareOps) {
                     this.sql.exec(`DELETE FROM _substrat_connection_keys WHERE tenant_id = ?`, tenantId);
                     for (const k of connectionKeys) {
                         this.sql.exec(`INSERT OR REPLACE INTO _substrat_connection_keys
-                 (tenant_id, connection_id, provider, key_id, public_key)
-               VALUES (?, ?, ?, ?, ?)`, tenantId, k.connection_id, k.provider, k.key_id, k.public_key);
+                   (tenant_id, connection_id, provider, key_id, public_key)
+                 VALUES (?, ?, ?, ?, ?)`, tenantId, k.connection_id, k.provider, k.key_id, k.public_key);
                     }
                 }
                 // #332: scope-level grants (the owner's role tuple at provision) are written in the
@@ -4072,9 +4389,14 @@ export function defineScopeDO(modules, bareOps) {
                         if (!st.lockout_reseat)
                             continue;
                         this.sql.exec(`INSERT OR REPLACE INTO _substrat_tuples (subject, relation, object, expires_at, revoked_at)
-               VALUES (?, ?, ?, ?, NULL)`, st.subject, st.relation, st.object, st.expires_at);
+                 VALUES (?, ?, ?, ?, NULL)`, st.subject, st.relation, st.object, st.expires_at);
                     }
                 }
+                // #1742: the directory's recorded-off modules go back off HERE, after the seat and in
+                // its unit. A wiped scope's seat has just re-created their `system:` grants (#1659), and
+                // a re-assert arriving later from the control plane left a window in which this
+                // deployment's own sweeper could fire them. Nothing runs between these two lines.
+                const switched = switchOff ? switchRecordedOff(this.switchSql(), switchOff) : [];
                 // #332: only switch on strict local enforcement when SOMEONE actually holds a role.
                 // A projection that leaves role definitions but no effective principal→role grant would
                 // make every check fail closed — a scope serving nothing but denials, unfixable from
@@ -4083,9 +4405,10 @@ export function defineScopeDO(modules, bareOps) {
                 // regardless of this flag, so the owner grant is written above in the same unit — this
                 // guard is the belt to that suspenders, and it protects the CP-backed flip outright.)
                 if (roles.length > 0 && !this.hasEffectiveRoleGrant(tenantId))
-                    return;
+                    return switched;
                 this.sql.exec(`INSERT OR REPLACE INTO _substrat_meta (key, value) VALUES ('permission_source', 'local')`);
-            });
+                return switched;
+            }));
         }
         /**
          * Resolve an external identity from this scope's projected links (#406) — the