@substrat-run/adapter-cloudflare 0.137.0 → 0.138.0

package/dist/control-plane-do.js

@@ -1,8 +1,9 @@
 import { DurableObject } from 'cloudflare:workers';
-import { IMPERSONATION_COLUMNS, IMPERSONATION_DDL, impersonationByIdQuery, impersonationListQuery, impersonationRowValues, ISSUE_RETENTION_DAYS, telemetryRetentionStatements, assertRowLimit, OPS_FAILURE_RETENTION_DAYS, SWEEP_RUN_RETENTION_DAYS, SWEEP_RUNS_INTENT_INDEX, sweepRunsIntentHasKind, SYSTEM_SWITCHES_BACKFILL_SQL, SYSTEM_SWITCHES_TABLE, SYSTEM_SWITCHES_DDL, forgetSystemSwitchesOf, listSystemSwitchRecords, recordSystemSwitchedOff, recordSystemSwitchedOn, restoreSystemSwitchRecord, scopesSwitchedOffFor, tenantHoldsSystemGrant, switchedOffModulesOf, systemSwitchRecordsOf, systemSwitchesTableExists, VERSION_MIGRATIONS_DDL, splitVersionMigrationsBatch, versionMigrationsOf, versionsAwaitSplit, writeVersionMigrations, MODEL_USAGE_RETENTION_DAYS, DO_SQL_LIMITS, ulid, isPrimaryScopeRow, resolveVerticalInstanceFrom, LEGACY_SCOPE_ROWS_BACKFILL, loadDirectoryDump, redactSubjectDirectoryText, ISSUE_EXEMPLAR_OWNER_BACKFILL_SQL, issueExemplarOwner, } from '@substrat-run/kernel';
+import { membershipFencesBackfillSql, MEMBERSHIP_FENCES_DDL, MEMBERSHIP_FENCES_TABLE, membershipFencesTableExists, MEMBERSHIP_FENCE_SINCE_SQL, RAISE_MEMBERSHIP_FENCE_SQL, tenantCoverage, IMPERSONATION_COLUMNS, IMPERSONATION_DDL, impersonationByIdQuery, impersonationListQuery, impersonationRowValues, ISSUE_RETENTION_DAYS, telemetryRetentionStatements, assertRowLimit, OPS_FAILURE_RETENTION_DAYS, SWEEP_RUN_RETENTION_DAYS, SWEEP_RUNS_INTENT_INDEX, sweepRunsIntentHasKind, PEER_SWITCHES_DDL, SWITCH_OWED_DDL, clearSwitchOwed, markSwitchOwed, scopeOwesSwitch, switchesOwedOf, SWITCH_KINDS, SYSTEM_SWITCHES_DDL, forgetSwitchesOf, listSystemSwitchRecords, recordSwitchedOff, recordSwitchedOn, recordWriteSuperseded, restoreSwitchRecord, scopesSwitchedOffFor, switchRecordStatesOf, switchRecordsOf, switchesBackfillSqlOf, switchesDdlOf, switchesTableExists, switchesTableOf, tenantHeldOf, VERSION_MIGRATIONS_DDL, splitVersionMigrationsBatch, versionMigrationsOf, versionsAwaitSplit, writeVersionMigrations, MODEL_USAGE_RETENTION_DAYS, DO_SQL_LIMITS, ulid, isPrimaryScopeRow, resolveVerticalInstanceFrom, LEGACY_SCOPE_ROWS_BACKFILL, loadDirectoryDump, redactSubjectDirectoryText, ISSUE_EXEMPLAR_OWNER_BACKFILL_SQL, issueExemplarOwner, } from '@substrat-run/kernel';
+import { replyOf } from './do-reply.js';
 import { splitSqlStatements, switchSqlOver } from './scope-do.js';
 import { doBuiltColumnsOf, doRedactionSql } from './sql.js';
-import { assertReplayableDump, opsFailureFingerprint, ROUTE_SCOPE_HOSTNAMES_MAX, substratError } from '@substrat-run/contracts';
+import { assertReplayableDump, opsFailureFingerprint, ROUTE_SCOPE_HOSTNAMES_MAX, SCOPE_GATE_REASONS, substratError } from '@substrat-run/contracts';
 /**
  * The router's per-request directory read (`readRoute`), lifted out so its plan can be asked for
  * (#2005): the scope's hostname set is a correlated subquery on the hot path every request takes,
@@ -456,6 +457,9 @@ const DIRECTORY_DDL = `
   );
   ${IMPERSONATION_DDL}
   ${SYSTEM_SWITCHES_DDL}
+  ${MEMBERSHIP_FENCES_DDL}
+  ${PEER_SWITCHES_DDL}
+  ${SWITCH_OWED_DDL}
   CREATE TABLE IF NOT EXISTS _substrat_admin_log (
     id TEXT PRIMARY KEY,
     actor TEXT NOT NULL,
@@ -709,6 +713,19 @@ export const BACKFILL_FAILURES_KEY = 'versionMigrationsBackfillFailures';
 export function backfillBackoffMs(failures) {
     return Math.min(BACKFILL_PAUSE_MS * 2 ** failures, BACKFILL_BACKOFF_MAX_MS);
 }
+/** The directory writes `ControlPlaneDO.reply` answers for (#113): each refuses with a typed throw. */
+export const REPLIED_METHODS = [
+    'createTenant',
+    'setTenantStatus',
+    'reapTenant',
+    'setTenantName',
+    'putTenantStore',
+    'putBlobStore',
+    'provisionScope',
+    'createOrg',
+    'registerIdentityPool',
+    'linkIdentity',
+];
 export class ControlPlaneDO extends DurableObject {
     sql;
     /** The directory's store as the kernel's SQL handle — the switch record's helpers (#1674). */
@@ -792,20 +809,30 @@ export class ControlPlaneDO extends DurableObject {
      * construction. A directory restore runs `buildDirectorySchema` inside its own transaction
      * instead (`importDump`).
      *
-     * The schedule switch's record (#1674) is backfilled from the admin log on a run that
-     * creates its table, and its table is created in the SAME transaction as the backfill
-     * (Copilot review): the gate is "the table does not exist yet", so a table committed ahead
-     * of a backfill that then failed would read as already migrated on every later run. On
+     * Each kill switch's record (#1674, and #2029's for a peer) is backfilled from the admin log
+     * on a run that creates its table, and its table is created in the SAME transaction as the
+     * backfill (Copilot review): the gate is "the table does not exist yet", so a table committed
+     * ahead of a backfill that then failed would read as already migrated on every later run. On
      * such a run its statements are held back from the loop below and run with the backfill.
      */
     applyDirectorySchema() {
-        const switchRecordIsNew = !systemSwitchesTableExists(this.kernelSql);
-        this.buildDirectorySchema({ holdSwitchRecord: switchRecordIsNew });
-        if (switchRecordIsNew) {
+        const newRecords = SWITCH_KINDS.filter((kind) => !switchesTableExists(this.kernelSql, kind));
+        const fencesAreNew = !membershipFencesTableExists(this.kernelSql);
+        this.buildDirectorySchema({ holdSwitchRecords: newRecords, holdFences: fencesAreNew });
+        for (const kind of newRecords) {
+            this.ctx.storage.transactionSync(() => {
+                for (const stmt of splitSqlStatements(switchesDdlOf(kind)))
+                    this.sql.exec(stmt);
+                this.sql.exec(switchesBackfillSqlOf(kind));
+            });
+        }
+        // #1184: the removal fence and its backfill from the admin log, together, the same way.
+        if (fencesAreNew) {
             this.ctx.storage.transactionSync(() => {
-                for (const stmt of splitSqlStatements(SYSTEM_SWITCHES_DDL))
+                for (const stmt of splitSqlStatements(MEMBERSHIP_FENCES_DDL))
                     this.sql.exec(stmt);
-                this.sql.exec(SYSTEM_SWITCHES_BACKFILL_SQL);
+                // Stamped now, inside this unit (`membershipFencesBackfillSql` says why).
+                this.sql.exec(membershipFencesBackfillSql(new Date().toISOString()));
             });
         }
     }
@@ -813,12 +840,15 @@ export class ControlPlaneDO extends DurableObject {
      * The directory's tables as this code builds them, carrying whatever the directory already
      * holds forward to that shape: the construction's pass, and a restore's (#1898, #1912), which
      * runs it inside its transaction onto an emptied directory, before loading any row.
-     * `holdSwitchRecord` leaves the #1674 record's statements to the caller, which creates the
-     * table with its backfill.
+     * `holdSwitchRecords` leaves those kinds' record statements (#1674, #2029) to the caller, which
+     * creates each table with its backfill; `holdFences` does the same for #1184's removal fence.
      */
-    buildDirectorySchema({ holdSwitchRecord }) {
+    buildDirectorySchema({ holdSwitchRecords, holdFences = false }) {
+        const held = holdSwitchRecords.map(switchesTableOf);
         for (const stmt of DIRECTORY_DDL_PLAN.loop) {
-            if (holdSwitchRecord && stmt.includes(SYSTEM_SWITCHES_TABLE))
+            if (held.some((table) => stmt.includes(table)))
+                continue;
+            if (holdFences && stmt.includes(MEMBERSHIP_FENCES_TABLE))
                 continue;
             this.sql.exec(stmt);
         }
@@ -1135,12 +1165,13 @@ export class ControlPlaneDO extends DurableObject {
             // Every table from this code's DDL, onto an empty directory, so none of the pass's
             // rebuilds fires (each would reach `transactionSync`, which this async transaction cannot
             // hold): every table is created on the current shape.
-            this.buildDirectorySchema({ holdSwitchRecord: false });
+            this.buildDirectorySchema({ holdSwitchRecords: [] });
             // Refuses a table this code does not build, adds unknown columns bare, loads the rows by
             // name, then the legacy scope and #1674 backfills, all inside this transaction.
             loadDirectoryDump(tables, {
                 columnsOf: (name) => doBuiltColumnsOf(this.sql, name),
                 exec: (sql) => this.sql.exec(sql),
+                now: () => new Date().toISOString(),
                 maxColumns: DO_SQL_LIMITS.columns,
                 insert: (sql, rows) => {
                     for (const row of rows)
@@ -1199,7 +1230,7 @@ export class ControlPlaneDO extends DurableObject {
             .exec('SELECT tenant_id FROM tenants WHERE slug = ?', slug)
             .toArray()[0];
         if (slugOwner) {
-            throw new Error(`tenant slug '${slug}' already taken by ${slugOwner.tenant_id} (slugs are unique)`);
+            throw substratError('conflict', `tenant slug '${slug}' already taken by ${slugOwner.tenant_id} (slugs are unique)`);
         }
         this.sql.exec(`INSERT INTO tenants (tenant_id, slug, name, status, created_at, provisioned_by_tenant)
        VALUES (?, ?, ?, 'active', ?, ?)`, id, slug, name, createdAt, provisionedByTenant);
@@ -1209,12 +1240,12 @@ export class ControlPlaneDO extends DurableObject {
     setTenantStatus(tenantId, status) {
         const before = this.readTenant(tenantId);
         if (!before)
-            throw new Error(`unknown tenant: ${tenantId}`);
+            throw substratError('not_found', `unknown tenant: ${tenantId}`);
         // `reaped` is terminal and destroys data — unreachable here, only ever set by
         // reapTenant, so a plain status flip cannot forge a tombstone over live data
         // (§4.8, the tenant analogue of reapScope's archived-only gate).
         if (status === 'reaped') {
-            throw new Error(`tenant ${tenantId} cannot be set to 'reaped' via setTenantStatus — reap goes through reapTenant (control-plane.md §4.8)`);
+            throw substratError('validation_failed', `tenant ${tenantId} cannot be set to 'reaped' via setTenantStatus — reap goes through reapTenant (control-plane.md §4.8)`);
         }
         // Stamp/clear deleting_at so the grace-window sweep can age tenants (§4.8):
         // entering `deleting` stamps, leaving it (an un-delete) clears — the same shape
@@ -1241,9 +1272,9 @@ export class ControlPlaneDO extends DurableObject {
     reapTenant(tenantId) {
         const before = this.readTenant(tenantId);
         if (!before)
-            throw new Error(`unknown tenant: ${tenantId}`);
+            throw substratError('not_found', `unknown tenant: ${tenantId}`);
         if (before.status !== 'deleting') {
-            throw new Error(`tenant ${tenantId} is ${before.status}, not deleting — only a deleting tenant may be reaped`);
+            throw substratError('conflict', `tenant ${tenantId} is ${before.status}, not deleting — only a deleting tenant may be reaped`);
         }
         // KEPT as the tombstone: the `tenants` row itself and `_substrat_admin_log` (the
         // compliance witness — never swept). Scope rows were already reaped individually.
@@ -1255,6 +1286,9 @@ export class ControlPlaneDO extends DurableObject {
             '_substrat_entitlements', // per-tenant SKU flags
             'orgs', // K-22 org records
             '_substrat_system_switches', // #1674: the schedule switch's record, per scope
+            '_substrat_membership_fences', // #1184: the latest removal, per principal
+            '_substrat_peer_switches', // #2029: the peer switch's record, per scope
+            '_substrat_switch_owed', // #2045: subjects owed a re-assert, per scope
         ]) {
             this.sql.exec(`DELETE FROM ${table} WHERE tenant_id = ?`, tenantId);
         }
@@ -1270,7 +1304,7 @@ export class ControlPlaneDO extends DurableObject {
     setTenantName(tenantId, name) {
         const before = this.readTenant(tenantId);
         if (!before)
-            throw new Error(`unknown tenant: ${tenantId}`);
+            throw substratError('not_found', `unknown tenant: ${tenantId}`);
         this.sql.exec('UPDATE tenants SET name = ? WHERE tenant_id = ?', name, tenantId);
         return before.name;
     }
@@ -1304,10 +1338,10 @@ export class ControlPlaneDO extends DurableObject {
             .exec('SELECT status FROM tenants WHERE tenant_id = ?', row.tenantId)
             .toArray()[0];
         if (!tenantRow) {
-            throw new Error(`cannot provision tenant store under unknown tenant: ${row.tenantId}`);
+            throw substratError('not_found', `cannot provision tenant store under unknown tenant: ${row.tenantId}`);
         }
         if (tenantRow.status !== 'active') {
-            throw new Error(`cannot provision tenant store under non-active tenant (status: ${tenantRow.status}): ${row.tenantId}`);
+            throw substratError('conflict', `cannot provision tenant store under non-active tenant (status: ${tenantRow.status}): ${row.tenantId}`);
         }
         this.sql.exec(`INSERT OR IGNORE INTO tenant_stores (tenant_id, vertical, binding, kind, ref, created_at)
        VALUES (?, ?, ?, ?, ?, ?)`, row.tenantId, row.vertical, row.binding, row.kind, row.ref, row.createdAt);
@@ -1348,10 +1382,10 @@ export class ControlPlaneDO extends DurableObject {
             .exec('SELECT status FROM tenants WHERE tenant_id = ?', row.tenantId)
             .toArray()[0];
         if (!tenantRow) {
-            throw new Error(`cannot provision blob store under unknown tenant: ${row.tenantId}`);
+            throw substratError('not_found', `cannot provision blob store under unknown tenant: ${row.tenantId}`);
         }
         if (tenantRow.status !== 'active') {
-            throw new Error(`cannot provision blob store under non-active tenant (status: ${tenantRow.status}): ${row.tenantId}`);
+            throw substratError('conflict', `cannot provision blob store under non-active tenant (status: ${tenantRow.status}): ${row.tenantId}`);
         }
         this.sql.exec(`INSERT OR IGNORE INTO blob_stores (tenant_id, vertical, binding, kind, ref, created_at)
        VALUES (?, ?, ?, ?, ?, ?)`, row.tenantId, row.vertical, row.binding, row.kind, row.ref, row.createdAt);
@@ -1388,10 +1422,10 @@ export class ControlPlaneDO extends DurableObject {
             .exec('SELECT status FROM tenants WHERE tenant_id = ?', tenantId)
             .toArray()[0];
         if (!tenantRow) {
-            throw new Error(`cannot provision scope under unknown tenant: ${tenantId}`);
+            throw substratError('conflict', `cannot provision scope under unknown tenant: ${tenantId}`);
         }
         if (tenantRow.status !== 'active') {
-            throw new Error(`cannot provision scope under non-active tenant (status: ${tenantRow.status}): ${tenantId}`);
+            throw substratError('conflict', `cannot provision scope under non-active tenant (status: ${tenantRow.status}): ${tenantId}`);
         }
         // Idempotency is on the scope_id (§3.3: idempotent and journaled — safe to
         // re-run), so an existing scope short-circuits before the slug check:
@@ -1404,7 +1438,7 @@ export class ControlPlaneDO extends DurableObject {
                 .exec("SELECT scope_id FROM scopes WHERE tenant_id = ? AND slug = ? AND status NOT IN ('archived', 'reaped')", tenantId, record.slug)
                 .toArray()[0];
             if (slugOwner) {
-                throw new Error(`scope slug '${record.slug}' already taken under tenant ${tenantId} ` +
+                throw substratError('conflict', `scope slug '${record.slug}' already taken under tenant ${tenantId} ` +
                     `by ${slugOwner.scope_id} (slugs are unique within a tenant)`);
             }
             // A scope born while its vertical serves in place (#286) is born ON the serving
@@ -1486,6 +1520,21 @@ export class ControlPlaneDO extends DurableObject {
             return undefined;
         return row;
     }
+    /**
+     * A directory write, its refusal answered as DATA (#113): the typed throws in these methods
+     * would reach the coordinator flattened, so it calls them through here and rethrows the
+     * refusal with its code (`unwrapReply`). Called directly, they still throw — a coordinator
+     * from before this change does — and that caller reads the code folded into the message.
+     */
+    async reply(method, args) {
+        return replyOf(() => {
+            // An RPC argument, so held to the list rather than trusted to be one of its names.
+            if (!REPLIED_METHODS.includes(method))
+                throw new Error(`not a replied directory method: ${String(method)}`);
+            const run = this[method];
+            return run.apply(this, args);
+        });
+    }
     /**
      * The getScope gate (control-plane.md §4.1/§4.2): validate the scope belongs
      * to the tenant and both records are active, or throw the fail-closed reason.
@@ -1502,11 +1551,10 @@ export class ControlPlaneDO extends DurableObject {
      * The same gate, answering its refusal as DATA (#1718). An error thrown in a Durable
      * Object arrives at the coordinator flattened, its code gone, so a typed throw here
      * would still reach the caller untyped (#1714 measured it). A record crosses intact, so
-     * the coordinator throws the refusal itself, typed. `code` is set only where a code is
-     * known; the lifecycle refusals keep their bare messages. Null means access is allowed.
+     * the coordinator throws the refusal itself, typed (#113: every refusal but a failed
+     * migration carries its code). Null means access is allowed.
      */
     scopeAccessRefusal(tenantId, scopeId) {
-        const refuse = (message) => ({ code: null, message });
         const row = this.sql
             .exec('SELECT tenant_id, status FROM scopes WHERE scope_id = ?', scopeId)
             .toArray()[0];
@@ -1517,10 +1565,14 @@ export class ControlPlaneDO extends DurableObject {
             .exec('SELECT status FROM tenants WHERE tenant_id = ?', tenantId)
             .toArray()[0];
         if (!tenantRow) {
-            return refuse(`scope has no tenant record: (${tenantId}, ${scopeId})`);
+            return { code: 'not_found', message: `scope has no tenant record: (${tenantId}, ${scopeId})`, reason: SCOPE_GATE_REASONS.unrecorded };
         }
         if (tenantRow.status !== 'active') {
-            return refuse(`tenant not active (status: ${tenantRow.status}): ${tenantId}`);
+            return {
+                code: 'conflict',
+                message: `tenant not active (status: ${tenantRow.status}): ${tenantId}`,
+                reason: SCOPE_GATE_REASONS.notActive,
+            };
         }
         if (row.status !== 'active') {
             // A scope stuck in provisioning because its migrations failed must say so.
@@ -1531,10 +1583,17 @@ export class ControlPlaneDO extends DurableObject {
                 .exec('SELECT migration_failed_version, migration_error FROM scopes WHERE scope_id = ?', scopeId)
                 .toArray()[0];
             if (failure?.migration_failed_version) {
-                return refuse(`migration failed for ${failure.migration_failed_version} — scope fails closed: ` +
-                    `${failure.migration_error ?? 'unknown error'}`);
+                return {
+                    code: null,
+                    message: `migration failed for ${failure.migration_failed_version} — scope fails closed: ` +
+                        `${failure.migration_error ?? 'unknown error'}`,
+                };
             }
-            return refuse(`scope not active (status: ${row.status}): ${scopeId}`);
+            return {
+                code: 'conflict',
+                message: `scope not active (status: ${row.status}): ${scopeId}`,
+                reason: SCOPE_GATE_REASONS.notActive,
+            };
         }
         return null;
     }
@@ -1569,7 +1628,7 @@ export class ControlPlaneDO extends DurableObject {
         if (!from.includes(row.status)) {
             return {
                 ok: false,
-                code: null,
+                code: 'conflict',
                 message: `illegal scope transition for ${action}: ${row.status} → ${to} ` +
                     `(allowed from: ${from.join('|')})`,
             };
@@ -1590,7 +1649,7 @@ export class ControlPlaneDO extends DurableObject {
                 this.sql.exec('UPDATE scopes SET status = ? WHERE scope_id = ?', to, scopeId);
             }
             if (to === 'reaped') {
-                forgetSystemSwitchesOf(this.kernelSql, scopeId);
+                forgetSwitchesOf(this.kernelSql, scopeId);
                 this.forgetLifecycleDeliveries('scope_id = ?', scopeId);
             }
             this.bumpLifecycleRevision(`scope:${scopeId}`); // #1713, with the status it counts
@@ -1998,6 +2057,11 @@ export class ControlPlaneDO extends DurableObject {
      * for a repair that failed, which is the one outcome that must keep being retried.
      */
     markScopeProvisioned(scopeId, versionId) {
+        // #2045: a scope owed a switch re-assert keeps no receipt, so the sweep reconciles it again
+        // until its re-assert has settled every mark — including a mark written while this
+        // reconcile ran, after its re-assert had read none.
+        if (versionId !== null && scopeOwesSwitch(this.kernelSql, scopeId))
+            versionId = null;
         this.sql.exec('UPDATE scopes SET provisioned_version_id = ? WHERE scope_id = ?', versionId, scopeId);
     }
     /**
@@ -2008,7 +2072,7 @@ export class ControlPlaneDO extends DurableObject {
     deleteScopeDirectory(scopeId) {
         this.ctx.storage.transactionSync(() => {
             this.sql.exec('DELETE FROM hostnames WHERE scope_id = ?', scopeId);
-            forgetSystemSwitchesOf(this.kernelSql, scopeId);
+            forgetSwitchesOf(this.kernelSql, scopeId);
             this.forgetLifecycleDeliveries('scope_id = ?', scopeId);
             this.sql.exec('DELETE FROM scopes WHERE scope_id = ?', scopeId);
         });
@@ -2075,7 +2139,7 @@ export class ControlPlaneDO extends DurableObject {
             .exec('SELECT org_id FROM orgs WHERE tenant_id = ? AND slug = ?', tenantId, slug)
             .toArray()[0];
         if (slugOwner) {
-            throw new Error(`org slug '${slug}' already taken by ${slugOwner.org_id} (slugs are unique per tenant)`);
+            throw substratError('conflict', `org slug '${slug}' already taken by ${slugOwner.org_id} (slugs are unique per tenant)`);
         }
         this.sql.exec('INSERT INTO orgs (org_id, tenant_id, slug, name, created_at) VALUES (?, ?, ?, ?, ?)', orgId, tenantId, slug, name, createdAt);
         return true;
@@ -2085,13 +2149,69 @@ export class ControlPlaneDO extends DurableObject {
             .exec('SELECT * FROM orgs WHERE tenant_id = ? ORDER BY slug', tenantId)
             .toArray();
     }
+    /** The tenant-level reads, synchronous, for a bound asked inside one of this object's units (#1184). */
+    directoryReader = {
+        now: () => new Date().toISOString(),
+        tenantTuples: (tenantId, subject, prefix) => this.tenantTuples(tenantId, subject, prefix),
+        getRole: (tenantId, key) => this.getRole(tenantId, key),
+    };
     /**
-     * Tombstone a membership (K-21), never DELETE. Guarded on `revoked_at IS NULL`
-     * so a repeat revoke neither moves the timestamp nor produces a second audit
-     * row. Returns whether it changed, so the coordinator can skip the audit write.
+     * A membership change as ONE DO unit (#1184) — `HostAdmin.applyMembership`. Synchronous, so
+     * no other request into this object runs between its reads and its writes: an add's removal
+     * fence, then the bound — `boundedBy` covering every permission `roleKey` carries, against the
+     * tuples and the role as they stand now (`tenantCoverage`) — then the write: the TENANT role
+     * and its audit row for an add, `revokeAndFence`'s three for a removal. The coordinator mints
+     * the row (id, actor, attribution, `causedBy`).
      */
-    revokeMember(tenantId, subject, object, at) {
-        return this.revokeTenantTuple(tenantId, subject, 'member', object, at);
+    applyMembership(change, row) {
+        const { tenantId, principal, roleKey, boundedBy } = change;
+        return this.ctx.storage.transactionSync(() => {
+            if (change.op === 'add') {
+                const fence = this.sql.exec(MEMBERSHIP_FENCE_SINCE_SQL, tenantId, principal, change.unlessRemovedSince).toArray()[0];
+                if (fence)
+                    return { applied: false, removedAt: fence.removed_at };
+            }
+            const role = this.getRole(tenantId, roleKey);
+            if (!role)
+                return { applied: false, unknownRole: roleKey };
+            const bound = tenantCoverage(this.directoryReader, tenantId, boundedBy, role.permissions);
+            if (!bound.covered)
+                return { applied: false, missing: bound.missing };
+            if (change.op === 'remove') {
+                return { applied: true, changed: this.revokeAndFenceIn(tenantId, principal, `role:${roleKey}`, `tenant:${tenantId}`, row) };
+            }
+            this.writeTenantTuple(tenantId, `principal:${principal}`, `role:${roleKey}`, `tenant:${tenantId}`, null);
+            this.recordAdmin({ ...row, at: new Date().toISOString() });
+            return { applied: true };
+        });
+    }
+    /**
+     * A tenant-level removal as ONE DO unit (#1184) — `unassignRole` at the tenant node, and
+     * `removeMember`: the tombstone, the principal's removal fence, and the audit row, which
+     * K-21 writes only when something changed. The fence is raised either way: a removal of
+     * someone whose add is still on its way takes nothing, and must still win. One method, so
+     * no add can run between the revoke and the fence. Returns whether anything changed.
+     */
+    revokeAndFence(tenantId, principal, relation, object, row) {
+        return this.ctx.storage.transactionSync(() => this.revokeAndFenceIn(tenantId, principal, relation, object, row));
+    }
+    /**
+     * `revokeAndFence`'s writes, for inside a transaction a caller already holds.
+     *
+     * The time is minted HERE, in the unit, never taken from `row.at`: the coordinator mints its
+     * row before an awaited RPC, and a queue delay in between would age the fence by however
+     * long the call waited, which an add's cutoff would read as a removal from before its window
+     * (Codex round 3). The fence is compared with an add's `occurredAt`, stamped by the ScopeDO's
+     * clock. Both are this platform's own NTP-disciplined time, sub-second apart, which is far
+     * inside `MEMBERSHIP_REMOVAL_SKEW_MS`.
+     */
+    revokeAndFenceIn(tenantId, principal, relation, object, row) {
+        const at = new Date().toISOString();
+        const changed = this.revokeTenantTuple(tenantId, `principal:${principal}`, relation, object, at);
+        this.sql.exec(RAISE_MEMBERSHIP_FENCE_SQL, tenantId, principal, at);
+        if (changed)
+            this.recordAdmin({ ...row, at });
+        return changed;
     }
     /** Tombstone any tenant tuple by its exact (subject, relation, object). Returns
      *  whether a live row changed, so a repeat revoke is a silent no-op. */
@@ -2248,7 +2368,7 @@ export class ControlPlaneDO extends DurableObject {
         if (existing) {
             if (existing.topology === topology && existing.tenant_id === tenantId)
                 return false;
-            throw new Error(`identity pool '${provider}' is already registered as ${existing.topology}` +
+            throw substratError('conflict', `identity pool '${provider}' is already registered as ${existing.topology}` +
                 `${existing.tenant_id ? ` for tenant ${existing.tenant_id}` : ''}`);
         }
         this.sql.exec('INSERT INTO _substrat_identity_pools (provider, topology, tenant_id, created_at) VALUES (?, ?, ?, ?)', provider, topology, tenantId, createdAt);
@@ -2451,38 +2571,74 @@ export class ControlPlaneDO extends DurableObject {
           granted_by, granted_at, revoked_at)
        VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL)`, row.connectionId, row.tenantId, row.vertical, row.permission, row.scopeId, row.expiresAt, row.grantedBy, row.grantedAt);
     }
-    // -- the schedule switch's record (#1674) — `system-switch-record.ts` is the whole rule ---
+    // -- the kill switches' record (#1674, #2029) — `system-switch-record.ts` is the whole rule ---
     /** OFF's write, before the scope's switch moves (#1823). Answers the row as it was. */
-    recordSystemSwitchedOff(row) {
-        return recordSystemSwitchedOff(this.kernelSql, row);
+    recordSwitchedOff(row) {
+        return this.writeSwitchIntent(row, recordSwitchedOff);
     }
-    /** ON's write, before the scope moves. Answers the row as it was, for a failed ON to restore. */
-    recordSystemSwitchedOn(row) {
-        return recordSystemSwitchedOn(this.kernelSql, row);
+    /** ON's write, before the scope moves. Answers the row as it was, for a call that held nothing to restore. */
+    recordSwitchedOn(row) {
+        return this.writeSwitchIntent(row, recordSwitchedOn);
     }
-    /** Put a row back as the switch call's record write found it — the move that followed it failed. */
-    restoreSystemSwitchRecord(key, prior) {
-        restoreSystemSwitchRecord(this.kernelSql, key, prior);
+    /**
+     * #2045 (Codex r3): a switch call's record write is also its write-ahead intent, in ONE
+     * transaction: the record, the subject's owed mark under this call's id, and the scope's reconcile
+     * receipt cleared, so the sweep reconciles it and its re-assert settles the mark. Nothing that
+     * happens to the move after this (a throw, a lost answer, a failed clear) can leave the scope
+     * apart from its record with no mark to repair it. A write a newer call's row refuses writes
+     * nothing at all.
+     */
+    writeSwitchIntent(row, write) {
+        return this.ctx.storage.transactionSync(() => {
+            const prior = write(this.kernelSql, row);
+            if (recordWriteSuperseded(prior, row))
+                return prior;
+            markSwitchOwed(this.kernelSql, row.kind, row.tenantId, row.scopeId, row.key, row.operationId);
+            this.sql.exec('UPDATE scopes SET provisioned_version_id = NULL WHERE scope_id = ?', row.scopeId);
+            return prior;
+        });
+    }
+    /**
+     * Put a row back as the switch call's record write found it — the move that followed it held
+     * nothing. #2045: with `clearOwed`, the call's own owed mark goes in the same transaction. The
+     * row put back carries the PRIOR call's operation id, which is the fence every later re-assert
+     * confirms under, and a mark newer than that fence is never cleared by one; so a mark left behind
+     * by a separate clear that failed would keep the scope un-receipted on every pass.
+     */
+    restoreSwitchRecord(key, prior, clearOwed = false) {
+        this.ctx.storage.transactionSync(() => {
+            restoreSwitchRecord(this.kernelSql, key, prior);
+            if (clearOwed)
+                clearSwitchOwed(this.kernelSql, key.kind, key.tenantId, key.scopeId, key.key, key.operationId);
+        });
     }
     /**
-     * #1823: which of these modules the tenant holds a live TENANT-level grant for — what the
-     * switch passes the scope as `tenantHeld`, so a module whose only authority there is a tenant
-     * tuple is still switchable. Read here, where the tenant tuples live.
+     * #1823 (#2030 for a peer): which of these subjects the tenant holds a live TENANT-level grant
+     * for — what the switch passes the scope as `tenantHeld`, so a subject whose only authority
+     * there is a tenant tuple is still switchable. Read here, where the tenant tuples live.
      */
-    tenantHeldSystemModules(tenantId, moduleIds, now) {
-        return moduleIds.filter((m) => tenantHoldsSystemGrant(this.kernelSql, tenantId, m, now));
+    tenantHeldOf(kind, tenantId, keys, now) {
+        return tenantHeldOf(this.kernelSql, kind, tenantId, keys, now);
     }
     /** The fleet read. */
     listSystemSwitches(filter) {
         return listSystemSwitchRecords(this.kernelSql, filter);
     }
-    /** One scope's recorded positions, by module — the status read's `recorded` join. */
-    systemSwitchRecordsOf(tenantId, scopeId) {
-        return [...systemSwitchRecordsOf(this.kernelSql, tenantId, scopeId)];
+    /** One scope's recorded positions of one kind, by subject — the status read's `recorded` join. */
+    switchRecordsOf(kind, tenantId, scopeId) {
+        return [...switchRecordsOf(this.kernelSql, kind, tenantId, scopeId)];
+    }
+    /** #2045: each record row's position and fence, by subject, read together — `switchRecordStatesOf`. */
+    switchRecordStatesOf(kind, tenantId, scopeId) {
+        return [...switchRecordStatesOf(this.kernelSql, kind, tenantId, scopeId)];
+    }
+    /** #2045: the subjects of one kind owed a re-assert on one scope. */
+    switchesOwedOf(kind, tenantId, scopeId) {
+        return [...switchesOwedOf(this.kernelSql, kind, tenantId, scopeId)];
     }
-    /** The modules a re-assert switches back off on one scope. */
-    switchedOffModulesOf(tenantId, scopeId) {
-        return switchedOffModulesOf(this.kernelSql, tenantId, scopeId);
+    /** #2045: the scope confirmed a move of this subject under `fence` — `clearSwitchOwed`. */
+    clearSwitchOwed(kind, tenantId, scopeId, key, fence) {
+        clearSwitchOwed(this.kernelSql, kind, tenantId, scopeId, key, fence);
     }
     /** The tenant's LIVE connection grants (#592) — the provision/reconcile gather read. */
     listConnectionGrants(tenantId) {
@@ -2546,7 +2702,7 @@ export class ControlPlaneDO extends DurableObject {
         if (existing) {
             if (existing.principal_id === principal)
                 return false;
-            throw new Error(`identity ${provider}:${externalId} in tenant ${tenantId} is already bound to ${existing.principal_id}`);
+            throw substratError('conflict', `identity ${provider}:${externalId} in tenant ${tenantId} is already bound to ${existing.principal_id}`);
         }
         this.sql.exec(`INSERT INTO _substrat_identities
          (provider, external_id, principal_id, tenant_id, scope_id, created_at)