@substrat-run/adapter-cloudflare 0.136.0 → 0.138.0

package/dist/control-plane-do.js

@@ -1,8 +1,9 @@
 import { DurableObject } from 'cloudflare:workers';
-import { IMPERSONATION_COLUMNS, IMPERSONATION_DDL, impersonationByIdQuery, impersonationListQuery, impersonationRowValues, ISSUE_RETENTION_DAYS, telemetryRetentionStatements, assertRowLimit, OPS_FAILURE_RETENTION_DAYS, SWEEP_RUN_RETENTION_DAYS, SWEEP_RUNS_INTENT_INDEX, sweepRunsIntentHasKind, SYSTEM_SWITCHES_BACKFILL_SQL, SYSTEM_SWITCHES_TABLE, SYSTEM_SWITCHES_DDL, forgetSystemSwitchesOf, listSystemSwitchRecords, recordSystemSwitchedOff, recordSystemSwitchedOn, restoreSystemSwitchRecord, scopesSwitchedOffFor, switchedOffModulesOf, systemSwitchRecordsOf, systemSwitchesTableExists, VERSION_MIGRATIONS_DDL, splitVersionMigrationsBatch, versionMigrationsOf, versionsAwaitSplit, writeVersionMigrations, MODEL_USAGE_RETENTION_DAYS, DO_SQL_LIMITS, ulid, isPrimaryScopeRow, resolveVerticalInstanceFrom, LEGACY_SCOPE_ROWS_BACKFILL, loadDirectoryDump, redactSubjectDirectoryText, ISSUE_EXEMPLAR_OWNER_BACKFILL_SQL, issueExemplarOwner, } from '@substrat-run/kernel';
+import { membershipFencesBackfillSql, MEMBERSHIP_FENCES_DDL, MEMBERSHIP_FENCES_TABLE, membershipFencesTableExists, MEMBERSHIP_FENCE_SINCE_SQL, RAISE_MEMBERSHIP_FENCE_SQL, tenantCoverage, IMPERSONATION_COLUMNS, IMPERSONATION_DDL, impersonationByIdQuery, impersonationListQuery, impersonationRowValues, ISSUE_RETENTION_DAYS, telemetryRetentionStatements, assertRowLimit, OPS_FAILURE_RETENTION_DAYS, SWEEP_RUN_RETENTION_DAYS, SWEEP_RUNS_INTENT_INDEX, sweepRunsIntentHasKind, PEER_SWITCHES_DDL, SWITCH_OWED_DDL, clearSwitchOwed, markSwitchOwed, scopeOwesSwitch, switchesOwedOf, SWITCH_KINDS, SYSTEM_SWITCHES_DDL, forgetSwitchesOf, listSystemSwitchRecords, recordSwitchedOff, recordSwitchedOn, recordWriteSuperseded, restoreSwitchRecord, scopesSwitchedOffFor, switchRecordStatesOf, switchRecordsOf, switchesBackfillSqlOf, switchesDdlOf, switchesTableExists, switchesTableOf, tenantHeldOf, VERSION_MIGRATIONS_DDL, splitVersionMigrationsBatch, versionMigrationsOf, versionsAwaitSplit, writeVersionMigrations, MODEL_USAGE_RETENTION_DAYS, DO_SQL_LIMITS, ulid, isPrimaryScopeRow, resolveVerticalInstanceFrom, LEGACY_SCOPE_ROWS_BACKFILL, loadDirectoryDump, redactSubjectDirectoryText, ISSUE_EXEMPLAR_OWNER_BACKFILL_SQL, issueExemplarOwner, } from '@substrat-run/kernel';
+import { replyOf } from './do-reply.js';
 import { splitSqlStatements, switchSqlOver } from './scope-do.js';
 import { doBuiltColumnsOf, doRedactionSql } from './sql.js';
-import { assertReplayableDump, opsFailureFingerprint, ROUTE_SCOPE_HOSTNAMES_MAX, substratError } from '@substrat-run/contracts';
+import { assertReplayableDump, opsFailureFingerprint, ROUTE_SCOPE_HOSTNAMES_MAX, SCOPE_GATE_REASONS, substratError } from '@substrat-run/contracts';
 /**
  * The router's per-request directory read (`readRoute`), lifted out so its plan can be asked for
  * (#2005): the scope's hostname set is a correlated subquery on the hot path every request takes,
@@ -71,6 +72,39 @@ function keysetTail(where, params, key, page, defaultOrder = 'asc') {
     }
     return tail;
 }
+/**
+ * The directory spine, as this adapter builds it — one of two hand-written copies (#969).
+ *
+ * The other is `applyDirectorySchema()` in `adapter-sqlite/src/index.ts`, and the two must
+ * describe the same schema: this side is production, that side is dev, CI, self-host and
+ * escrow. So a new directory or `_substrat_*` table both adapters need, or a new column on
+ * a table they both build, is added HERE and THERE — and for a directory created before it,
+ * to both column-addition lists as well (`ensureDirectoryColumns` below, and the pure
+ * adapter's method of the same name).
+ *
+ * `pnpm lint:spine-ddl` (`tools/spine-ddl-drift.mjs`) is what refuses a divergence: it
+ * executes each side's DDL plus those later ALTERs and compares the schemas a query would
+ * actually meet — columns, indexes (including the ones a UNIQUE creates) and foreign keys.
+ * Two things it does NOT judge. A table present on one side only is a note, not a failure,
+ * because the adapters legitimately partition the spine differently (this one projects
+ * tenant tuples, roles and entitlements into each scope where the pure adapter keeps one
+ * shared directory). And triggers and CHECK constraints are not compared at all; the spine
+ * has none today, so adding one here means adding it there with nothing to catch you.
+ * Keeping the copies and gating them, rather than moving the DDL into the kernel, is the
+ * recorded answer to #969; `docs/architecture/kernel-design.md` §8 says why.
+ */
+/** #1713: one hosted scope, its directory lifecycle, and what its deployment last acknowledged. */
+/**
+ * #2016, Codex #2037 r2: the condition every delivery-bookkeeping write carries, in its own statement —
+ * the scope (bound as the last parameter) still exists, is not reaped, and its tenant is not reaped.
+ * A heal selects its targets, then writes its asks and its receipts after a network round trip; a
+ * delete or a reap that commits in between has already forgotten the scope's rows
+ * (`forgetLifecycleDeliveries`), and an unconditional write would bring them back as orphans.
+ */
+const STILL_DELIVERED = `EXISTS (
+  SELECT 1 FROM scopes s JOIN tenants t ON t.tenant_id = s.tenant_id
+   WHERE s.scope_id = ? AND s.status <> 'reaped' AND t.status <> 'reaped'
+)`;
 const DIRECTORY_DDL = `
   -- The ';' in this comment is a deliberate tripwire; the DDL must go through
   -- splitSqlStatements, and a naive split(';') fails the directory at construction.
@@ -423,6 +457,9 @@ const DIRECTORY_DDL = `
   );
   ${IMPERSONATION_DDL}
   ${SYSTEM_SWITCHES_DDL}
+  ${MEMBERSHIP_FENCES_DDL}
+  ${PEER_SWITCHES_DDL}
+  ${SWITCH_OWED_DDL}
   CREATE TABLE IF NOT EXISTS _substrat_admin_log (
     id TEXT PRIMARY KEY,
     actor TEXT NOT NULL,
@@ -573,10 +610,21 @@ const DIRECTORY_DDL = `
   -- directory, so the platform delivers a scope's lifecycle to it, and the heal sweep
   -- re-delivers wherever this differs from the directory. No row reads as active/active,
   -- which is what a deployment holding no lifecycle runs as.
+  -- #2016: tenant_recorded, whether that deployment answered that the scope holds a record of its
+  -- tenant (1) or not (0); NULL is never asked. The heal keeps delivering to any served scope, a
+  -- copy as much as a primary, until it answers 1.
   CREATE TABLE IF NOT EXISTS scope_lifecycle_receipts (
     scope_id TEXT PRIMARY KEY,
     delivered TEXT NOT NULL,
-    at TEXT NOT NULL
+    at TEXT NOT NULL,
+    tenant_recorded INTEGER
+  );
+  -- #2016: when the heal last asked a scope for its tenant record, whatever came of it. Kept apart
+  -- from the receipt, which only an answer writes, so a scope whose deployment never answers still
+  -- rotates behind the rest of the walk instead of heading it every pass.
+  CREATE TABLE IF NOT EXISTS scope_tenant_asks (
+    scope_id TEXT PRIMARY KEY,
+    asked_at TEXT NOT NULL
   );
   -- #1713: the lifecycle's revisions, one counter per subject ("scope:<id>" counts a
   -- scope's status transitions, "tenant:<id>" a tenant's status changes), each bumped in
@@ -665,6 +713,19 @@ export const BACKFILL_FAILURES_KEY = 'versionMigrationsBackfillFailures';
 export function backfillBackoffMs(failures) {
     return Math.min(BACKFILL_PAUSE_MS * 2 ** failures, BACKFILL_BACKOFF_MAX_MS);
 }
+/** The directory writes `ControlPlaneDO.reply` answers for (#113): each refuses with a typed throw. */
+export const REPLIED_METHODS = [
+    'createTenant',
+    'setTenantStatus',
+    'reapTenant',
+    'setTenantName',
+    'putTenantStore',
+    'putBlobStore',
+    'provisionScope',
+    'createOrg',
+    'registerIdentityPool',
+    'linkIdentity',
+];
 export class ControlPlaneDO extends DurableObject {
     sql;
     /** The directory's store as the kernel's SQL handle — the switch record's helpers (#1674). */
@@ -748,20 +809,30 @@ export class ControlPlaneDO extends DurableObject {
      * construction. A directory restore runs `buildDirectorySchema` inside its own transaction
      * instead (`importDump`).
      *
-     * The schedule switch's record (#1674) is backfilled from the admin log on a run that
-     * creates its table, and its table is created in the SAME transaction as the backfill
-     * (Copilot review): the gate is "the table does not exist yet", so a table committed ahead
-     * of a backfill that then failed would read as already migrated on every later run. On
+     * Each kill switch's record (#1674, and #2029's for a peer) is backfilled from the admin log
+     * on a run that creates its table, and its table is created in the SAME transaction as the
+     * backfill (Copilot review): the gate is "the table does not exist yet", so a table committed
+     * ahead of a backfill that then failed would read as already migrated on every later run. On
      * such a run its statements are held back from the loop below and run with the backfill.
      */
     applyDirectorySchema() {
-        const switchRecordIsNew = !systemSwitchesTableExists(this.kernelSql);
-        this.buildDirectorySchema({ holdSwitchRecord: switchRecordIsNew });
-        if (switchRecordIsNew) {
+        const newRecords = SWITCH_KINDS.filter((kind) => !switchesTableExists(this.kernelSql, kind));
+        const fencesAreNew = !membershipFencesTableExists(this.kernelSql);
+        this.buildDirectorySchema({ holdSwitchRecords: newRecords, holdFences: fencesAreNew });
+        for (const kind of newRecords) {
+            this.ctx.storage.transactionSync(() => {
+                for (const stmt of splitSqlStatements(switchesDdlOf(kind)))
+                    this.sql.exec(stmt);
+                this.sql.exec(switchesBackfillSqlOf(kind));
+            });
+        }
+        // #1184: the removal fence and its backfill from the admin log, together, the same way.
+        if (fencesAreNew) {
             this.ctx.storage.transactionSync(() => {
-                for (const stmt of splitSqlStatements(SYSTEM_SWITCHES_DDL))
+                for (const stmt of splitSqlStatements(MEMBERSHIP_FENCES_DDL))
                     this.sql.exec(stmt);
-                this.sql.exec(SYSTEM_SWITCHES_BACKFILL_SQL);
+                // Stamped now, inside this unit (`membershipFencesBackfillSql` says why).
+                this.sql.exec(membershipFencesBackfillSql(new Date().toISOString()));
             });
         }
     }
@@ -769,12 +840,15 @@ export class ControlPlaneDO extends DurableObject {
      * The directory's tables as this code builds them, carrying whatever the directory already
      * holds forward to that shape: the construction's pass, and a restore's (#1898, #1912), which
      * runs it inside its transaction onto an emptied directory, before loading any row.
-     * `holdSwitchRecord` leaves the #1674 record's statements to the caller, which creates the
-     * table with its backfill.
+     * `holdSwitchRecords` leaves those kinds' record statements (#1674, #2029) to the caller, which
+     * creates each table with its backfill; `holdFences` does the same for #1184's removal fence.
      */
-    buildDirectorySchema({ holdSwitchRecord }) {
+    buildDirectorySchema({ holdSwitchRecords, holdFences = false }) {
+        const held = holdSwitchRecords.map(switchesTableOf);
         for (const stmt of DIRECTORY_DDL_PLAN.loop) {
-            if (holdSwitchRecord && stmt.includes(SYSTEM_SWITCHES_TABLE))
+            if (held.some((table) => stmt.includes(table)))
+                continue;
+            if (holdFences && stmt.includes(MEMBERSHIP_FENCES_TABLE))
                 continue;
             this.sql.exec(stmt);
         }
@@ -930,6 +1004,8 @@ export class ControlPlaneDO extends DurableObject {
         }
         // §4.8's grace-window timestamp on tenants (mirrors scopes' archived_at).
         this.addColumn('tenants', 'deleting_at TEXT');
+        // #2016: whether a scope's deployment holds a record of its tenant (see the table's DDL).
+        this.addColumn('scope_lifecycle_receipts', 'tenant_recorded INTEGER');
         // #412 provenance: the manager tenant that provisioned this one (NULL = direct staff).
         this.addColumn('tenants', 'provisioned_by_tenant TEXT REFERENCES tenants(tenant_id)');
         // §4.7 custom-hostname issuance: CF's hostname id + the DNS records to publish (JSON).
@@ -1089,12 +1165,13 @@ export class ControlPlaneDO extends DurableObject {
             // Every table from this code's DDL, onto an empty directory, so none of the pass's
             // rebuilds fires (each would reach `transactionSync`, which this async transaction cannot
             // hold): every table is created on the current shape.
-            this.buildDirectorySchema({ holdSwitchRecord: false });
+            this.buildDirectorySchema({ holdSwitchRecords: [] });
             // Refuses a table this code does not build, adds unknown columns bare, loads the rows by
             // name, then the legacy scope and #1674 backfills, all inside this transaction.
             loadDirectoryDump(tables, {
                 columnsOf: (name) => doBuiltColumnsOf(this.sql, name),
                 exec: (sql) => this.sql.exec(sql),
+                now: () => new Date().toISOString(),
                 maxColumns: DO_SQL_LIMITS.columns,
                 insert: (sql, rows) => {
                     for (const row of rows)
@@ -1153,7 +1230,7 @@ export class ControlPlaneDO extends DurableObject {
             .exec('SELECT tenant_id FROM tenants WHERE slug = ?', slug)
             .toArray()[0];
         if (slugOwner) {
-            throw new Error(`tenant slug '${slug}' already taken by ${slugOwner.tenant_id} (slugs are unique)`);
+            throw substratError('conflict', `tenant slug '${slug}' already taken by ${slugOwner.tenant_id} (slugs are unique)`);
         }
         this.sql.exec(`INSERT INTO tenants (tenant_id, slug, name, status, created_at, provisioned_by_tenant)
        VALUES (?, ?, ?, 'active', ?, ?)`, id, slug, name, createdAt, provisionedByTenant);
@@ -1163,12 +1240,12 @@ export class ControlPlaneDO extends DurableObject {
     setTenantStatus(tenantId, status) {
         const before = this.readTenant(tenantId);
         if (!before)
-            throw new Error(`unknown tenant: ${tenantId}`);
+            throw substratError('not_found', `unknown tenant: ${tenantId}`);
         // `reaped` is terminal and destroys data — unreachable here, only ever set by
         // reapTenant, so a plain status flip cannot forge a tombstone over live data
         // (§4.8, the tenant analogue of reapScope's archived-only gate).
         if (status === 'reaped') {
-            throw new Error(`tenant ${tenantId} cannot be set to 'reaped' via setTenantStatus — reap goes through reapTenant (control-plane.md §4.8)`);
+            throw substratError('validation_failed', `tenant ${tenantId} cannot be set to 'reaped' via setTenantStatus — reap goes through reapTenant (control-plane.md §4.8)`);
         }
         // Stamp/clear deleting_at so the grace-window sweep can age tenants (§4.8):
         // entering `deleting` stamps, leaving it (an un-delete) clears — the same shape
@@ -1195,9 +1272,9 @@ export class ControlPlaneDO extends DurableObject {
     reapTenant(tenantId) {
         const before = this.readTenant(tenantId);
         if (!before)
-            throw new Error(`unknown tenant: ${tenantId}`);
+            throw substratError('not_found', `unknown tenant: ${tenantId}`);
         if (before.status !== 'deleting') {
-            throw new Error(`tenant ${tenantId} is ${before.status}, not deleting — only a deleting tenant may be reaped`);
+            throw substratError('conflict', `tenant ${tenantId} is ${before.status}, not deleting — only a deleting tenant may be reaped`);
         }
         // KEPT as the tombstone: the `tenants` row itself and `_substrat_admin_log` (the
         // compliance witness — never swept). Scope rows were already reaped individually.
@@ -1209,11 +1286,16 @@ export class ControlPlaneDO extends DurableObject {
             '_substrat_entitlements', // per-tenant SKU flags
             'orgs', // K-22 org records
             '_substrat_system_switches', // #1674: the schedule switch's record, per scope
+            '_substrat_membership_fences', // #1184: the latest removal, per principal
+            '_substrat_peer_switches', // #2029: the peer switch's record, per scope
+            '_substrat_switch_owed', // #2045: subjects owed a re-assert, per scope
         ]) {
             this.sql.exec(`DELETE FROM ${table} WHERE tenant_id = ?`, tenantId);
         }
         this.ctx.storage.transactionSync(() => {
             this.sql.exec("UPDATE tenants SET status = 'reaped', deleting_at = NULL WHERE tenant_id = ?", tenantId);
+            // Its scopes were reaped one by one, each forgetting its own; any left over go here.
+            this.forgetLifecycleDeliveries('scope_id IN (SELECT scope_id FROM scopes WHERE tenant_id = ?)', tenantId);
             this.bumpLifecycleRevision(`tenant:${tenantId}`); // #1713
         });
         return before.status;
@@ -1222,7 +1304,7 @@ export class ControlPlaneDO extends DurableObject {
     setTenantName(tenantId, name) {
         const before = this.readTenant(tenantId);
         if (!before)
-            throw new Error(`unknown tenant: ${tenantId}`);
+            throw substratError('not_found', `unknown tenant: ${tenantId}`);
         this.sql.exec('UPDATE tenants SET name = ? WHERE tenant_id = ?', name, tenantId);
         return before.name;
     }
@@ -1256,10 +1338,10 @@ export class ControlPlaneDO extends DurableObject {
             .exec('SELECT status FROM tenants WHERE tenant_id = ?', row.tenantId)
             .toArray()[0];
         if (!tenantRow) {
-            throw new Error(`cannot provision tenant store under unknown tenant: ${row.tenantId}`);
+            throw substratError('not_found', `cannot provision tenant store under unknown tenant: ${row.tenantId}`);
         }
         if (tenantRow.status !== 'active') {
-            throw new Error(`cannot provision tenant store under non-active tenant (status: ${tenantRow.status}): ${row.tenantId}`);
+            throw substratError('conflict', `cannot provision tenant store under non-active tenant (status: ${tenantRow.status}): ${row.tenantId}`);
         }
         this.sql.exec(`INSERT OR IGNORE INTO tenant_stores (tenant_id, vertical, binding, kind, ref, created_at)
        VALUES (?, ?, ?, ?, ?, ?)`, row.tenantId, row.vertical, row.binding, row.kind, row.ref, row.createdAt);
@@ -1300,10 +1382,10 @@ export class ControlPlaneDO extends DurableObject {
             .exec('SELECT status FROM tenants WHERE tenant_id = ?', row.tenantId)
             .toArray()[0];
         if (!tenantRow) {
-            throw new Error(`cannot provision blob store under unknown tenant: ${row.tenantId}`);
+            throw substratError('not_found', `cannot provision blob store under unknown tenant: ${row.tenantId}`);
         }
         if (tenantRow.status !== 'active') {
-            throw new Error(`cannot provision blob store under non-active tenant (status: ${tenantRow.status}): ${row.tenantId}`);
+            throw substratError('conflict', `cannot provision blob store under non-active tenant (status: ${tenantRow.status}): ${row.tenantId}`);
         }
         this.sql.exec(`INSERT OR IGNORE INTO blob_stores (tenant_id, vertical, binding, kind, ref, created_at)
        VALUES (?, ?, ?, ?, ?, ?)`, row.tenantId, row.vertical, row.binding, row.kind, row.ref, row.createdAt);
@@ -1340,10 +1422,10 @@ export class ControlPlaneDO extends DurableObject {
             .exec('SELECT status FROM tenants WHERE tenant_id = ?', tenantId)
             .toArray()[0];
         if (!tenantRow) {
-            throw new Error(`cannot provision scope under unknown tenant: ${tenantId}`);
+            throw substratError('conflict', `cannot provision scope under unknown tenant: ${tenantId}`);
         }
         if (tenantRow.status !== 'active') {
-            throw new Error(`cannot provision scope under non-active tenant (status: ${tenantRow.status}): ${tenantId}`);
+            throw substratError('conflict', `cannot provision scope under non-active tenant (status: ${tenantRow.status}): ${tenantId}`);
         }
         // Idempotency is on the scope_id (§3.3: idempotent and journaled — safe to
         // re-run), so an existing scope short-circuits before the slug check:
@@ -1356,7 +1438,7 @@ export class ControlPlaneDO extends DurableObject {
                 .exec("SELECT scope_id FROM scopes WHERE tenant_id = ? AND slug = ? AND status NOT IN ('archived', 'reaped')", tenantId, record.slug)
                 .toArray()[0];
             if (slugOwner) {
-                throw new Error(`scope slug '${record.slug}' already taken under tenant ${tenantId} ` +
+                throw substratError('conflict', `scope slug '${record.slug}' already taken under tenant ${tenantId} ` +
                     `by ${slugOwner.scope_id} (slugs are unique within a tenant)`);
             }
             // A scope born while its vertical serves in place (#286) is born ON the serving
@@ -1438,6 +1520,21 @@ export class ControlPlaneDO extends DurableObject {
             return undefined;
         return row;
     }
+    /**
+     * A directory write, its refusal answered as DATA (#113): the typed throws in these methods
+     * would reach the coordinator flattened, so it calls them through here and rethrows the
+     * refusal with its code (`unwrapReply`). Called directly, they still throw — a coordinator
+     * from before this change does — and that caller reads the code folded into the message.
+     */
+    async reply(method, args) {
+        return replyOf(() => {
+            // An RPC argument, so held to the list rather than trusted to be one of its names.
+            if (!REPLIED_METHODS.includes(method))
+                throw new Error(`not a replied directory method: ${String(method)}`);
+            const run = this[method];
+            return run.apply(this, args);
+        });
+    }
     /**
      * The getScope gate (control-plane.md §4.1/§4.2): validate the scope belongs
      * to the tenant and both records are active, or throw the fail-closed reason.
@@ -1454,11 +1551,10 @@ export class ControlPlaneDO extends DurableObject {
      * The same gate, answering its refusal as DATA (#1718). An error thrown in a Durable
      * Object arrives at the coordinator flattened, its code gone, so a typed throw here
      * would still reach the caller untyped (#1714 measured it). A record crosses intact, so
-     * the coordinator throws the refusal itself, typed. `code` is set only where a code is
-     * known; the lifecycle refusals keep their bare messages. Null means access is allowed.
+     * the coordinator throws the refusal itself, typed (#113: every refusal but a failed
+     * migration carries its code). Null means access is allowed.
      */
     scopeAccessRefusal(tenantId, scopeId) {
-        const refuse = (message) => ({ code: null, message });
         const row = this.sql
             .exec('SELECT tenant_id, status FROM scopes WHERE scope_id = ?', scopeId)
             .toArray()[0];
@@ -1469,10 +1565,14 @@ export class ControlPlaneDO extends DurableObject {
             .exec('SELECT status FROM tenants WHERE tenant_id = ?', tenantId)
             .toArray()[0];
         if (!tenantRow) {
-            return refuse(`scope has no tenant record: (${tenantId}, ${scopeId})`);
+            return { code: 'not_found', message: `scope has no tenant record: (${tenantId}, ${scopeId})`, reason: SCOPE_GATE_REASONS.unrecorded };
         }
         if (tenantRow.status !== 'active') {
-            return refuse(`tenant not active (status: ${tenantRow.status}): ${tenantId}`);
+            return {
+                code: 'conflict',
+                message: `tenant not active (status: ${tenantRow.status}): ${tenantId}`,
+                reason: SCOPE_GATE_REASONS.notActive,
+            };
         }
         if (row.status !== 'active') {
             // A scope stuck in provisioning because its migrations failed must say so.
@@ -1483,10 +1583,17 @@ export class ControlPlaneDO extends DurableObject {
                 .exec('SELECT migration_failed_version, migration_error FROM scopes WHERE scope_id = ?', scopeId)
                 .toArray()[0];
             if (failure?.migration_failed_version) {
-                return refuse(`migration failed for ${failure.migration_failed_version} — scope fails closed: ` +
-                    `${failure.migration_error ?? 'unknown error'}`);
+                return {
+                    code: null,
+                    message: `migration failed for ${failure.migration_failed_version} — scope fails closed: ` +
+                        `${failure.migration_error ?? 'unknown error'}`,
+                };
             }
-            return refuse(`scope not active (status: ${row.status}): ${scopeId}`);
+            return {
+                code: 'conflict',
+                message: `scope not active (status: ${row.status}): ${scopeId}`,
+                reason: SCOPE_GATE_REASONS.notActive,
+            };
         }
         return null;
     }
@@ -1521,7 +1628,7 @@ export class ControlPlaneDO extends DurableObject {
         if (!from.includes(row.status)) {
             return {
                 ok: false,
-                code: null,
+                code: 'conflict',
                 message: `illegal scope transition for ${action}: ${row.status} → ${to} ` +
                     `(allowed from: ${from.join('|')})`,
             };
@@ -1541,8 +1648,10 @@ export class ControlPlaneDO extends DurableObject {
             else {
                 this.sql.exec('UPDATE scopes SET status = ? WHERE scope_id = ?', to, scopeId);
             }
-            if (to === 'reaped')
-                forgetSystemSwitchesOf(this.kernelSql, scopeId);
+            if (to === 'reaped') {
+                forgetSwitchesOf(this.kernelSql, scopeId);
+                this.forgetLifecycleDeliveries('scope_id = ?', scopeId);
+            }
             this.bumpLifecycleRevision(`scope:${scopeId}`); // #1713, with the status it counts
         });
         return { ok: true, status: row.status, vertical: row.vertical };
@@ -1948,6 +2057,11 @@ export class ControlPlaneDO extends DurableObject {
      * for a repair that failed, which is the one outcome that must keep being retried.
      */
     markScopeProvisioned(scopeId, versionId) {
+        // #2045: a scope owed a switch re-assert keeps no receipt, so the sweep reconciles it again
+        // until its re-assert has settled every mark — including a mark written while this
+        // reconcile ran, after its re-assert had read none.
+        if (versionId !== null && scopeOwesSwitch(this.kernelSql, scopeId))
+            versionId = null;
         this.sql.exec('UPDATE scopes SET provisioned_version_id = ? WHERE scope_id = ?', versionId, scopeId);
     }
     /**
@@ -1956,9 +2070,24 @@ export class ControlPlaneDO extends DurableObject {
      * wiping the scope DO's storage — the fork-only refusal lives there, not here.
      */
     deleteScopeDirectory(scopeId) {
-        this.sql.exec('DELETE FROM hostnames WHERE scope_id = ?', scopeId);
-        forgetSystemSwitchesOf(this.kernelSql, scopeId);
-        this.sql.exec('DELETE FROM scopes WHERE scope_id = ?', scopeId);
+        this.ctx.storage.transactionSync(() => {
+            this.sql.exec('DELETE FROM hostnames WHERE scope_id = ?', scopeId);
+            forgetSwitchesOf(this.kernelSql, scopeId);
+            this.forgetLifecycleDeliveries('scope_id = ?', scopeId);
+            this.sql.exec('DELETE FROM scopes WHERE scope_id = ?', scopeId);
+        });
+    }
+    /**
+     * #1713 / #2016: drop the delivery bookkeeping of scopes that will never be delivered to again —
+     * what each deployment acknowledged (`scope_lifecycle_receipts`) and when the heal last asked it
+     * for its tenant record (`scope_tenant_asks`). A hard-deleted fork's rows would otherwise outlive
+     * it, and every preview created and deleted would grow both tables, and every directory dump, for
+     * good. A reaped scope keeps its tombstone row but leaves every delivery walk, so its go too.
+     */
+    forgetLifecycleDeliveries(where, ...params) {
+        for (const table of ['scope_lifecycle_receipts', 'scope_tenant_asks']) {
+            this.sql.exec(`DELETE FROM ${table} WHERE ${where}`, ...params);
+        }
     }
     readChannel(verticalSlug, channel) {
         return this.sql
@@ -2010,7 +2139,7 @@ export class ControlPlaneDO extends DurableObject {
             .exec('SELECT org_id FROM orgs WHERE tenant_id = ? AND slug = ?', tenantId, slug)
             .toArray()[0];
         if (slugOwner) {
-            throw new Error(`org slug '${slug}' already taken by ${slugOwner.org_id} (slugs are unique per tenant)`);
+            throw substratError('conflict', `org slug '${slug}' already taken by ${slugOwner.org_id} (slugs are unique per tenant)`);
         }
         this.sql.exec('INSERT INTO orgs (org_id, tenant_id, slug, name, created_at) VALUES (?, ?, ?, ?, ?)', orgId, tenantId, slug, name, createdAt);
         return true;
@@ -2020,13 +2149,69 @@ export class ControlPlaneDO extends DurableObject {
             .exec('SELECT * FROM orgs WHERE tenant_id = ? ORDER BY slug', tenantId)
             .toArray();
     }
+    /** The tenant-level reads, synchronous, for a bound asked inside one of this object's units (#1184). */
+    directoryReader = {
+        now: () => new Date().toISOString(),
+        tenantTuples: (tenantId, subject, prefix) => this.tenantTuples(tenantId, subject, prefix),
+        getRole: (tenantId, key) => this.getRole(tenantId, key),
+    };
     /**
-     * Tombstone a membership (K-21), never DELETE. Guarded on `revoked_at IS NULL`
-     * so a repeat revoke neither moves the timestamp nor produces a second audit
-     * row. Returns whether it changed, so the coordinator can skip the audit write.
+     * A membership change as ONE DO unit (#1184) — `HostAdmin.applyMembership`. Synchronous, so
+     * no other request into this object runs between its reads and its writes: an add's removal
+     * fence, then the bound — `boundedBy` covering every permission `roleKey` carries, against the
+     * tuples and the role as they stand now (`tenantCoverage`) — then the write: the TENANT role
+     * and its audit row for an add, `revokeAndFence`'s three for a removal. The coordinator mints
+     * the row (id, actor, attribution, `causedBy`).
      */
-    revokeMember(tenantId, subject, object, at) {
-        return this.revokeTenantTuple(tenantId, subject, 'member', object, at);
+    applyMembership(change, row) {
+        const { tenantId, principal, roleKey, boundedBy } = change;
+        return this.ctx.storage.transactionSync(() => {
+            if (change.op === 'add') {
+                const fence = this.sql.exec(MEMBERSHIP_FENCE_SINCE_SQL, tenantId, principal, change.unlessRemovedSince).toArray()[0];
+                if (fence)
+                    return { applied: false, removedAt: fence.removed_at };
+            }
+            const role = this.getRole(tenantId, roleKey);
+            if (!role)
+                return { applied: false, unknownRole: roleKey };
+            const bound = tenantCoverage(this.directoryReader, tenantId, boundedBy, role.permissions);
+            if (!bound.covered)
+                return { applied: false, missing: bound.missing };
+            if (change.op === 'remove') {
+                return { applied: true, changed: this.revokeAndFenceIn(tenantId, principal, `role:${roleKey}`, `tenant:${tenantId}`, row) };
+            }
+            this.writeTenantTuple(tenantId, `principal:${principal}`, `role:${roleKey}`, `tenant:${tenantId}`, null);
+            this.recordAdmin({ ...row, at: new Date().toISOString() });
+            return { applied: true };
+        });
+    }
+    /**
+     * A tenant-level removal as ONE DO unit (#1184) — `unassignRole` at the tenant node, and
+     * `removeMember`: the tombstone, the principal's removal fence, and the audit row, which
+     * K-21 writes only when something changed. The fence is raised either way: a removal of
+     * someone whose add is still on its way takes nothing, and must still win. One method, so
+     * no add can run between the revoke and the fence. Returns whether anything changed.
+     */
+    revokeAndFence(tenantId, principal, relation, object, row) {
+        return this.ctx.storage.transactionSync(() => this.revokeAndFenceIn(tenantId, principal, relation, object, row));
+    }
+    /**
+     * `revokeAndFence`'s writes, for inside a transaction a caller already holds.
+     *
+     * The time is minted HERE, in the unit, never taken from `row.at`: the coordinator mints its
+     * row before an awaited RPC, and a queue delay in between would age the fence by however
+     * long the call waited, which an add's cutoff would read as a removal from before its window
+     * (Codex round 3). The fence is compared with an add's `occurredAt`, stamped by the ScopeDO's
+     * clock. Both are this platform's own NTP-disciplined time, sub-second apart, which is far
+     * inside `MEMBERSHIP_REMOVAL_SKEW_MS`.
+     */
+    revokeAndFenceIn(tenantId, principal, relation, object, row) {
+        const at = new Date().toISOString();
+        const changed = this.revokeTenantTuple(tenantId, `principal:${principal}`, relation, object, at);
+        this.sql.exec(RAISE_MEMBERSHIP_FENCE_SQL, tenantId, principal, at);
+        if (changed)
+            this.recordAdmin({ ...row, at });
+        return changed;
     }
     /** Tombstone any tenant tuple by its exact (subject, relation, object). Returns
      *  whether a live row changed, so a repeat revoke is a silent no-op. */
@@ -2183,7 +2368,7 @@ export class ControlPlaneDO extends DurableObject {
         if (existing) {
             if (existing.topology === topology && existing.tenant_id === tenantId)
                 return false;
-            throw new Error(`identity pool '${provider}' is already registered as ${existing.topology}` +
+            throw substratError('conflict', `identity pool '${provider}' is already registered as ${existing.topology}` +
                 `${existing.tenant_id ? ` for tenant ${existing.tenant_id}` : ''}`);
         }
         this.sql.exec('INSERT INTO _substrat_identity_pools (provider, topology, tenant_id, created_at) VALUES (?, ?, ?, ?)', provider, topology, tenantId, createdAt);
@@ -2386,30 +2571,74 @@ export class ControlPlaneDO extends DurableObject {
           granted_by, granted_at, revoked_at)
        VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL)`, row.connectionId, row.tenantId, row.vertical, row.permission, row.scopeId, row.expiresAt, row.grantedBy, row.grantedAt);
     }
-    // -- the schedule switch's record (#1674) — `system-switch-record.ts` is the whole rule ---
-    /** OFF's write, after the scope's switch held. */
-    recordSystemSwitchedOff(row) {
-        recordSystemSwitchedOff(this.kernelSql, row);
+    // -- the kill switches' record (#1674, #2029) — `system-switch-record.ts` is the whole rule ---
+    /** OFF's write, before the scope's switch moves (#1823). Answers the row as it was. */
+    recordSwitchedOff(row) {
+        return this.writeSwitchIntent(row, recordSwitchedOff);
     }
-    /** ON's write, before the scope moves. Answers the row as it was, for a failed ON to restore. */
-    recordSystemSwitchedOn(row) {
-        return recordSystemSwitchedOn(this.kernelSql, row);
+    /** ON's write, before the scope moves. Answers the row as it was, for a call that held nothing to restore. */
+    recordSwitchedOn(row) {
+        return this.writeSwitchIntent(row, recordSwitchedOn);
     }
-    /** Put a row back as `recordSystemSwitchedOn` found it — the ON that followed it failed. */
-    restoreSystemSwitchRecord(key, prior) {
-        restoreSystemSwitchRecord(this.kernelSql, key, prior);
+    /**
+     * #2045 (Codex r3): a switch call's record write is also its write-ahead intent, in ONE
+     * transaction: the record, the subject's owed mark under this call's id, and the scope's reconcile
+     * receipt cleared, so the sweep reconciles it and its re-assert settles the mark. Nothing that
+     * happens to the move after this (a throw, a lost answer, a failed clear) can leave the scope
+     * apart from its record with no mark to repair it. A write a newer call's row refuses writes
+     * nothing at all.
+     */
+    writeSwitchIntent(row, write) {
+        return this.ctx.storage.transactionSync(() => {
+            const prior = write(this.kernelSql, row);
+            if (recordWriteSuperseded(prior, row))
+                return prior;
+            markSwitchOwed(this.kernelSql, row.kind, row.tenantId, row.scopeId, row.key, row.operationId);
+            this.sql.exec('UPDATE scopes SET provisioned_version_id = NULL WHERE scope_id = ?', row.scopeId);
+            return prior;
+        });
+    }
+    /**
+     * Put a row back as the switch call's record write found it — the move that followed it held
+     * nothing. #2045: with `clearOwed`, the call's own owed mark goes in the same transaction. The
+     * row put back carries the PRIOR call's operation id, which is the fence every later re-assert
+     * confirms under, and a mark newer than that fence is never cleared by one; so a mark left behind
+     * by a separate clear that failed would keep the scope un-receipted on every pass.
+     */
+    restoreSwitchRecord(key, prior, clearOwed = false) {
+        this.ctx.storage.transactionSync(() => {
+            restoreSwitchRecord(this.kernelSql, key, prior);
+            if (clearOwed)
+                clearSwitchOwed(this.kernelSql, key.kind, key.tenantId, key.scopeId, key.key, key.operationId);
+        });
+    }
+    /**
+     * #1823 (#2030 for a peer): which of these subjects the tenant holds a live TENANT-level grant
+     * for — what the switch passes the scope as `tenantHeld`, so a subject whose only authority
+     * there is a tenant tuple is still switchable. Read here, where the tenant tuples live.
+     */
+    tenantHeldOf(kind, tenantId, keys, now) {
+        return tenantHeldOf(this.kernelSql, kind, tenantId, keys, now);
     }
     /** The fleet read. */
     listSystemSwitches(filter) {
         return listSystemSwitchRecords(this.kernelSql, filter);
     }
-    /** One scope's recorded positions, by module — the status read's `recorded` join. */
-    systemSwitchRecordsOf(tenantId, scopeId) {
-        return [...systemSwitchRecordsOf(this.kernelSql, tenantId, scopeId)];
+    /** One scope's recorded positions of one kind, by subject — the status read's `recorded` join. */
+    switchRecordsOf(kind, tenantId, scopeId) {
+        return [...switchRecordsOf(this.kernelSql, kind, tenantId, scopeId)];
+    }
+    /** #2045: each record row's position and fence, by subject, read together — `switchRecordStatesOf`. */
+    switchRecordStatesOf(kind, tenantId, scopeId) {
+        return [...switchRecordStatesOf(this.kernelSql, kind, tenantId, scopeId)];
+    }
+    /** #2045: the subjects of one kind owed a re-assert on one scope. */
+    switchesOwedOf(kind, tenantId, scopeId) {
+        return [...switchesOwedOf(this.kernelSql, kind, tenantId, scopeId)];
     }
-    /** The modules a re-assert switches back off on one scope. */
-    switchedOffModulesOf(tenantId, scopeId) {
-        return switchedOffModulesOf(this.kernelSql, tenantId, scopeId);
+    /** #2045: the scope confirmed a move of this subject under `fence` — `clearSwitchOwed`. */
+    clearSwitchOwed(kind, tenantId, scopeId, key, fence) {
+        clearSwitchOwed(this.kernelSql, kind, tenantId, scopeId, key, fence);
     }
     /** The tenant's LIVE connection grants (#592) — the provision/reconcile gather read. */
     listConnectionGrants(tenantId) {
@@ -2473,7 +2702,7 @@ export class ControlPlaneDO extends DurableObject {
         if (existing) {
             if (existing.principal_id === principal)
                 return false;
-            throw new Error(`identity ${provider}:${externalId} in tenant ${tenantId} is already bound to ${existing.principal_id}`);
+            throw substratError('conflict', `identity ${provider}:${externalId} in tenant ${tenantId} is already bound to ${existing.principal_id}`);
         }
         this.sql.exec(`INSERT INTO _substrat_identities
          (provider, external_id, principal_id, tenant_id, scope_id, created_at)
@@ -2733,17 +2962,28 @@ export class ControlPlaneDO extends DurableObject {
             where.push(`(${drifted} OR s.status NOT IN ('active', 'archived') OR t.status <> 'active')`);
             params.push(...driftParams);
         }
-        params.push(...driftParams, filter.limit ?? 1000);
+        if (filter.unrecorded)
+            where.push('COALESCE(r.tenant_recorded, 0) <> 1');
+        // The unrecorded walk rotates on the ask, not the answer: never asked first, then the one asked
+        // longest ago, so a scope that answers 0, or never answers at all, cannot starve the rest of a
+        // bounded pass.
+        const order = filter.unrecorded
+            ? 'a.asked_at IS NOT NULL, a.asked_at, s.scope_id'
+            : `CASE WHEN ${drifted} THEN 0 ELSE 1 END, s.scope_id`;
+        if (!filter.unrecorded)
+            params.push(...driftParams);
+        params.push(filter.limit ?? 1000);
         return this.sql
             .exec(`SELECT s.tenant_id, s.scope_id, s.status AS scope_status, t.status AS tenant_status, r.delivered,
                 ${epoch} AS epoch, COALESCE(rs.revision, 0) AS scope_rev, COALESCE(rt.revision, 0) AS tenant_rev
            FROM scopes s
            JOIN tenants t ON t.tenant_id = s.tenant_id
            LEFT JOIN scope_lifecycle_receipts r ON r.scope_id = s.scope_id
+           LEFT JOIN scope_tenant_asks a ON a.scope_id = s.scope_id
            LEFT JOIN lifecycle_revisions rs ON rs.subject = 'scope:' || s.scope_id
            LEFT JOIN lifecycle_revisions rt ON rt.subject = 'tenant:' || s.tenant_id
           WHERE ${where.join(' AND ')}
-          ORDER BY CASE WHEN ${drifted} THEN 0 ELSE 1 END, s.scope_id
+          ORDER BY ${order}
           LIMIT ?`, ...params)
             .toArray();
     }
@@ -2774,9 +3014,17 @@ export class ControlPlaneDO extends DurableObject {
     bumpLifecycleRevision(subject) {
         this.sql.exec('INSERT INTO lifecycle_revisions (subject, revision) VALUES (?, 1) ON CONFLICT (subject) DO UPDATE SET revision = revision + 1', subject);
     }
-    /** #1713: record what a scope's deployment acknowledged holding (`lifecycleReceipt`). */
-    recordLifecycleReceipt(scopeId, delivered, at) {
-        this.sql.exec('INSERT OR REPLACE INTO scope_lifecycle_receipts (scope_id, delivered, at) VALUES (?, ?, ?)', scopeId, delivered, at);
+    /** #2016: note that the heal asked these scopes for their tenant record at `at`, before it asks. */
+    recordTenantAsks(scopeIds, at) {
+        for (const scopeId of scopeIds) {
+            this.sql.exec(`INSERT OR REPLACE INTO scope_tenant_asks (scope_id, asked_at) SELECT ?, ? WHERE ${STILL_DELIVERED}`, scopeId, at, scopeId);
+        }
+    }
+    /** #1713: record what a scope's deployment acknowledged holding (`lifecycleReceipt`), and #2016:
+     *  whether it answered that the scope holds a record of its tenant. */
+    recordLifecycleReceipt(scopeId, delivered, at, tenantRecorded = false) {
+        this.sql.exec(`INSERT OR REPLACE INTO scope_lifecycle_receipts (scope_id, delivered, at, tenant_recorded)
+       SELECT ?, ?, ?, ? WHERE ${STILL_DELIVERED}`, scopeId, delivered, at, tenantRecorded ? 1 : 0, scopeId);
     }
     recordOpsFailure(row) {
         this.sql.exec(`INSERT INTO _substrat_ops_failures