@substrat-run/adapter-cloudflare 0.137.0 → 0.139.0

package/dist/control-plane-do.js

@@ -1,8 +1,9 @@
 import { DurableObject } from 'cloudflare:workers';
-import { IMPERSONATION_COLUMNS, IMPERSONATION_DDL, impersonationByIdQuery, impersonationListQuery, impersonationRowValues, ISSUE_RETENTION_DAYS, telemetryRetentionStatements, assertRowLimit, OPS_FAILURE_RETENTION_DAYS, SWEEP_RUN_RETENTION_DAYS, SWEEP_RUNS_INTENT_INDEX, sweepRunsIntentHasKind, SYSTEM_SWITCHES_BACKFILL_SQL, SYSTEM_SWITCHES_TABLE, SYSTEM_SWITCHES_DDL, forgetSystemSwitchesOf, listSystemSwitchRecords, recordSystemSwitchedOff, recordSystemSwitchedOn, restoreSystemSwitchRecord, scopesSwitchedOffFor, tenantHoldsSystemGrant, switchedOffModulesOf, systemSwitchRecordsOf, systemSwitchesTableExists, VERSION_MIGRATIONS_DDL, splitVersionMigrationsBatch, versionMigrationsOf, versionsAwaitSplit, writeVersionMigrations, MODEL_USAGE_RETENTION_DAYS, DO_SQL_LIMITS, ulid, isPrimaryScopeRow, resolveVerticalInstanceFrom, LEGACY_SCOPE_ROWS_BACKFILL, loadDirectoryDump, redactSubjectDirectoryText, ISSUE_EXEMPLAR_OWNER_BACKFILL_SQL, issueExemplarOwner, } from '@substrat-run/kernel';
+import { membershipFencesBackfillSql, MEMBERSHIP_FENCES_DDL, MEMBERSHIP_FENCES_TABLE, membershipFencesTableExists, MEMBERSHIP_FENCE_SINCE_SQL, RAISE_MEMBERSHIP_FENCE_SQL, memberAddedAudit, orgChangeBound, tenantCoverage, IMPERSONATION_COLUMNS, IMPERSONATION_DDL, impersonationByIdQuery, impersonationListQuery, impersonationRowValues, ISSUE_RETENTION_DAYS, telemetryRetentionStatements, FINDINGS_DDL, createFindingRule, findingOfOpsFailure, findingOfSweepRun, pruneFindings, listFindingRules, listFindings, observeFinding, revokeFindingRule, setFindingStatus, assertRowLimit, OPS_FAILURE_RETENTION_DAYS, SWEEP_RUN_RETENTION_DAYS, SWEEP_RUNS_INTENT_INDEX, sweepRunsIntentHasKind, PEER_SWITCHES_DDL, SWITCH_OWED_DDL, clearSwitchOwed, markSwitchOwed, scopeOwesSwitch, switchesOwedOf, SWITCH_KINDS, SYSTEM_SWITCHES_DDL, forgetSwitchesOf, listSystemSwitchRecords, recordSwitchedOff, recordSwitchedOn, recordWriteSuperseded, restoreSwitchRecord, scopesSwitchedOffFor, switchRecordStatesOf, switchRecordsOf, switchesBackfillSqlOf, switchesDdlOf, switchesTableExists, switchesTableOf, tenantHeldOf, VERSION_MIGRATIONS_DDL, splitVersionMigrationsBatch, versionMigrationsOf, versionsAwaitSplit, writeVersionMigrations, MODEL_USAGE_RETENTION_DAYS, DO_SQL_LIMITS, ulid, isPrimaryScopeRow, resolveVerticalInstanceFrom, LEGACY_SCOPE_ROWS_BACKFILL, loadDirectoryDump, redactSubjectDirectoryText, ISSUE_EXEMPLAR_OWNER_BACKFILL_SQL, issueExemplarOwner, } from '@substrat-run/kernel';
+import { replyOf } from './do-reply.js';
 import { splitSqlStatements, switchSqlOver } from './scope-do.js';
 import { doBuiltColumnsOf, doRedactionSql } from './sql.js';
-import { assertReplayableDump, opsFailureFingerprint, ROUTE_SCOPE_HOSTNAMES_MAX, substratError } from '@substrat-run/contracts';
+import { assertReplayableDump, opsFailureFingerprint, ROUTE_SCOPE_HOSTNAMES_MAX, SCOPE_GATE_REASONS, substratError } from '@substrat-run/contracts';
 /**
  * The router's per-request directory read (`readRoute`), lifted out so its plan can be asked for
  * (#2005): the scope's hostname set is a correlated subquery on the hot path every request takes,
@@ -456,6 +457,9 @@ const DIRECTORY_DDL = `
   );
   ${IMPERSONATION_DDL}
   ${SYSTEM_SWITCHES_DDL}
+  ${MEMBERSHIP_FENCES_DDL}
+  ${PEER_SWITCHES_DDL}
+  ${SWITCH_OWED_DDL}
   CREATE TABLE IF NOT EXISTS _substrat_admin_log (
     id TEXT PRIMARY KEY,
     actor TEXT NOT NULL,
@@ -537,6 +541,8 @@ const DIRECTORY_DDL = `
     resolved_at TEXT
   );
   CREATE INDEX IF NOT EXISTS _substrat_issues_seen ON _substrat_issues (last_seen);
+  -- #1748: tenant findings and their suppress rules - kernel-owned DDL (findings.ts).
+  ${FINDINGS_DDL}
   -- The durable sweep record (#1232): one row per unit outcome per pass - a
   -- connection swept/skipped/failed, a schedule fired/skipped/failed. What makes
   -- "when was this last swept" answerable after the log line rolls off. Pruned on
@@ -709,6 +715,21 @@ export const BACKFILL_FAILURES_KEY = 'versionMigrationsBackfillFailures';
 export function backfillBackoffMs(failures) {
     return Math.min(BACKFILL_PAUSE_MS * 2 ** failures, BACKFILL_BACKOFF_MAX_MS);
 }
+/** The directory writes `ControlPlaneDO.reply` answers for (#113): each refuses with a typed throw. */
+export const REPLIED_METHODS = [
+    'createTenant',
+    'setTenantStatus',
+    'reapTenant',
+    'setTenantName',
+    'putTenantStore',
+    'putBlobStore',
+    'provisionScope',
+    'createOrg',
+    'registerIdentityPool',
+    'linkIdentity',
+    // #1748: refuses an expiry outside its bounds with validation_failed.
+    'createFindingRule',
+];
 export class ControlPlaneDO extends DurableObject {
     sql;
     /** The directory's store as the kernel's SQL handle — the switch record's helpers (#1674). */
@@ -792,20 +813,30 @@ export class ControlPlaneDO extends DurableObject {
      * construction. A directory restore runs `buildDirectorySchema` inside its own transaction
      * instead (`importDump`).
      *
-     * The schedule switch's record (#1674) is backfilled from the admin log on a run that
-     * creates its table, and its table is created in the SAME transaction as the backfill
-     * (Copilot review): the gate is "the table does not exist yet", so a table committed ahead
-     * of a backfill that then failed would read as already migrated on every later run. On
+     * Each kill switch's record (#1674, and #2029's for a peer) is backfilled from the admin log
+     * on a run that creates its table, and its table is created in the SAME transaction as the
+     * backfill (Copilot review): the gate is "the table does not exist yet", so a table committed
+     * ahead of a backfill that then failed would read as already migrated on every later run. On
      * such a run its statements are held back from the loop below and run with the backfill.
      */
     applyDirectorySchema() {
-        const switchRecordIsNew = !systemSwitchesTableExists(this.kernelSql);
-        this.buildDirectorySchema({ holdSwitchRecord: switchRecordIsNew });
-        if (switchRecordIsNew) {
+        const newRecords = SWITCH_KINDS.filter((kind) => !switchesTableExists(this.kernelSql, kind));
+        const fencesAreNew = !membershipFencesTableExists(this.kernelSql);
+        this.buildDirectorySchema({ holdSwitchRecords: newRecords, holdFences: fencesAreNew });
+        for (const kind of newRecords) {
+            this.ctx.storage.transactionSync(() => {
+                for (const stmt of splitSqlStatements(switchesDdlOf(kind)))
+                    this.sql.exec(stmt);
+                this.sql.exec(switchesBackfillSqlOf(kind));
+            });
+        }
+        // #1184: the removal fence and its backfill from the admin log, together, the same way.
+        if (fencesAreNew) {
             this.ctx.storage.transactionSync(() => {
-                for (const stmt of splitSqlStatements(SYSTEM_SWITCHES_DDL))
+                for (const stmt of splitSqlStatements(MEMBERSHIP_FENCES_DDL))
                     this.sql.exec(stmt);
-                this.sql.exec(SYSTEM_SWITCHES_BACKFILL_SQL);
+                // Stamped now, inside this unit (`membershipFencesBackfillSql` says why).
+                this.sql.exec(membershipFencesBackfillSql(new Date().toISOString()));
             });
         }
     }
@@ -813,12 +844,15 @@ export class ControlPlaneDO extends DurableObject {
      * The directory's tables as this code builds them, carrying whatever the directory already
      * holds forward to that shape: the construction's pass, and a restore's (#1898, #1912), which
      * runs it inside its transaction onto an emptied directory, before loading any row.
-     * `holdSwitchRecord` leaves the #1674 record's statements to the caller, which creates the
-     * table with its backfill.
+     * `holdSwitchRecords` leaves those kinds' record statements (#1674, #2029) to the caller, which
+     * creates each table with its backfill; `holdFences` does the same for #1184's removal fence.
      */
-    buildDirectorySchema({ holdSwitchRecord }) {
+    buildDirectorySchema({ holdSwitchRecords, holdFences = false }) {
+        const held = holdSwitchRecords.map(switchesTableOf);
         for (const stmt of DIRECTORY_DDL_PLAN.loop) {
-            if (holdSwitchRecord && stmt.includes(SYSTEM_SWITCHES_TABLE))
+            if (held.some((table) => stmt.includes(table)))
+                continue;
+            if (holdFences && stmt.includes(MEMBERSHIP_FENCES_TABLE))
                 continue;
             this.sql.exec(stmt);
         }
@@ -1135,12 +1169,13 @@ export class ControlPlaneDO extends DurableObject {
             // Every table from this code's DDL, onto an empty directory, so none of the pass's
             // rebuilds fires (each would reach `transactionSync`, which this async transaction cannot
             // hold): every table is created on the current shape.
-            this.buildDirectorySchema({ holdSwitchRecord: false });
+            this.buildDirectorySchema({ holdSwitchRecords: [] });
             // Refuses a table this code does not build, adds unknown columns bare, loads the rows by
             // name, then the legacy scope and #1674 backfills, all inside this transaction.
             loadDirectoryDump(tables, {
                 columnsOf: (name) => doBuiltColumnsOf(this.sql, name),
                 exec: (sql) => this.sql.exec(sql),
+                now: () => new Date().toISOString(),
                 maxColumns: DO_SQL_LIMITS.columns,
                 insert: (sql, rows) => {
                     for (const row of rows)
@@ -1199,7 +1234,7 @@ export class ControlPlaneDO extends DurableObject {
             .exec('SELECT tenant_id FROM tenants WHERE slug = ?', slug)
             .toArray()[0];
         if (slugOwner) {
-            throw new Error(`tenant slug '${slug}' already taken by ${slugOwner.tenant_id} (slugs are unique)`);
+            throw substratError('conflict', `tenant slug '${slug}' already taken by ${slugOwner.tenant_id} (slugs are unique)`);
         }
         this.sql.exec(`INSERT INTO tenants (tenant_id, slug, name, status, created_at, provisioned_by_tenant)
        VALUES (?, ?, ?, 'active', ?, ?)`, id, slug, name, createdAt, provisionedByTenant);
@@ -1209,12 +1244,12 @@ export class ControlPlaneDO extends DurableObject {
     setTenantStatus(tenantId, status) {
         const before = this.readTenant(tenantId);
         if (!before)
-            throw new Error(`unknown tenant: ${tenantId}`);
+            throw substratError('not_found', `unknown tenant: ${tenantId}`);
         // `reaped` is terminal and destroys data — unreachable here, only ever set by
         // reapTenant, so a plain status flip cannot forge a tombstone over live data
         // (§4.8, the tenant analogue of reapScope's archived-only gate).
         if (status === 'reaped') {
-            throw new Error(`tenant ${tenantId} cannot be set to 'reaped' via setTenantStatus — reap goes through reapTenant (control-plane.md §4.8)`);
+            throw substratError('validation_failed', `tenant ${tenantId} cannot be set to 'reaped' via setTenantStatus — reap goes through reapTenant (control-plane.md §4.8)`);
         }
         // Stamp/clear deleting_at so the grace-window sweep can age tenants (§4.8):
         // entering `deleting` stamps, leaving it (an un-delete) clears — the same shape
@@ -1241,9 +1276,9 @@ export class ControlPlaneDO extends DurableObject {
     reapTenant(tenantId) {
         const before = this.readTenant(tenantId);
         if (!before)
-            throw new Error(`unknown tenant: ${tenantId}`);
+            throw substratError('not_found', `unknown tenant: ${tenantId}`);
         if (before.status !== 'deleting') {
-            throw new Error(`tenant ${tenantId} is ${before.status}, not deleting — only a deleting tenant may be reaped`);
+            throw substratError('conflict', `tenant ${tenantId} is ${before.status}, not deleting — only a deleting tenant may be reaped`);
         }
         // KEPT as the tombstone: the `tenants` row itself and `_substrat_admin_log` (the
         // compliance witness — never swept). Scope rows were already reaped individually.
@@ -1255,6 +1290,11 @@ export class ControlPlaneDO extends DurableObject {
             '_substrat_entitlements', // per-tenant SKU flags
             'orgs', // K-22 org records
             '_substrat_system_switches', // #1674: the schedule switch's record, per scope
+            '_substrat_membership_fences', // #1184: the latest removal, per principal
+            '_substrat_peer_switches', // #2029: the peer switch's record, per scope
+            '_substrat_switch_owed', // #2045: subjects owed a re-assert, per scope
+            '_substrat_findings', // #1748: the tenant's findings
+            '_substrat_finding_rules', // #1748: the tenant's suppress rules
         ]) {
             this.sql.exec(`DELETE FROM ${table} WHERE tenant_id = ?`, tenantId);
         }
@@ -1270,7 +1310,7 @@ export class ControlPlaneDO extends DurableObject {
     setTenantName(tenantId, name) {
         const before = this.readTenant(tenantId);
         if (!before)
-            throw new Error(`unknown tenant: ${tenantId}`);
+            throw substratError('not_found', `unknown tenant: ${tenantId}`);
         this.sql.exec('UPDATE tenants SET name = ? WHERE tenant_id = ?', name, tenantId);
         return before.name;
     }
@@ -1304,10 +1344,10 @@ export class ControlPlaneDO extends DurableObject {
             .exec('SELECT status FROM tenants WHERE tenant_id = ?', row.tenantId)
             .toArray()[0];
         if (!tenantRow) {
-            throw new Error(`cannot provision tenant store under unknown tenant: ${row.tenantId}`);
+            throw substratError('not_found', `cannot provision tenant store under unknown tenant: ${row.tenantId}`);
         }
         if (tenantRow.status !== 'active') {
-            throw new Error(`cannot provision tenant store under non-active tenant (status: ${tenantRow.status}): ${row.tenantId}`);
+            throw substratError('conflict', `cannot provision tenant store under non-active tenant (status: ${tenantRow.status}): ${row.tenantId}`);
         }
         this.sql.exec(`INSERT OR IGNORE INTO tenant_stores (tenant_id, vertical, binding, kind, ref, created_at)
        VALUES (?, ?, ?, ?, ?, ?)`, row.tenantId, row.vertical, row.binding, row.kind, row.ref, row.createdAt);
@@ -1348,10 +1388,10 @@ export class ControlPlaneDO extends DurableObject {
             .exec('SELECT status FROM tenants WHERE tenant_id = ?', row.tenantId)
             .toArray()[0];
         if (!tenantRow) {
-            throw new Error(`cannot provision blob store under unknown tenant: ${row.tenantId}`);
+            throw substratError('not_found', `cannot provision blob store under unknown tenant: ${row.tenantId}`);
         }
         if (tenantRow.status !== 'active') {
-            throw new Error(`cannot provision blob store under non-active tenant (status: ${tenantRow.status}): ${row.tenantId}`);
+            throw substratError('conflict', `cannot provision blob store under non-active tenant (status: ${tenantRow.status}): ${row.tenantId}`);
         }
         this.sql.exec(`INSERT OR IGNORE INTO blob_stores (tenant_id, vertical, binding, kind, ref, created_at)
        VALUES (?, ?, ?, ?, ?, ?)`, row.tenantId, row.vertical, row.binding, row.kind, row.ref, row.createdAt);
@@ -1388,10 +1428,10 @@ export class ControlPlaneDO extends DurableObject {
             .exec('SELECT status FROM tenants WHERE tenant_id = ?', tenantId)
             .toArray()[0];
         if (!tenantRow) {
-            throw new Error(`cannot provision scope under unknown tenant: ${tenantId}`);
+            throw substratError('conflict', `cannot provision scope under unknown tenant: ${tenantId}`);
         }
         if (tenantRow.status !== 'active') {
-            throw new Error(`cannot provision scope under non-active tenant (status: ${tenantRow.status}): ${tenantId}`);
+            throw substratError('conflict', `cannot provision scope under non-active tenant (status: ${tenantRow.status}): ${tenantId}`);
         }
         // Idempotency is on the scope_id (§3.3: idempotent and journaled — safe to
         // re-run), so an existing scope short-circuits before the slug check:
@@ -1404,7 +1444,7 @@ export class ControlPlaneDO extends DurableObject {
                 .exec("SELECT scope_id FROM scopes WHERE tenant_id = ? AND slug = ? AND status NOT IN ('archived', 'reaped')", tenantId, record.slug)
                 .toArray()[0];
             if (slugOwner) {
-                throw new Error(`scope slug '${record.slug}' already taken under tenant ${tenantId} ` +
+                throw substratError('conflict', `scope slug '${record.slug}' already taken under tenant ${tenantId} ` +
                     `by ${slugOwner.scope_id} (slugs are unique within a tenant)`);
             }
             // A scope born while its vertical serves in place (#286) is born ON the serving
@@ -1486,6 +1526,21 @@ export class ControlPlaneDO extends DurableObject {
             return undefined;
         return row;
     }
+    /**
+     * A directory write, its refusal answered as DATA (#113): the typed throws in these methods
+     * would reach the coordinator flattened, so it calls them through here and rethrows the
+     * refusal with its code (`unwrapReply`). Called directly, they still throw — a coordinator
+     * from before this change does — and that caller reads the code folded into the message.
+     */
+    async reply(method, args) {
+        return replyOf(() => {
+            // An RPC argument, so held to the list rather than trusted to be one of its names.
+            if (!REPLIED_METHODS.includes(method))
+                throw new Error(`not a replied directory method: ${String(method)}`);
+            const run = this[method];
+            return run.apply(this, args);
+        });
+    }
     /**
      * The getScope gate (control-plane.md §4.1/§4.2): validate the scope belongs
      * to the tenant and both records are active, or throw the fail-closed reason.
@@ -1502,11 +1557,10 @@ export class ControlPlaneDO extends DurableObject {
      * The same gate, answering its refusal as DATA (#1718). An error thrown in a Durable
      * Object arrives at the coordinator flattened, its code gone, so a typed throw here
      * would still reach the caller untyped (#1714 measured it). A record crosses intact, so
-     * the coordinator throws the refusal itself, typed. `code` is set only where a code is
-     * known; the lifecycle refusals keep their bare messages. Null means access is allowed.
+     * the coordinator throws the refusal itself, typed (#113: every refusal but a failed
+     * migration carries its code). Null means access is allowed.
      */
     scopeAccessRefusal(tenantId, scopeId) {
-        const refuse = (message) => ({ code: null, message });
         const row = this.sql
             .exec('SELECT tenant_id, status FROM scopes WHERE scope_id = ?', scopeId)
             .toArray()[0];
@@ -1517,10 +1571,14 @@ export class ControlPlaneDO extends DurableObject {
             .exec('SELECT status FROM tenants WHERE tenant_id = ?', tenantId)
             .toArray()[0];
         if (!tenantRow) {
-            return refuse(`scope has no tenant record: (${tenantId}, ${scopeId})`);
+            return { code: 'not_found', message: `scope has no tenant record: (${tenantId}, ${scopeId})`, reason: SCOPE_GATE_REASONS.unrecorded };
         }
         if (tenantRow.status !== 'active') {
-            return refuse(`tenant not active (status: ${tenantRow.status}): ${tenantId}`);
+            return {
+                code: 'conflict',
+                message: `tenant not active (status: ${tenantRow.status}): ${tenantId}`,
+                reason: SCOPE_GATE_REASONS.notActive,
+            };
         }
         if (row.status !== 'active') {
             // A scope stuck in provisioning because its migrations failed must say so.
@@ -1531,10 +1589,17 @@ export class ControlPlaneDO extends DurableObject {
                 .exec('SELECT migration_failed_version, migration_error FROM scopes WHERE scope_id = ?', scopeId)
                 .toArray()[0];
             if (failure?.migration_failed_version) {
-                return refuse(`migration failed for ${failure.migration_failed_version} — scope fails closed: ` +
-                    `${failure.migration_error ?? 'unknown error'}`);
+                return {
+                    code: null,
+                    message: `migration failed for ${failure.migration_failed_version} — scope fails closed: ` +
+                        `${failure.migration_error ?? 'unknown error'}`,
+                };
             }
-            return refuse(`scope not active (status: ${row.status}): ${scopeId}`);
+            return {
+                code: 'conflict',
+                message: `scope not active (status: ${row.status}): ${scopeId}`,
+                reason: SCOPE_GATE_REASONS.notActive,
+            };
         }
         return null;
     }
@@ -1569,7 +1634,7 @@ export class ControlPlaneDO extends DurableObject {
         if (!from.includes(row.status)) {
             return {
                 ok: false,
-                code: null,
+                code: 'conflict',
                 message: `illegal scope transition for ${action}: ${row.status} → ${to} ` +
                     `(allowed from: ${from.join('|')})`,
             };
@@ -1590,7 +1655,7 @@ export class ControlPlaneDO extends DurableObject {
                 this.sql.exec('UPDATE scopes SET status = ? WHERE scope_id = ?', to, scopeId);
             }
             if (to === 'reaped') {
-                forgetSystemSwitchesOf(this.kernelSql, scopeId);
+                forgetSwitchesOf(this.kernelSql, scopeId);
                 this.forgetLifecycleDeliveries('scope_id = ?', scopeId);
             }
             this.bumpLifecycleRevision(`scope:${scopeId}`); // #1713, with the status it counts
@@ -1998,6 +2063,11 @@ export class ControlPlaneDO extends DurableObject {
      * for a repair that failed, which is the one outcome that must keep being retried.
      */
     markScopeProvisioned(scopeId, versionId) {
+        // #2045: a scope owed a switch re-assert keeps no receipt, so the sweep reconciles it again
+        // until its re-assert has settled every mark — including a mark written while this
+        // reconcile ran, after its re-assert had read none.
+        if (versionId !== null && scopeOwesSwitch(this.kernelSql, scopeId))
+            versionId = null;
         this.sql.exec('UPDATE scopes SET provisioned_version_id = ? WHERE scope_id = ?', versionId, scopeId);
     }
     /**
@@ -2008,7 +2078,7 @@ export class ControlPlaneDO extends DurableObject {
     deleteScopeDirectory(scopeId) {
         this.ctx.storage.transactionSync(() => {
             this.sql.exec('DELETE FROM hostnames WHERE scope_id = ?', scopeId);
-            forgetSystemSwitchesOf(this.kernelSql, scopeId);
+            forgetSwitchesOf(this.kernelSql, scopeId);
             this.forgetLifecycleDeliveries('scope_id = ?', scopeId);
             this.sql.exec('DELETE FROM scopes WHERE scope_id = ?', scopeId);
         });
@@ -2075,7 +2145,7 @@ export class ControlPlaneDO extends DurableObject {
             .exec('SELECT org_id FROM orgs WHERE tenant_id = ? AND slug = ?', tenantId, slug)
             .toArray()[0];
         if (slugOwner) {
-            throw new Error(`org slug '${slug}' already taken by ${slugOwner.org_id} (slugs are unique per tenant)`);
+            throw substratError('conflict', `org slug '${slug}' already taken by ${slugOwner.org_id} (slugs are unique per tenant)`);
         }
         this.sql.exec('INSERT INTO orgs (org_id, tenant_id, slug, name, created_at) VALUES (?, ?, ?, ?, ?)', orgId, tenantId, slug, name, createdAt);
         return true;
@@ -2085,13 +2155,89 @@ export class ControlPlaneDO extends DurableObject {
             .exec('SELECT * FROM orgs WHERE tenant_id = ? ORDER BY slug', tenantId)
             .toArray();
     }
+    /** The tenant-level reads, synchronous, for a bound asked inside one of this object's units (#1184). */
+    directoryReader = {
+        now: () => new Date().toISOString(),
+        tenantTuples: (tenantId, subject, prefix) => this.tenantTuples(tenantId, subject, prefix),
+        getRole: (tenantId, key) => this.getRole(tenantId, key),
+    };
+    /**
+     * A membership change as ONE DO unit (#1184) — `HostAdmin.applyMembership`. Synchronous, so
+     * no other request into this object runs between its reads and its writes: an add's removal
+     * fence, then the bound — `boundedBy` covering every permission `roleKey` carries, against the
+     * tuples and the role as they stand now (`tenantCoverage`) — then the write: the TENANT role
+     * and its audit row for an add, `revokeAndFence`'s three for a removal. The coordinator mints
+     * the rows (id, actor, attribution, `causedBy`): the role's, and the org's when the change
+     * names one (#2047) — then the org is bounded here too (`boundedBy` a live member of it) and
+     * joined or left in the same unit, a join expiring no later than `boundedBy`'s own membership.
+     * The org row's `before`/`after` are written here, since only the unit knows the expiry.
+     */
+    applyMembership(change, row, orgRow) {
+        const { tenantId, principal, roleKey, boundedBy, orgId } = change;
+        return this.ctx.storage.transactionSync(() => {
+            if (change.op === 'add') {
+                const fence = this.sql.exec(MEMBERSHIP_FENCE_SINCE_SQL, tenantId, principal, change.unlessRemovedSince).toArray()[0];
+                if (fence)
+                    return { applied: false, removedAt: fence.removed_at };
+            }
+            const role = this.getRole(tenantId, roleKey);
+            if (!role)
+                return { applied: false, unknownRole: roleKey };
+            const bound = tenantCoverage(this.directoryReader, tenantId, boundedBy, role.permissions);
+            if (!bound.covered)
+                return { applied: false, missing: bound.missing };
+            // #2047: the org, bounded by `boundedBy`'s own live membership of it.
+            const org = orgId ? { orgId, row: orgRow, ...orgChangeBound(this.directoryReader, tenantId, { ...change, orgId }) } : undefined;
+            if (org) {
+                if (!org.row)
+                    throw new Error('applyMembership: an org change needs its audit row');
+                if (!this.readOrg(tenantId, org.orgId))
+                    return { applied: false, unknownOrg: org.orgId };
+                if (!org.bounded)
+                    return { applied: false, notMember: org.orgId };
+            }
+            if (change.op === 'remove') {
+                const roleTaken = this.revokeAndFenceIn(tenantId, principal, `role:${roleKey}`, `tenant:${tenantId}`, row);
+                const orgLeft = !!org && this.revokeAndFenceIn(tenantId, principal, 'member', `org:${org.orgId}`, { ...org.row, before: { principal, orgId: org.orgId } });
+                return { applied: true, changed: roleTaken || orgLeft };
+            }
+            const at = new Date().toISOString();
+            this.writeTenantTuple(tenantId, `principal:${principal}`, `role:${roleKey}`, `tenant:${tenantId}`, null);
+            this.recordAdmin({ ...row, at });
+            if (org?.bounded) {
+                this.writeTenantTuple(tenantId, `principal:${principal}`, 'member', `org:${org.orgId}`, org.expiresAt);
+                this.recordAdmin({ ...org.row, after: memberAddedAudit(principal, org.orgId, org.expiresAt), at });
+            }
+            return { applied: true };
+        });
+    }
+    /**
+     * A tenant-level removal as ONE DO unit (#1184) — `unassignRole` at the tenant node, and
+     * `removeMember`: the tombstone, the principal's removal fence, and the audit row, which
+     * K-21 writes only when something changed. The fence is raised either way: a removal of
+     * someone whose add is still on its way takes nothing, and must still win. One method, so
+     * no add can run between the revoke and the fence. Returns whether anything changed.
+     */
+    revokeAndFence(tenantId, principal, relation, object, row) {
+        return this.ctx.storage.transactionSync(() => this.revokeAndFenceIn(tenantId, principal, relation, object, row));
+    }
     /**
-     * Tombstone a membership (K-21), never DELETE. Guarded on `revoked_at IS NULL`
-     * so a repeat revoke neither moves the timestamp nor produces a second audit
-     * row. Returns whether it changed, so the coordinator can skip the audit write.
+     * `revokeAndFence`'s writes, for inside a transaction a caller already holds.
+     *
+     * The time is minted HERE, in the unit, never taken from `row.at`: the coordinator mints its
+     * row before an awaited RPC, and a queue delay in between would age the fence by however
+     * long the call waited, which an add's cutoff would read as a removal from before its window
+     * (Codex round 3). The fence is compared with an add's `occurredAt`, stamped by the ScopeDO's
+     * clock. Both are this platform's own NTP-disciplined time, sub-second apart, which is far
+     * inside `MEMBERSHIP_REMOVAL_SKEW_MS`.
      */
-    revokeMember(tenantId, subject, object, at) {
-        return this.revokeTenantTuple(tenantId, subject, 'member', object, at);
+    revokeAndFenceIn(tenantId, principal, relation, object, row) {
+        const at = new Date().toISOString();
+        const changed = this.revokeTenantTuple(tenantId, `principal:${principal}`, relation, object, at);
+        this.sql.exec(RAISE_MEMBERSHIP_FENCE_SQL, tenantId, principal, at);
+        if (changed)
+            this.recordAdmin({ ...row, at });
+        return changed;
     }
     /** Tombstone any tenant tuple by its exact (subject, relation, object). Returns
      *  whether a live row changed, so a repeat revoke is a silent no-op. */
@@ -2111,7 +2257,7 @@ export class ControlPlaneDO extends DurableObject {
     /** Members of an org. Live only unless `includeRevoked` — revoked rows are evidence. */
     listMembers(tenantId, object, includeRevoked) {
         return this.sql
-            .exec(`SELECT subject, revoked_at FROM _substrat_tenant_tuples
+            .exec(`SELECT subject, revoked_at, expires_at FROM _substrat_tenant_tuples
          WHERE tenant_id = ? AND relation = 'member' AND object = ?
          ${includeRevoked ? '' : 'AND revoked_at IS NULL'}
          ORDER BY subject`, tenantId, object)
@@ -2248,7 +2394,7 @@ export class ControlPlaneDO extends DurableObject {
         if (existing) {
             if (existing.topology === topology && existing.tenant_id === tenantId)
                 return false;
-            throw new Error(`identity pool '${provider}' is already registered as ${existing.topology}` +
+            throw substratError('conflict', `identity pool '${provider}' is already registered as ${existing.topology}` +
                 `${existing.tenant_id ? ` for tenant ${existing.tenant_id}` : ''}`);
         }
         this.sql.exec('INSERT INTO _substrat_identity_pools (provider, topology, tenant_id, created_at) VALUES (?, ?, ?, ?)', provider, topology, tenantId, createdAt);
@@ -2451,38 +2597,74 @@ export class ControlPlaneDO extends DurableObject {
           granted_by, granted_at, revoked_at)
        VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL)`, row.connectionId, row.tenantId, row.vertical, row.permission, row.scopeId, row.expiresAt, row.grantedBy, row.grantedAt);
     }
-    // -- the schedule switch's record (#1674) — `system-switch-record.ts` is the whole rule ---
+    // -- the kill switches' record (#1674, #2029) — `system-switch-record.ts` is the whole rule ---
     /** OFF's write, before the scope's switch moves (#1823). Answers the row as it was. */
-    recordSystemSwitchedOff(row) {
-        return recordSystemSwitchedOff(this.kernelSql, row);
+    recordSwitchedOff(row) {
+        return this.writeSwitchIntent(row, recordSwitchedOff);
     }
-    /** ON's write, before the scope moves. Answers the row as it was, for a failed ON to restore. */
-    recordSystemSwitchedOn(row) {
-        return recordSystemSwitchedOn(this.kernelSql, row);
+    /** ON's write, before the scope moves. Answers the row as it was, for a call that held nothing to restore. */
+    recordSwitchedOn(row) {
+        return this.writeSwitchIntent(row, recordSwitchedOn);
+    }
+    /**
+     * #2045 (Codex r3): a switch call's record write is also its write-ahead intent, in ONE
+     * transaction: the record, the subject's owed mark under this call's id, and the scope's reconcile
+     * receipt cleared, so the sweep reconciles it and its re-assert settles the mark. Nothing that
+     * happens to the move after this (a throw, a lost answer, a failed clear) can leave the scope
+     * apart from its record with no mark to repair it. A write a newer call's row refuses writes
+     * nothing at all.
+     */
+    writeSwitchIntent(row, write) {
+        return this.ctx.storage.transactionSync(() => {
+            const prior = write(this.kernelSql, row);
+            if (recordWriteSuperseded(prior, row))
+                return prior;
+            markSwitchOwed(this.kernelSql, row.kind, row.tenantId, row.scopeId, row.key, row.operationId);
+            this.sql.exec('UPDATE scopes SET provisioned_version_id = NULL WHERE scope_id = ?', row.scopeId);
+            return prior;
+        });
     }
-    /** Put a row back as the switch call's record write found it — the move that followed it failed. */
-    restoreSystemSwitchRecord(key, prior) {
-        restoreSystemSwitchRecord(this.kernelSql, key, prior);
+    /**
+     * Put a row back as the switch call's record write found it — the move that followed it held
+     * nothing. #2045: with `clearOwed`, the call's own owed mark goes in the same transaction. The
+     * row put back carries the PRIOR call's operation id, which is the fence every later re-assert
+     * confirms under, and a mark newer than that fence is never cleared by one; so a mark left behind
+     * by a separate clear that failed would keep the scope un-receipted on every pass.
+     */
+    restoreSwitchRecord(key, prior, clearOwed = false) {
+        this.ctx.storage.transactionSync(() => {
+            restoreSwitchRecord(this.kernelSql, key, prior);
+            if (clearOwed)
+                clearSwitchOwed(this.kernelSql, key.kind, key.tenantId, key.scopeId, key.key, key.operationId);
+        });
     }
     /**
-     * #1823: which of these modules the tenant holds a live TENANT-level grant for — what the
-     * switch passes the scope as `tenantHeld`, so a module whose only authority there is a tenant
-     * tuple is still switchable. Read here, where the tenant tuples live.
+     * #1823 (#2030 for a peer): which of these subjects the tenant holds a live TENANT-level grant
+     * for — what the switch passes the scope as `tenantHeld`, so a subject whose only authority
+     * there is a tenant tuple is still switchable. Read here, where the tenant tuples live.
      */
-    tenantHeldSystemModules(tenantId, moduleIds, now) {
-        return moduleIds.filter((m) => tenantHoldsSystemGrant(this.kernelSql, tenantId, m, now));
+    tenantHeldOf(kind, tenantId, keys, now) {
+        return tenantHeldOf(this.kernelSql, kind, tenantId, keys, now);
     }
     /** The fleet read. */
     listSystemSwitches(filter) {
         return listSystemSwitchRecords(this.kernelSql, filter);
     }
-    /** One scope's recorded positions, by module — the status read's `recorded` join. */
-    systemSwitchRecordsOf(tenantId, scopeId) {
-        return [...systemSwitchRecordsOf(this.kernelSql, tenantId, scopeId)];
+    /** One scope's recorded positions of one kind, by subject — the status read's `recorded` join. */
+    switchRecordsOf(kind, tenantId, scopeId) {
+        return [...switchRecordsOf(this.kernelSql, kind, tenantId, scopeId)];
+    }
+    /** #2045: each record row's position and fence, by subject, read together — `switchRecordStatesOf`. */
+    switchRecordStatesOf(kind, tenantId, scopeId) {
+        return [...switchRecordStatesOf(this.kernelSql, kind, tenantId, scopeId)];
     }
-    /** The modules a re-assert switches back off on one scope. */
-    switchedOffModulesOf(tenantId, scopeId) {
-        return switchedOffModulesOf(this.kernelSql, tenantId, scopeId);
+    /** #2045: the subjects of one kind owed a re-assert on one scope. */
+    switchesOwedOf(kind, tenantId, scopeId) {
+        return [...switchesOwedOf(this.kernelSql, kind, tenantId, scopeId)];
+    }
+    /** #2045: the scope confirmed a move of this subject under `fence` — `clearSwitchOwed`. */
+    clearSwitchOwed(kind, tenantId, scopeId, key, fence) {
+        clearSwitchOwed(this.kernelSql, kind, tenantId, scopeId, key, fence);
     }
     /** The tenant's LIVE connection grants (#592) — the provision/reconcile gather read. */
     listConnectionGrants(tenantId) {
@@ -2546,7 +2728,7 @@ export class ControlPlaneDO extends DurableObject {
         if (existing) {
             if (existing.principal_id === principal)
                 return false;
-            throw new Error(`identity ${provider}:${externalId} in tenant ${tenantId} is already bound to ${existing.principal_id}`);
+            throw substratError('conflict', `identity ${provider}:${externalId} in tenant ${tenantId} is already bound to ${existing.principal_id}`);
         }
         this.sql.exec(`INSERT INTO _substrat_identities
          (provider, external_id, principal_id, tenant_id, scope_id, created_at)
@@ -2871,35 +3053,51 @@ export class ControlPlaneDO extends DurableObject {
        SELECT ?, ?, ?, ? WHERE ${STILL_DELIVERED}`, scopeId, delivered, at, tenantRecorded ? 1 : 0, scopeId);
     }
     recordOpsFailure(row) {
-        this.sql.exec(`INSERT INTO _substrat_ops_failures
-         (id, actor, operation, stage, tenant_id, scope_id, vertical, version, status, message, reference, origin, code, fingerprint, at)
-       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, row.id, row.actor, row.operation, row.stage, row.tenant_id, row.scope_id, row.vertical, row.version, row.status, row.message, row.reference, row.origin, row.code, row.fingerprint, row.at);
-        // Prune-on-write (#559): the retention lives HERE, not in a cron — every insert
-        // pays for its own housekeeping, so the table stays bounded even on a deployment
-        // whose scheduled pass is broken (the exact circumstance this table records).
-        const horizon = new Date(Date.now() - OPS_FAILURE_RETENTION_DAYS * 86_400_000).toISOString();
-        this.sql.exec('DELETE FROM _substrat_ops_failures WHERE at < ?', horizon);
-        // The issues materialization (#1233): the group's counters live on their own
-        // row, bumped in the same call, because the evidence self-prunes above and a
-        // count must survive its own exemplars. A fresh arrival regresses a resolved
-        // issue; an ignored one stays ignored — that is what ignoring means.
-        if (row.fingerprint !== null) {
-            this.sql.exec(`INSERT INTO _substrat_issues
-           (fingerprint, operation, stage, origin, code, status, seen_count, first_seen, last_seen, last_message, last_tenant_id, last_owner_kind, last_vertical, last_version, resolved_version, resolved_at)
-         VALUES (?, ?, ?, ?, ?, 'new', 1, ?, ?, ?, ?, ?, ?, ?, NULL, NULL)
-         ON CONFLICT (fingerprint) DO UPDATE SET
-           seen_count = seen_count + 1,
-           last_seen = excluded.last_seen,
-           last_message = excluded.last_message,
-           last_tenant_id = excluded.last_tenant_id,
-           last_owner_kind = excluded.last_owner_kind,
-           last_vertical = COALESCE(excluded.last_vertical, last_vertical),
-           last_version = COALESCE(excluded.last_version, last_version),
-           origin = COALESCE(excluded.origin, origin),
-           status = CASE WHEN status = 'resolved' THEN 'regressed' ELSE status END`, row.fingerprint, row.operation, row.stage, row.origin, row.code, row.at, row.at, row.message, row.tenant_id, issueExemplarOwner(row.tenant_id), row.vertical, row.version);
-            const issueHorizon = new Date(Date.now() - ISSUE_RETENTION_DAYS * 86_400_000).toISOString();
-            this.sql.exec('DELETE FROM _substrat_issues WHERE last_seen < ?', issueHorizon);
-        }
+        // #1748: the evidence, its issue and its finding in ONE unit — a detector that throws takes
+        // the evidence with it, so the caller's retry observes it rather than losing the observation.
+        this.ctx.storage.transactionSync(() => {
+            this.sql.exec(`INSERT INTO _substrat_ops_failures
+           (id, actor, operation, stage, tenant_id, scope_id, vertical, version, status, message, reference, origin, code, fingerprint, at)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, row.id, row.actor, row.operation, row.stage, row.tenant_id, row.scope_id, row.vertical, row.version, row.status, row.message, row.reference, row.origin, row.code, row.fingerprint, row.at);
+            // Prune-on-write (#559): the retention lives HERE, not in a cron — every insert
+            // pays for its own housekeeping, so the table stays bounded even on a deployment
+            // whose scheduled pass is broken (the exact circumstance this table records).
+            const horizon = new Date(Date.now() - OPS_FAILURE_RETENTION_DAYS * 86_400_000).toISOString();
+            this.sql.exec('DELETE FROM _substrat_ops_failures WHERE at < ?', horizon);
+            // The issues materialization (#1233): the group's counters live on their own
+            // row, bumped in the same call, because the evidence self-prunes above and a
+            // count must survive its own exemplars. A fresh arrival regresses a resolved
+            // issue; an ignored one stays ignored — that is what ignoring means.
+            if (row.fingerprint !== null) {
+                this.sql.exec(`INSERT INTO _substrat_issues
+             (fingerprint, operation, stage, origin, code, status, seen_count, first_seen, last_seen, last_message, last_tenant_id, last_owner_kind, last_vertical, last_version, resolved_version, resolved_at)
+           VALUES (?, ?, ?, ?, ?, 'new', 1, ?, ?, ?, ?, ?, ?, ?, NULL, NULL)
+           ON CONFLICT (fingerprint) DO UPDATE SET
+             seen_count = seen_count + 1,
+             last_seen = excluded.last_seen,
+             last_message = excluded.last_message,
+             last_tenant_id = excluded.last_tenant_id,
+             last_owner_kind = excluded.last_owner_kind,
+             last_vertical = COALESCE(excluded.last_vertical, last_vertical),
+             last_version = COALESCE(excluded.last_version, last_version),
+             origin = COALESCE(excluded.origin, origin),
+             status = CASE WHEN status = 'resolved' THEN 'regressed' ELSE status END`, row.fingerprint, row.operation, row.stage, row.origin, row.code, row.at, row.at, row.message, row.tenant_id, issueExemplarOwner(row.tenant_id), row.vertical, row.version);
+                const issueHorizon = new Date(Date.now() - ISSUE_RETENTION_DAYS * 86_400_000).toISOString();
+                this.sql.exec('DELETE FROM _substrat_issues WHERE last_seen < ?', issueHorizon);
+                // #1748: the tenant's Recurring finding, over the tenant's own occurrences only.
+                const finding = findingOfOpsFailure({
+                    tenantId: row.tenant_id,
+                    scopeId: row.scope_id,
+                    operation: row.operation,
+                    stage: row.stage,
+                    code: row.code,
+                    vertical: row.vertical,
+                    version: row.version,
+                });
+                if (finding)
+                    observeFinding(doRedactionSql(this.sql), finding, row.at);
+            }
+        });
     }
     /**
      * #1632: subject erasure's directory half — the failure text this directory keeps about a
@@ -2985,14 +3183,35 @@ export class ControlPlaneDO extends DurableObject {
         }));
     }
     recordSweepRun(row) {
-        this.sql.exec(`INSERT OR IGNORE INTO _substrat_sweep_runs
-         (id, kind, unit, outcome, tenant_id, scope_id, vertical, version, operation,
-          connection_id, error, elapsed_ms, request_id, event_type, observed_at, platform_requests, at)
-       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, row.id, row.kind, row.unit, row.outcome, row.tenant_id, row.scope_id, row.vertical, row.version, row.operation, row.connection_id, row.error, row.elapsed_ms, row.request_id, row.event_type, row.observed_at, row.platform_requests, row.at);
-        // Prune-on-write, like ops failures and for the same reason: bounded even on a
-        // deployment whose scheduled pass is broken.
-        const horizon = new Date(Date.now() - SWEEP_RUN_RETENTION_DAYS * 86_400_000).toISOString();
-        this.sql.exec('DELETE FROM _substrat_sweep_runs WHERE at < ?', horizon);
+        // #1748: the row and its finding in ONE unit, so a replayed drain the unique index ignores
+        // never meets a row whose observation did not commit.
+        this.ctx.storage.transactionSync(() => {
+            const { rowsWritten } = this.sql.exec(`INSERT OR IGNORE INTO _substrat_sweep_runs
+           (id, kind, unit, outcome, tenant_id, scope_id, vertical, version, operation,
+            connection_id, error, elapsed_ms, request_id, event_type, observed_at, platform_requests, at)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, row.id, row.kind, row.unit, row.outcome, row.tenant_id, row.scope_id, row.vertical, row.version, row.operation, row.connection_id, row.error, row.elapsed_ms, row.request_id, row.event_type, row.observed_at, row.platform_requests, row.at);
+            // Prune-on-write, like ops failures and for the same reason: bounded even on a
+            // deployment whose scheduled pass is broken.
+            const horizon = new Date(Date.now() - SWEEP_RUN_RETENTION_DAYS * 86_400_000).toISOString();
+            this.sql.exec('DELETE FROM _substrat_sweep_runs WHERE at < ?', horizon);
+            // #1748: a failed schedule (Invariant) or a stale freshness verdict (Drift). Only when the
+            // row was written: a replayed drain the unique index ignored is not a second occurrence.
+            const finding = rowsWritten > 0
+                ? findingOfSweepRun({
+                    kind: row.kind,
+                    unit: row.unit,
+                    outcome: row.outcome,
+                    tenantId: row.tenant_id,
+                    scopeId: row.scope_id,
+                    vertical: row.vertical,
+                    version: row.version,
+                    operation: row.operation,
+                    eventType: row.event_type,
+                })
+                : null;
+            if (finding)
+                observeFinding(doRedactionSql(this.sql), finding, row.at);
+        });
     }
     listSweepRuns(query) {
         const where = [];
@@ -3062,6 +3281,47 @@ export class ControlPlaneDO extends DurableObject {
         }
         return pruned;
     }
+    /** #1748: the findings retention pass, in one unit with its audit rows. */
+    pruneFindings(limit, audit) {
+        assertRowLimit('limit', limit);
+        return this.audited(audit, (sql, write) => pruneFindings(sql, Date.now(), limit, write));
+    }
+    /**
+     * #1748: a findings mutation and the audit rows it hands back, in ONE unit, so an audit write
+     * that fails rolls the mutation back. `audit` is the row the host minted, with its actor and
+     * attribution; each row the mutation hands back is written as a copy under its own id.
+     */
+    audited(audit, run) {
+        return this.ctx.storage.transactionSync(() => run(doRedactionSql(this.sql), (a) => this.recordAdmin({
+            ...audit,
+            id: ulid(),
+            action: a.action,
+            tenantId: a.target.tenantId,
+            vertical: a.target.vertical,
+            before: a.before,
+            after: a.after,
+        })));
+    }
+    /** #1748: findings, most recently seen first — `listFindings`. */
+    listFindings(filter) {
+        return listFindings(doRedactionSql(this.sql), filter);
+    }
+    /** #1748: a verdict on one tenant finding, audited in the same unit. Undefined for an unknown one. */
+    setFindingStatus(tenantId, id, status, at, audit) {
+        return this.audited(audit, (sql, write) => setFindingStatus(sql, tenantId, id, status, at, write));
+    }
+    /** #1748: a suppress rule, applied to the findings it covers now, audited in the same unit. */
+    createFindingRule(tenantId, input, createdBy, at, audit) {
+        return this.audited(audit, (sql, write) => createFindingRule(sql, tenantId, input, createdBy, at, write));
+    }
+    /** #1748: end a rule now, audited in the same unit. */
+    revokeFindingRule(tenantId, ruleId, at, audit) {
+        return this.audited(audit, (sql, write) => revokeFindingRule(sql, tenantId, ruleId, at, write));
+    }
+    /** #1748: a tenant's suppress rules. */
+    listFindingRules(tenantId, activeAt, limit) {
+        return listFindingRules(doRedactionSql(this.sql), tenantId, activeAt, limit);
+    }
     listIssues(query) {
         const where = [];
         const params = [];