@substrat-run/adapter-cloudflare 0.138.0 → 0.140.0

package/dist/control-plane-do.js

@@ -1,7 +1,8 @@
 import { DurableObject } from 'cloudflare:workers';
-import { membershipFencesBackfillSql, MEMBERSHIP_FENCES_DDL, MEMBERSHIP_FENCES_TABLE, membershipFencesTableExists, MEMBERSHIP_FENCE_SINCE_SQL, RAISE_MEMBERSHIP_FENCE_SQL, tenantCoverage, IMPERSONATION_COLUMNS, IMPERSONATION_DDL, impersonationByIdQuery, impersonationListQuery, impersonationRowValues, ISSUE_RETENTION_DAYS, telemetryRetentionStatements, assertRowLimit, OPS_FAILURE_RETENTION_DAYS, SWEEP_RUN_RETENTION_DAYS, SWEEP_RUNS_INTENT_INDEX, sweepRunsIntentHasKind, PEER_SWITCHES_DDL, SWITCH_OWED_DDL, clearSwitchOwed, markSwitchOwed, scopeOwesSwitch, switchesOwedOf, SWITCH_KINDS, SYSTEM_SWITCHES_DDL, forgetSwitchesOf, listSystemSwitchRecords, recordSwitchedOff, recordSwitchedOn, recordWriteSuperseded, restoreSwitchRecord, scopesSwitchedOffFor, switchRecordStatesOf, switchRecordsOf, switchesBackfillSqlOf, switchesDdlOf, switchesTableExists, switchesTableOf, tenantHeldOf, VERSION_MIGRATIONS_DDL, splitVersionMigrationsBatch, versionMigrationsOf, versionsAwaitSplit, writeVersionMigrations, MODEL_USAGE_RETENTION_DAYS, DO_SQL_LIMITS, ulid, isPrimaryScopeRow, resolveVerticalInstanceFrom, LEGACY_SCOPE_ROWS_BACKFILL, loadDirectoryDump, redactSubjectDirectoryText, ISSUE_EXEMPLAR_OWNER_BACKFILL_SQL, issueExemplarOwner, } from '@substrat-run/kernel';
+import { membershipFencesBackfillSql, MEMBERSHIP_FENCES_DDL, MEMBERSHIP_FENCES_TABLE, membershipFencesTableExists, MEMBERSHIP_FENCE_SINCE_SQL, RAISE_MEMBERSHIP_FENCE_SQL, memberAddedAudit, orgChangeBound, tenantCoverage, IMPERSONATION_COLUMNS, IMPERSONATION_DDL, impersonationByIdQuery, impersonationListQuery, impersonationRowValues, ISSUE_RETENTION_DAYS, telemetryRetentionStatements, FINDINGS_DDL, CONNECT_LINKS_DDL, consumeConnectLinkRow, insertConnectLink as insertConnectLinkRow, listConnectLinks as listConnectLinkRows, readConnectLink as readConnectLinkRow, restoreConnectLinkRow, revokeConnectLinkRow, createFindingRule, findingOfOpsFailure, findingOfSweepRun, pruneFindings, listFindingRules, listFindings, observeFinding, revokeFindingRule, setFindingStatus, assertRowLimit, OPS_FAILURE_RETENTION_DAYS, SWEEP_RUN_RETENTION_DAYS, SWEEP_RUNS_INTENT_INDEX, sweepRunsIntentHasKind, PEER_SWITCHES_DDL, PEER_BINDINGS_DDL, SWITCH_OWED_DDL, clearSwitchOwed, markSwitchOwed, scopeOwesSwitch, switchesOwedOf, SWITCH_KINDS, SYSTEM_SWITCHES_DDL, forgetSwitchesOf, listSystemSwitchRecords, recordSwitchedOff, recordSwitchedOn, recordWriteSuperseded, restoreSwitchRecord, scopesSwitchedOffFor, switchRecordStatesOf, switchRecordsOf, switchesBackfillSqlOf, switchesDdlOf, switchesTableExists, switchesTableOf, tenantHeldOf, VERSION_MIGRATIONS_DDL, splitVersionMigrationsBatch, versionMigrationsOf, versionsAwaitSplit, writeVersionMigrations, MODEL_USAGE_RETENTION_DAYS, DO_SQL_LIMITS, ulid, isPrimaryScopeRow, resolveVerticalInstanceFrom, resolvePeerInstanceFrom, LEGACY_SCOPE_ROWS_BACKFILL, loadDirectoryDump, redactSubjectDirectoryText, ISSUE_EXEMPLAR_OWNER_BACKFILL_SQL, issueExemplarOwner, SETTLE_INTENT_SQL, SETTLE_OUTCOME_SQL, settleOutcomeParamsOf, unknownOutcomeOf, ADMIN_LOG_INDEX_DDL, ADMIN_LOG_INDEXES_SQL, readAuditedOperations, } from '@substrat-run/kernel';
 import { replyOf } from './do-reply.js';
-import { splitSqlStatements, switchSqlOver } from './scope-do.js';
+import { switchSqlOver } from './scope-do.js';
+import { blankSqlComments, executableSqlStatements } from '@substrat-run/kernel';
 import { doBuiltColumnsOf, doRedactionSql } from './sql.js';
 import { assertReplayableDump, opsFailureFingerprint, ROUTE_SCOPE_HOSTNAMES_MAX, SCOPE_GATE_REASONS, substratError } from '@substrat-run/contracts';
 /**
@@ -159,6 +160,7 @@ const DIRECTORY_DDL = `
     serving_ref TEXT,
     created_at TEXT NOT NULL
   );
+  ${PEER_BINDINGS_DDL}
   CREATE TABLE IF NOT EXISTS hostnames (
     hostname      TEXT PRIMARY KEY,
     tenant_id     TEXT NOT NULL,
@@ -416,6 +418,9 @@ const DIRECTORY_DDL = `
     updated_at    TEXT NOT NULL,
     PRIMARY KEY (connection_id, state_key)
   );
+  -- A vertical's mailed connect links (connections.md §3.5.4) — kernel-owned DDL, so
+  -- both adapters build the same table.
+  ${CONNECT_LINKS_DDL}
   CREATE TABLE IF NOT EXISTS _substrat_identities (
     provider     TEXT NOT NULL,
     external_id  TEXT NOT NULL,
@@ -478,16 +483,12 @@ const DIRECTORY_DDL = `
     on_behalf_of TEXT,
     at TEXT NOT NULL
   );
-  -- Read-path indexes for the console (control-plane.md §4.5). The admin log is
-  -- append-only and only grows, so every filter it offers needs one.
+  -- Every admin-log index, from the kernel's one list (#2064), which the legacy
+  -- rebuild runs again after its rename.
   -- NOTE: keep every comment in this DDL free of semicolons. The constructor
   -- splits this string on the statement separator, and a semicolon inside a
   -- comment strands a comment-only fragment that execs as "no statement".
-  CREATE INDEX IF NOT EXISTS _substrat_admin_log_tenant ON _substrat_admin_log (tenant_id, id);
-  CREATE INDEX IF NOT EXISTS _substrat_admin_log_scope ON _substrat_admin_log (scope_id, id);
-  CREATE INDEX IF NOT EXISTS _substrat_admin_log_actor ON _substrat_admin_log (actor, id);
-  CREATE INDEX IF NOT EXISTS _substrat_admin_log_action ON _substrat_admin_log (action, id);
-  CREATE INDEX IF NOT EXISTS _substrat_admin_log_at ON _substrat_admin_log (at);
+  ${ADMIN_LOG_INDEXES_SQL}
   -- Operational failures (#559) - what the platform could NOT do, and why. NOT the
   -- admin log (that is the never-swept compliance witness of successful mutations)
   -- but retention-bounded telemetry, pruned on write after OPS_FAILURE_RETENTION_DAYS.
@@ -541,6 +542,8 @@ const DIRECTORY_DDL = `
     resolved_at TEXT
   );
   CREATE INDEX IF NOT EXISTS _substrat_issues_seen ON _substrat_issues (last_seen);
+  -- #1748: tenant findings and their suppress rules - kernel-owned DDL (findings.ts).
+  ${FINDINGS_DDL}
   -- The durable sweep record (#1232): one row per unit outcome per pass - a
   -- connection swept/skipped/failed, a schedule fired/skipped/failed. What makes
   -- "when was this last swept" answerable after the log line rolls off. Pruned on
@@ -672,8 +675,8 @@ const SCOPE_COLUMNS_ADDED = [
  * fragment and the DDL had parted company (its test holds it empty).
  */
 export function planDirectoryDdl(ddl) {
-    const all = splitSqlStatements(ddl);
-    const afterColumns = splitSqlStatements(VERSION_MIGRATIONS_DDL);
+    const all = executableSqlStatements(ddl);
+    const afterColumns = executableSqlStatements(VERSION_MIGRATIONS_DDL);
     const held = new Set(afterColumns);
     return {
         loop: all.filter((stmt) => !held.has(stmt)),
@@ -725,6 +728,8 @@ export const REPLIED_METHODS = [
     'createOrg',
     'registerIdentityPool',
     'linkIdentity',
+    // #1748: refuses an expiry outside its bounds with validation_failed.
+    'createFindingRule',
 ];
 export class ControlPlaneDO extends DurableObject {
     sql;
@@ -821,7 +826,7 @@ export class ControlPlaneDO extends DurableObject {
         this.buildDirectorySchema({ holdSwitchRecords: newRecords, holdFences: fencesAreNew });
         for (const kind of newRecords) {
             this.ctx.storage.transactionSync(() => {
-                for (const stmt of splitSqlStatements(switchesDdlOf(kind)))
+                for (const stmt of executableSqlStatements(switchesDdlOf(kind)))
                     this.sql.exec(stmt);
                 this.sql.exec(switchesBackfillSqlOf(kind));
             });
@@ -829,7 +834,7 @@ export class ControlPlaneDO extends DurableObject {
         // #1184: the removal fence and its backfill from the admin log, together, the same way.
         if (fencesAreNew) {
             this.ctx.storage.transactionSync(() => {
-                for (const stmt of splitSqlStatements(MEMBERSHIP_FENCES_DDL))
+                for (const stmt of executableSqlStatements(MEMBERSHIP_FENCES_DDL))
                     this.sql.exec(stmt);
                 // Stamped now, inside this unit (`membershipFencesBackfillSql` says why).
                 this.sql.exec(membershipFencesBackfillSql(new Date().toISOString()));
@@ -903,7 +908,7 @@ export class ControlPlaneDO extends DurableObject {
     rebuildAtomically(statements) {
         this.ctx.storage.transactionSync(() => {
             for (const stmt of statements)
-                this.sql.exec(stmt);
+                this.sql.exec(blankSqlComments(stmt));
         });
     }
     /**
@@ -968,6 +973,8 @@ export class ControlPlaneDO extends DurableObject {
          FROM _substrat_admin_log`,
             'DROP TABLE _substrat_admin_log',
             'ALTER TABLE _substrat_admin_log_new RENAME TO _substrat_admin_log',
+            // The DROP took every index on the table: rebuild them from the kernel's list (#2064).
+            ...ADMIN_LOG_INDEX_DDL,
         ]);
     }
     /**
@@ -1288,7 +1295,10 @@ export class ControlPlaneDO extends DurableObject {
             '_substrat_system_switches', // #1674: the schedule switch's record, per scope
             '_substrat_membership_fences', // #1184: the latest removal, per principal
             '_substrat_peer_switches', // #2029: the peer switch's record, per scope
+            'peer_bindings', // #1720: a tenant's explicit caller-to-target choices
             '_substrat_switch_owed', // #2045: subjects owed a re-assert, per scope
+            '_substrat_findings', // #1748: the tenant's findings
+            '_substrat_finding_rules', // #1748: the tenant's suppress rules
         ]) {
             this.sql.exec(`DELETE FROM ${table} WHERE tenant_id = ?`, tenantId);
         }
@@ -1648,6 +1658,9 @@ export class ControlPlaneDO extends DurableObject {
             else {
                 this.sql.exec('UPDATE scopes SET status = ? WHERE scope_id = ?', to, scopeId);
             }
+            if (to === 'archived' || to === 'reaped') {
+                this.sql.exec('UPDATE peer_bindings SET invalidated = 1 WHERE tenant_id = ? AND target_scope_id = ?', tenantId, scopeId);
+            }
             if (to === 'reaped') {
                 forgetSwitchesOf(this.kernelSql, scopeId);
                 this.forgetLifecycleDeliveries('scope_id = ?', scopeId);
@@ -1755,21 +1768,74 @@ export class ControlPlaneDO extends DurableObject {
             forkedFrom: r.forked_from ?? null,
         })), tenantId, vertical);
     }
+    /** Binding is read at call execution, so a queued invoke follows the current explicit choice. */
+    resolvePeerInstance(tenantId, callerScopeId, vertical) {
+        const rows = this.sql.exec(`SELECT scope_id, tenant_id, vertical, status, kind, forked_from FROM scopes
+       WHERE tenant_id = ? AND vertical = ?`, tenantId, vertical).toArray();
+        const binding = this.sql.exec('SELECT target_scope_id, invalidated FROM peer_bindings WHERE tenant_id = ? AND caller_scope_id = ? AND vertical = ?', tenantId, callerScopeId, vertical).toArray()[0];
+        return resolvePeerInstanceFrom(rows.map((r) => ({
+            id: r.scope_id,
+            tenantId: r.tenant_id,
+            vertical: r.vertical,
+            status: r.status,
+            kind: r.kind ?? '',
+            forkedFrom: r.forked_from,
+        })), tenantId, vertical, binding?.target_scope_id ?? null, binding?.invalidated === 1);
+    }
+    peerBinding(tenantId, callerScopeId, vertical) {
+        return this.sql.exec('SELECT target_scope_id, invalidated FROM peer_bindings WHERE tenant_id = ? AND caller_scope_id = ? AND vertical = ?', tenantId, callerScopeId, vertical).toArray()[0];
+    }
+    peerBindings(tenantId, callerScopeId) {
+        return this.sql.exec('SELECT vertical, target_scope_id, invalidated FROM peer_bindings WHERE tenant_id = ? AND caller_scope_id = ? ORDER BY vertical', tenantId, callerScopeId).toArray();
+    }
+    /** Directory write and endpoint checks share this DO's serialized unit. */
+    setPeerBinding(tenantId, callerScopeId, vertical, targetScopeId) {
+        const caller = this.sql.exec('SELECT tenant_id, status, kind, forked_from FROM scopes WHERE scope_id = ?', callerScopeId).toArray()[0];
+        if (!caller || caller.tenant_id !== tenantId || caller.status !== 'active' || !isPrimaryScopeRow(caller)) {
+            return { ok: false, code: 'not_found', message: 'calling app is not a live primary scope in this tenant' };
+        }
+        if (targetScopeId !== null) {
+            const target = this.sql.exec('SELECT tenant_id, vertical, status, kind, forked_from FROM scopes WHERE scope_id = ?', targetScopeId).toArray()[0];
+            if (!target || target.tenant_id !== tenantId || target.vertical !== vertical || target.status !== 'active' || !isPrimaryScopeRow(target)) {
+                return { ok: false, code: 'not_found', message: 'target is not a live primary instance of this vertical in this tenant' };
+            }
+        }
+        const before = this.peerBinding(tenantId, callerScopeId, vertical);
+        const previous = before?.target_scope_id ?? null;
+        if (previous === targetScopeId && before?.invalidated !== 1)
+            return { ok: true, changed: false, previous };
+        if (targetScopeId === null) {
+            this.sql.exec('DELETE FROM peer_bindings WHERE tenant_id = ? AND caller_scope_id = ? AND vertical = ?', tenantId, callerScopeId, vertical);
+        }
+        else {
+            this.sql.exec(`INSERT INTO peer_bindings (tenant_id, caller_scope_id, vertical, target_scope_id)
+         VALUES (?, ?, ?, ?) ON CONFLICT (tenant_id, caller_scope_id, vertical)
+         DO UPDATE SET target_scope_id = excluded.target_scope_id, invalidated = 0`, tenantId, callerScopeId, vertical, targetScopeId);
+        }
+        return { ok: true, changed: true, previous };
+    }
     /**
      * Everything the router needs to place ONE peer call (#1706), in one round trip: the
      * CALLER's own scope record, and the target instance of `vertical` in that tenant with the
      * script and dispatch parameters that instance runs under.
      *
-     * One read rather than three, because this sits on a request path. The caller's record is
+     * One directory RPC and two SQL reads, because this sits on a request path. The binding
+     * joins the caller read, so even an unbound singleton adds no round trip. The caller's record is
      * what decides whether it may call at all — live, primary, and the vertical it claims — and
      * it is read HERE rather than inferred from routing, so this path does not inherit #1713.
-     * The target side applies the kernel's one rule (`resolveVerticalInstanceFrom`), the same
+     * The target side applies the kernel's one rule (`resolvePeerInstanceFrom`), the same
      * rule the pure host and the local broker apply.
      */
     peerCallTarget(tenantId, callerScopeId, callerVertical, vertical) {
         const caller = this.sql
-            .exec(`SELECT s.tenant_id, s.vertical, s.status, s.kind, s.forked_from, t.status AS tenant_status
-           FROM scopes s LEFT JOIN tenants t ON t.tenant_id = s.tenant_id WHERE s.scope_id = ?`, callerScopeId)
+            .exec(`SELECT s.tenant_id, s.vertical, s.status, s.kind, s.forked_from,
+                t.status AS tenant_status, pb.target_scope_id AS bound_target_scope_id,
+                pb.invalidated AS binding_invalidated
+           FROM scopes s
+           LEFT JOIN tenants t ON t.tenant_id = s.tenant_id
+           LEFT JOIN peer_bindings pb ON pb.tenant_id = s.tenant_id
+             AND pb.caller_scope_id = s.scope_id AND pb.vertical = ?
+          WHERE s.scope_id = ?`, vertical, callerScopeId)
             .toArray()[0];
         const candidates = this.sql
             .exec(`SELECT s.scope_id, s.tenant_id, s.vertical, s.status, s.kind, s.forked_from,
@@ -1786,14 +1852,14 @@ export class ControlPlaneDO extends DurableObject {
            LEFT JOIN vertical_versions sv ON sv.id = vr.serving_version_id
           WHERE s.tenant_id = ? AND s.vertical = ?`, tenantId, vertical)
             .toArray();
-        const resolution = resolveVerticalInstanceFrom(candidates.map((r) => ({
+        const resolution = resolvePeerInstanceFrom(candidates.map((r) => ({
             id: r.scope_id,
             tenantId: r.tenant_id,
             vertical: r.vertical,
             status: r.status,
             kind: r.kind ?? '',
             forkedFrom: r.forked_from ?? null,
-        })), tenantId, vertical);
+        })), tenantId, vertical, caller?.bound_target_scope_id ?? null, caller?.binding_invalidated === 1);
         const target = resolution.outcome === 'resolved'
             ? (candidates.find((r) => r.scope_id === resolution.instance.scopeId) ?? null)
             : null;
@@ -2071,6 +2137,7 @@ export class ControlPlaneDO extends DurableObject {
      */
     deleteScopeDirectory(scopeId) {
         this.ctx.storage.transactionSync(() => {
+            this.sql.exec('UPDATE peer_bindings SET invalidated = 1 WHERE target_scope_id = ?', scopeId);
             this.sql.exec('DELETE FROM hostnames WHERE scope_id = ?', scopeId);
             forgetSwitchesOf(this.kernelSql, scopeId);
             this.forgetLifecycleDeliveries('scope_id = ?', scopeId);
@@ -2161,10 +2228,13 @@ export class ControlPlaneDO extends DurableObject {
      * fence, then the bound — `boundedBy` covering every permission `roleKey` carries, against the
      * tuples and the role as they stand now (`tenantCoverage`) — then the write: the TENANT role
      * and its audit row for an add, `revokeAndFence`'s three for a removal. The coordinator mints
-     * the row (id, actor, attribution, `causedBy`).
+     * the rows (id, actor, attribution, `causedBy`): the role's, and the org's when the change
+     * names one (#2047) — then the org is bounded here too (`boundedBy` a live member of it) and
+     * joined or left in the same unit, a join expiring no later than `boundedBy`'s own membership.
+     * The org row's `before`/`after` are written here, since only the unit knows the expiry.
      */
-    applyMembership(change, row) {
-        const { tenantId, principal, roleKey, boundedBy } = change;
+    applyMembership(change, row, orgRow) {
+        const { tenantId, principal, roleKey, boundedBy, orgId } = change;
         return this.ctx.storage.transactionSync(() => {
             if (change.op === 'add') {
                 const fence = this.sql.exec(MEMBERSHIP_FENCE_SINCE_SQL, tenantId, principal, change.unlessRemovedSince).toArray()[0];
@@ -2177,11 +2247,28 @@ export class ControlPlaneDO extends DurableObject {
             const bound = tenantCoverage(this.directoryReader, tenantId, boundedBy, role.permissions);
             if (!bound.covered)
                 return { applied: false, missing: bound.missing };
+            // #2047: the org, bounded by `boundedBy`'s own live membership of it.
+            const org = orgId ? { orgId, row: orgRow, ...orgChangeBound(this.directoryReader, tenantId, { ...change, orgId }) } : undefined;
+            if (org) {
+                if (!org.row)
+                    throw new Error('applyMembership: an org change needs its audit row');
+                if (!this.readOrg(tenantId, org.orgId))
+                    return { applied: false, unknownOrg: org.orgId };
+                if (!org.bounded)
+                    return { applied: false, notMember: org.orgId };
+            }
             if (change.op === 'remove') {
-                return { applied: true, changed: this.revokeAndFenceIn(tenantId, principal, `role:${roleKey}`, `tenant:${tenantId}`, row) };
+                const roleTaken = this.revokeAndFenceIn(tenantId, principal, `role:${roleKey}`, `tenant:${tenantId}`, row);
+                const orgLeft = !!org && this.revokeAndFenceIn(tenantId, principal, 'member', `org:${org.orgId}`, { ...org.row, before: { principal, orgId: org.orgId } });
+                return { applied: true, changed: roleTaken || orgLeft };
             }
+            const at = new Date().toISOString();
             this.writeTenantTuple(tenantId, `principal:${principal}`, `role:${roleKey}`, `tenant:${tenantId}`, null);
-            this.recordAdmin({ ...row, at: new Date().toISOString() });
+            this.recordAdmin({ ...row, at });
+            if (org?.bounded) {
+                this.writeTenantTuple(tenantId, `principal:${principal}`, 'member', `org:${org.orgId}`, org.expiresAt);
+                this.recordAdmin({ ...org.row, after: memberAddedAudit(principal, org.orgId, org.expiresAt), at });
+            }
             return { applied: true };
         });
     }
@@ -2231,7 +2318,7 @@ export class ControlPlaneDO extends DurableObject {
     /** Members of an org. Live only unless `includeRevoked` — revoked rows are evidence. */
     listMembers(tenantId, object, includeRevoked) {
         return this.sql
-            .exec(`SELECT subject, revoked_at FROM _substrat_tenant_tuples
+            .exec(`SELECT subject, revoked_at, expires_at FROM _substrat_tenant_tuples
          WHERE tenant_id = ? AND relation = 'member' AND object = ?
          ${includeRevoked ? '' : 'AND revoked_at IS NULL'}
          ORDER BY subject`, tenantId, object)
@@ -2559,6 +2646,42 @@ export class ControlPlaneDO extends DurableObject {
        WHERE connection_id = ? AND revoked_at IS NULL`, at, id);
         return true;
     }
+    // -- a vertical's mailed connect links (connections.md §3.5.4) ------------------
+    // The kernel's statements (`connect-links.ts`), shared with the pure adapter. Each
+    // runs inside this single-threaded object with no await between its read and its
+    // write, which is what makes consume's single-use hold across racing callbacks here;
+    // the UPDATE's own WHERE holds it again regardless. `now` is the coordinator's. A move
+    // and the audit row it hands back commit in one unit: `audit` is the row the host
+    // minted (actor, attribution), completed here from what the move changed.
+    insertConnectLink(row, audit) {
+        return this.auditedConnectLink(audit, (sql, write) => insertConnectLinkRow(sql, row, write));
+    }
+    readConnectLink(key) {
+        return readConnectLinkRow(doRedactionSql(this.sql), key);
+    }
+    listConnectLinks(filter, now) {
+        return listConnectLinkRows(doRedactionSql(this.sql), filter, now);
+    }
+    revokeConnectLink(key, audit) {
+        return this.auditedConnectLink(audit, (sql, write) => revokeConnectLinkRow(sql, key, write));
+    }
+    consumeConnectLink(input, now, audit) {
+        return this.auditedConnectLink(audit, (sql, write) => consumeConnectLinkRow(sql, input, now, write));
+    }
+    restoreConnectLink(key, now, audit) {
+        return this.auditedConnectLink(audit, (sql, write) => restoreConnectLinkRow(sql, key, now, write));
+    }
+    auditedConnectLink(audit, run) {
+        return this.ctx.storage.transactionSync(() => run(doRedactionSql(this.sql), (a) => this.recordAdmin({
+            ...audit,
+            action: a.action,
+            tenantId: a.target.tenantId,
+            scopeId: a.target.scopeId,
+            vertical: a.target.vertical,
+            before: a.before,
+            after: a.after,
+        })));
+    }
     /**
      * Record a connection grant directory-side (#592) — written by `grantToConnection`
      * alongside the enforcement tuple, upserting in place on re-grant (the unique
@@ -2929,7 +3052,7 @@ export class ControlPlaneDO extends DurableObject {
      * #1713: the scopes whose lifecycle a deployment must hold, with their lifecycle as this
      * directory holds it. A scope with a vertical (only a deployment serves one), past provisioning
      * and not reaped (a reaped store must not be recreated by a delivery), under a tenant that is
-     * not reaped. `scopeId` narrows it to one scope and `tenantId` to one tenant's scopes, the
+     * not reaped, of a vertical that does not declare its lifecycle held at the router. `scopeId` narrows it to one scope and `tenantId` to one tenant's scopes, the
      * fan-out of a tenant transition. With `drift`, only the scopes the heal sweep must deliver to:
      * a receipt that differs from the directory, and every scope held now. Re-delivering a held
      * scope every pass is what puts a hold back on a store a carry or a restore landed without it,
@@ -2940,6 +3063,10 @@ export class ControlPlaneDO extends DurableObject {
             's.vertical IS NOT NULL',
             "s.status NOT IN ('provisioning', 'reaped')",
             "t.status <> 'reaped'",
+            // A vertical that declares its lifecycle held at the router (`lifecycleHold`) has no route
+            // to deliver to and nothing a delivery would hold: none of its scopes is a target, for a
+            // transition, the heal's drift, or the #2016 tenant-record ask alike.
+            `COALESCE(CASE WHEN json_valid(v.install_spec) THEN json_extract(v.install_spec, '$.lifecycle') END, '') <> 'router'`,
         ];
         const params = [];
         if (filter.tenantId) {
@@ -2978,6 +3105,7 @@ export class ControlPlaneDO extends DurableObject {
                 ${epoch} AS epoch, COALESCE(rs.revision, 0) AS scope_rev, COALESCE(rt.revision, 0) AS tenant_rev
            FROM scopes s
            JOIN tenants t ON t.tenant_id = s.tenant_id
+           LEFT JOIN verticals v ON v.slug = s.vertical
            LEFT JOIN scope_lifecycle_receipts r ON r.scope_id = s.scope_id
            LEFT JOIN scope_tenant_asks a ON a.scope_id = s.scope_id
            LEFT JOIN lifecycle_revisions rs ON rs.subject = 'scope:' || s.scope_id
@@ -3027,6 +3155,59 @@ export class ControlPlaneDO extends DurableObject {
        SELECT ?, ?, ?, ? WHERE ${STILL_DELIVERED}`, scopeId, delivered, at, tenantRecorded ? 1 : 0, scopeId);
     }
     recordOpsFailure(row) {
+        // #1748: the evidence, its issue and its finding in ONE unit — a detector that throws takes
+        // the evidence with it, so the caller's retry observes it rather than losing the observation.
+        this.ctx.storage.transactionSync(() => this.insertOpsFailure(row));
+    }
+    /**
+     * #2064: settle an audited-change intent with no outcome — read it, and only if no outcome row
+     * exists for its operation, write the `unknown` row and its ops-failure row. One unit, so two
+     * racing passes settle once and the digest row cannot be lost. See the kernel's `audit-outcome.ts`.
+     */
+    settleUnrecordedOutcome(input) {
+        return this.ctx.storage.transactionSync(() => {
+            const row = this.sql.exec(SETTLE_INTENT_SQL, input.intentId).toArray()[0];
+            const outcome = unknownOutcomeOf(row, input.intentId, input.error);
+            if (this.sql.exec(SETTLE_OUTCOME_SQL, ...settleOutcomeParamsOf(outcome.operation)).toArray().length > 0) {
+                return false;
+            }
+            const at = new Date().toISOString();
+            this.recordAdmin({
+                id: ulid(),
+                actor: input.actor,
+                action: outcome.action,
+                ...outcome.target,
+                before: null,
+                after: outcome.after,
+                at,
+            });
+            const failure = { ...outcome.failure, actor: input.actor };
+            this.insertOpsFailure({
+                id: ulid(),
+                actor: input.actor,
+                operation: failure.operation,
+                stage: failure.stage ?? null,
+                tenant_id: failure.tenantId ?? null,
+                scope_id: failure.scopeId ?? null,
+                vertical: failure.vertical ?? null,
+                version: null,
+                status: null,
+                message: failure.message.slice(0, 2000),
+                reference: failure.reference ?? null,
+                origin: null,
+                code: null,
+                fingerprint: opsFailureFingerprint(failure),
+                at,
+            });
+            return true;
+        });
+    }
+    /** #2064: an audited operation's rows, by the operation-id index, in bounded batches. */
+    auditedOperations(refs) {
+        return readAuditedOperations((sql, params) => this.sql.exec(sql, ...params).toArray(), refs);
+    }
+    /** The body of `recordOpsFailure`, for a caller already inside a transaction. */
+    insertOpsFailure(row) {
         this.sql.exec(`INSERT INTO _substrat_ops_failures
          (id, actor, operation, stage, tenant_id, scope_id, vertical, version, status, message, reference, origin, code, fingerprint, at)
        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, row.id, row.actor, row.operation, row.stage, row.tenant_id, row.scope_id, row.vertical, row.version, row.status, row.message, row.reference, row.origin, row.code, row.fingerprint, row.at);
@@ -3055,6 +3236,18 @@ export class ControlPlaneDO extends DurableObject {
            status = CASE WHEN status = 'resolved' THEN 'regressed' ELSE status END`, row.fingerprint, row.operation, row.stage, row.origin, row.code, row.at, row.at, row.message, row.tenant_id, issueExemplarOwner(row.tenant_id), row.vertical, row.version);
             const issueHorizon = new Date(Date.now() - ISSUE_RETENTION_DAYS * 86_400_000).toISOString();
             this.sql.exec('DELETE FROM _substrat_issues WHERE last_seen < ?', issueHorizon);
+            // #1748: the tenant's Recurring finding, over the tenant's own occurrences only.
+            const finding = findingOfOpsFailure({
+                tenantId: row.tenant_id,
+                scopeId: row.scope_id,
+                operation: row.operation,
+                stage: row.stage,
+                code: row.code,
+                vertical: row.vertical,
+                version: row.version,
+            });
+            if (finding)
+                observeFinding(doRedactionSql(this.sql), finding, row.at);
         }
     }
     /**
@@ -3141,14 +3334,35 @@ export class ControlPlaneDO extends DurableObject {
         }));
     }
     recordSweepRun(row) {
-        this.sql.exec(`INSERT OR IGNORE INTO _substrat_sweep_runs
-         (id, kind, unit, outcome, tenant_id, scope_id, vertical, version, operation,
-          connection_id, error, elapsed_ms, request_id, event_type, observed_at, platform_requests, at)
-       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, row.id, row.kind, row.unit, row.outcome, row.tenant_id, row.scope_id, row.vertical, row.version, row.operation, row.connection_id, row.error, row.elapsed_ms, row.request_id, row.event_type, row.observed_at, row.platform_requests, row.at);
-        // Prune-on-write, like ops failures and for the same reason: bounded even on a
-        // deployment whose scheduled pass is broken.
-        const horizon = new Date(Date.now() - SWEEP_RUN_RETENTION_DAYS * 86_400_000).toISOString();
-        this.sql.exec('DELETE FROM _substrat_sweep_runs WHERE at < ?', horizon);
+        // #1748: the row and its finding in ONE unit, so a replayed drain the unique index ignores
+        // never meets a row whose observation did not commit.
+        this.ctx.storage.transactionSync(() => {
+            const { rowsWritten } = this.sql.exec(`INSERT OR IGNORE INTO _substrat_sweep_runs
+           (id, kind, unit, outcome, tenant_id, scope_id, vertical, version, operation,
+            connection_id, error, elapsed_ms, request_id, event_type, observed_at, platform_requests, at)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, row.id, row.kind, row.unit, row.outcome, row.tenant_id, row.scope_id, row.vertical, row.version, row.operation, row.connection_id, row.error, row.elapsed_ms, row.request_id, row.event_type, row.observed_at, row.platform_requests, row.at);
+            // Prune-on-write, like ops failures and for the same reason: bounded even on a
+            // deployment whose scheduled pass is broken.
+            const horizon = new Date(Date.now() - SWEEP_RUN_RETENTION_DAYS * 86_400_000).toISOString();
+            this.sql.exec('DELETE FROM _substrat_sweep_runs WHERE at < ?', horizon);
+            // #1748: a failed schedule (Invariant) or a stale freshness verdict (Drift). Only when the
+            // row was written: a replayed drain the unique index ignored is not a second occurrence.
+            const finding = rowsWritten > 0
+                ? findingOfSweepRun({
+                    kind: row.kind,
+                    unit: row.unit,
+                    outcome: row.outcome,
+                    tenantId: row.tenant_id,
+                    scopeId: row.scope_id,
+                    vertical: row.vertical,
+                    version: row.version,
+                    operation: row.operation,
+                    eventType: row.event_type,
+                })
+                : null;
+            if (finding)
+                observeFinding(doRedactionSql(this.sql), finding, row.at);
+        });
     }
     listSweepRuns(query) {
         const where = [];
@@ -3218,6 +3432,47 @@ export class ControlPlaneDO extends DurableObject {
         }
         return pruned;
     }
+    /** #1748: the findings retention pass, in one unit with its audit rows. */
+    pruneFindings(limit, audit) {
+        assertRowLimit('limit', limit);
+        return this.audited(audit, (sql, write) => pruneFindings(sql, Date.now(), limit, write));
+    }
+    /**
+     * #1748: a findings mutation and the audit rows it hands back, in ONE unit, so an audit write
+     * that fails rolls the mutation back. `audit` is the row the host minted, with its actor and
+     * attribution; each row the mutation hands back is written as a copy under its own id.
+     */
+    audited(audit, run) {
+        return this.ctx.storage.transactionSync(() => run(doRedactionSql(this.sql), (a) => this.recordAdmin({
+            ...audit,
+            id: ulid(),
+            action: a.action,
+            tenantId: a.target.tenantId,
+            vertical: a.target.vertical,
+            before: a.before,
+            after: a.after,
+        })));
+    }
+    /** #1748: findings, most recently seen first — `listFindings`. */
+    listFindings(filter) {
+        return listFindings(doRedactionSql(this.sql), filter);
+    }
+    /** #1748: a verdict on one tenant finding, audited in the same unit. Undefined for an unknown one. */
+    setFindingStatus(tenantId, id, status, at, audit) {
+        return this.audited(audit, (sql, write) => setFindingStatus(sql, tenantId, id, status, at, write));
+    }
+    /** #1748: a suppress rule, applied to the findings it covers now, audited in the same unit. */
+    createFindingRule(tenantId, input, createdBy, at, audit) {
+        return this.audited(audit, (sql, write) => createFindingRule(sql, tenantId, input, createdBy, at, write));
+    }
+    /** #1748: end a rule now, audited in the same unit. */
+    revokeFindingRule(tenantId, ruleId, at, audit) {
+        return this.audited(audit, (sql, write) => revokeFindingRule(sql, tenantId, ruleId, at, write));
+    }
+    /** #1748: a tenant's suppress rules. */
+    listFindingRules(tenantId, activeAt, limit) {
+        return listFindingRules(doRedactionSql(this.sql), tenantId, activeAt, limit);
+    }
     listIssues(query) {
         const where = [];
         const params = [];