@substrat-run/adapter-cloudflare 0.135.0 → 0.137.0

package/dist/control-plane-do.js

@@ -1,8 +1,37 @@
 import { DurableObject } from 'cloudflare:workers';
-import { IMPERSONATION_COLUMNS, IMPERSONATION_DDL, impersonationByIdQuery, impersonationListQuery, impersonationRowValues, ISSUE_RETENTION_DAYS, telemetryRetentionStatements, assertRowLimit, OPS_FAILURE_RETENTION_DAYS, SWEEP_RUN_RETENTION_DAYS, SWEEP_RUNS_INTENT_INDEX, sweepRunsIntentHasKind, SYSTEM_SWITCHES_BACKFILL_SQL, SYSTEM_SWITCHES_TABLE, SYSTEM_SWITCHES_DDL, forgetSystemSwitchesOf, listSystemSwitchRecords, recordSystemSwitchedOff, recordSystemSwitchedOn, restoreSystemSwitchRecord, scopesSwitchedOffFor, switchedOffModulesOf, systemSwitchRecordsOf, systemSwitchesTableExists, VERSION_MIGRATIONS_DDL, splitVersionMigrationsBatch, versionMigrationsOf, versionsAwaitSplit, writeVersionMigrations, MODEL_USAGE_RETENTION_DAYS, DO_SQL_LIMITS, ulid, isPrimaryScope, resolveVerticalInstanceFrom, LEGACY_SCOPE_ROWS_BACKFILL, loadDirectoryDump, redactSubjectDirectoryText, ISSUE_EXEMPLAR_OWNER_BACKFILL_SQL, issueExemplarOwner, } from '@substrat-run/kernel';
+import { IMPERSONATION_COLUMNS, IMPERSONATION_DDL, impersonationByIdQuery, impersonationListQuery, impersonationRowValues, ISSUE_RETENTION_DAYS, telemetryRetentionStatements, assertRowLimit, OPS_FAILURE_RETENTION_DAYS, SWEEP_RUN_RETENTION_DAYS, SWEEP_RUNS_INTENT_INDEX, sweepRunsIntentHasKind, SYSTEM_SWITCHES_BACKFILL_SQL, SYSTEM_SWITCHES_TABLE, SYSTEM_SWITCHES_DDL, forgetSystemSwitchesOf, listSystemSwitchRecords, recordSystemSwitchedOff, recordSystemSwitchedOn, restoreSystemSwitchRecord, scopesSwitchedOffFor, tenantHoldsSystemGrant, switchedOffModulesOf, systemSwitchRecordsOf, systemSwitchesTableExists, VERSION_MIGRATIONS_DDL, splitVersionMigrationsBatch, versionMigrationsOf, versionsAwaitSplit, writeVersionMigrations, MODEL_USAGE_RETENTION_DAYS, DO_SQL_LIMITS, ulid, isPrimaryScopeRow, resolveVerticalInstanceFrom, LEGACY_SCOPE_ROWS_BACKFILL, loadDirectoryDump, redactSubjectDirectoryText, ISSUE_EXEMPLAR_OWNER_BACKFILL_SQL, issueExemplarOwner, } from '@substrat-run/kernel';
 import { splitSqlStatements, switchSqlOver } from './scope-do.js';
 import { doBuiltColumnsOf, doRedactionSql } from './sql.js';
-import { assertReplayableDump, opsFailureFingerprint, substratError } from '@substrat-run/contracts';
+import { assertReplayableDump, opsFailureFingerprint, ROUTE_SCOPE_HOSTNAMES_MAX, substratError } from '@substrat-run/contracts';
+/**
+ * The router's per-request directory read (`readRoute`), lifted out so its plan can be asked for
+ * (#2005): the scope's hostname set is a correlated subquery on the hot path every request takes,
+ * and it must stay a range read on `hostnames_scope`, never a scan.
+ */
+export const READ_ROUTE_SQL = `SELECT h.tenant_id, h.scope_id, h.vertical_slug, h.surface, h.region,
+                COALESCE(s.serving_ref, vv.deployment_ref) AS deployment_ref,
+                CASE WHEN s.serving_ref IS NOT NULL
+                     THEN json_extract(sv.manifest_json, '$.outbound')
+                     ELSE json_extract(vv.manifest_json, '$.outbound') END AS outbound_json,
+                CASE WHEN s.serving_ref IS NOT NULL
+                     THEN json_extract(sv.manifest_json, '$.calls')
+                     ELSE json_extract(vv.manifest_json, '$.calls') END AS calls_json,
+                s.kind, s.forked_from,
+                (SELECT json_group_array(sh.hostname) FROM (
+                   -- INDEXED BY: unpinned, the planner reads hostnames_status — every active
+                   -- hostname in the fleet, per request. A range on the scope is what it is.
+                   SELECT hostname FROM hostnames INDEXED BY hostnames_scope
+                    WHERE scope_id = s.scope_id AND status = 'active'
+                    ORDER BY hostname LIMIT ${ROUTE_SCOPE_HOSTNAMES_MAX}
+                 ) sh) AS scope_hostnames_json
+           FROM hostnames h
+           JOIN scopes s ON s.scope_id = h.scope_id AND s.tenant_id = h.tenant_id
+           JOIN tenants t ON t.tenant_id = s.tenant_id
+           LEFT JOIN vertical_versions vv ON vv.id = s.vertical_version_id
+           LEFT JOIN verticals vr ON vr.slug = s.vertical
+           LEFT JOIN vertical_versions sv ON sv.id = vr.serving_version_id
+          WHERE h.hostname = ? AND h.status = 'active'
+            AND s.status = 'active' AND t.status = 'active'`;
 /** snake_case row → the camelCase `issueEntry` shape (#1233). */
 function issueOf(r) {
     return {
@@ -63,6 +92,18 @@ function keysetTail(where, params, key, page, defaultOrder = 'asc') {
  * Keeping the copies and gating them, rather than moving the DDL into the kernel, is the
  * recorded answer to #969; `docs/architecture/kernel-design.md` §8 says why.
  */
+/** #1713: one hosted scope, its directory lifecycle, and what its deployment last acknowledged. */
+/**
+ * #2016, Codex #2037 r2: the condition every delivery-bookkeeping write carries, in its own statement —
+ * the scope (bound as the last parameter) still exists, is not reaped, and its tenant is not reaped.
+ * A heal selects its targets, then writes its asks and its receipts after a network round trip; a
+ * delete or a reap that commits in between has already forgotten the scope's rows
+ * (`forgetLifecycleDeliveries`), and an unconditional write would bring them back as orphans.
+ */
+const STILL_DELIVERED = `EXISTS (
+  SELECT 1 FROM scopes s JOIN tenants t ON t.tenant_id = s.tenant_id
+   WHERE s.scope_id = ? AND s.status <> 'reaped' AND t.status <> 'reaped'
+)`;
 const DIRECTORY_DDL = `
   -- The ';' in this comment is a deliberate tripwire; the DDL must go through
   -- splitSqlStatements, and a naive split(';') fails the directory at construction.
@@ -560,6 +601,36 @@ const DIRECTORY_DDL = `
   CREATE INDEX IF NOT EXISTS _substrat_model_usage_tenant ON _substrat_model_usage (tenant_id, at);
   CREATE INDEX IF NOT EXISTS _substrat_model_usage_at ON _substrat_model_usage (at);
   CREATE INDEX IF NOT EXISTS scopes_tenant ON scopes (tenant_id, scope_id);
+  -- #1713: the lifecycle each hosted scope's deployment last acknowledged, as
+  -- "<scope status>/<tenant status>" (lifecycleReceipt). A CP-less deployment has no
+  -- directory, so the platform delivers a scope's lifecycle to it, and the heal sweep
+  -- re-delivers wherever this differs from the directory. No row reads as active/active,
+  -- which is what a deployment holding no lifecycle runs as.
+  -- #2016: tenant_recorded, whether that deployment answered that the scope holds a record of its
+  -- tenant (1) or not (0); NULL is never asked. The heal keeps delivering to any served scope, a
+  -- copy as much as a primary, until it answers 1.
+  CREATE TABLE IF NOT EXISTS scope_lifecycle_receipts (
+    scope_id TEXT PRIMARY KEY,
+    delivered TEXT NOT NULL,
+    at TEXT NOT NULL,
+    tenant_recorded INTEGER
+  );
+  -- #2016: when the heal last asked a scope for its tenant record, whatever came of it. Kept apart
+  -- from the receipt, which only an answer writes, so a scope whose deployment never answers still
+  -- rotates behind the rest of the walk instead of heading it every pass.
+  CREATE TABLE IF NOT EXISTS scope_tenant_asks (
+    scope_id TEXT PRIMARY KEY,
+    asked_at TEXT NOT NULL
+  );
+  -- #1713: the lifecycle's revisions, one counter per subject ("scope:<id>" counts a
+  -- scope's status transitions, "tenant:<id>" a tenant's status changes), each bumped in
+  -- the same transaction as the change it counts, and "directory:epoch", the history the
+  -- directory is in, minted afresh by every restore (importDump). A deployment orders
+  -- deliveries by them, never by a clock. No row reads as 0.
+  CREATE TABLE IF NOT EXISTS lifecycle_revisions (
+    subject TEXT PRIMARY KEY,
+    revision INTEGER NOT NULL
+  );
 `;
 /**
  * `DIRECTORY_DDL`, split once into what runs before the column additions and after (#1764).
@@ -903,6 +974,8 @@ export class ControlPlaneDO extends DurableObject {
         }
         // §4.8's grace-window timestamp on tenants (mirrors scopes' archived_at).
         this.addColumn('tenants', 'deleting_at TEXT');
+        // #2016: whether a scope's deployment holds a record of its tenant (see the table's DDL).
+        this.addColumn('scope_lifecycle_receipts', 'tenant_recorded INTEGER');
         // #412 provenance: the manager tenant that provisioned this one (NULL = direct staff).
         this.addColumn('tenants', 'provisioned_by_tenant TEXT REFERENCES tenants(tenant_id)');
         // §4.7 custom-hostname issuance: CF's hostname id + the DNS records to publish (JSON).
@@ -1013,7 +1086,11 @@ export class ControlPlaneDO extends DurableObject {
           ORDER BY name`)
             .toArray();
         return defs.map(({ name, sql }) => {
-            const cursor = this.sql.exec(`SELECT * FROM "${name}"`);
+            // #1713: the directory's epoch describes THIS store's history, not its data, so it never
+            // leaves in a dump. A restore mints its own from the store it lands in (`importDump`).
+            const cursor = name === 'lifecycle_revisions'
+                ? this.sql.exec(`SELECT * FROM "${name}" WHERE subject <> 'directory:epoch'`)
+                : this.sql.exec(`SELECT * FROM "${name}"`);
             const columns = cursor.columnNames;
             const rows = Array.from(cursor.raw(), (row) => row);
             return { name, ddl: sql, columns, rows };
@@ -1045,6 +1122,8 @@ export class ControlPlaneDO extends DurableObject {
         // nothing.
         assertReplayableDump(tables, { maxColumns: DO_SQL_LIMITS.columns });
         await this.ctx.storage.transaction(async () => {
+            // #1713: the epoch this directory was in, read before the drops take it.
+            const epochBefore = this.lifecycleEpoch();
             this.sql.exec('PRAGMA defer_foreign_keys = ON');
             // `_cf_*` is workerd's, and dropping it is refused (SQLITE_AUTH), as `exportDump` says.
             const existing = this.sql
@@ -1068,6 +1147,12 @@ export class ControlPlaneDO extends DurableObject {
                         this.sql.exec(sql, ...row);
                 },
             });
+            // #1713: a restore is a new history. Its lifecycle counters went back with it, so it
+            // mints an epoch newer than the one this store was in — and than any minted on a store it
+            // no longer remembers (a fresh directory restored from a copy), since an epoch is a time.
+            // A dump never carries one (`exportDump`); a hand-made dump that does is outranked too.
+            // Every deployment then takes this history's deliveries, whatever its counters say.
+            this.sql.exec("INSERT OR REPLACE INTO lifecycle_revisions (subject, revision) VALUES ('directory:epoch', ?)", Math.max(epochBefore + 1, this.lifecycleEpoch() + 1, Date.now()));
         });
         // A restore is a new #1764 backfill episode. It replaced the data the old failure count and
         // backoff were about, so neither may delay or silence it: the count is cleared and the next
@@ -1134,12 +1219,15 @@ export class ControlPlaneDO extends DurableObject {
         // Stamp/clear deleting_at so the grace-window sweep can age tenants (§4.8):
         // entering `deleting` stamps, leaving it (an un-delete) clears — the same shape
         // transitionScope uses for archived_at.
-        if (status === 'deleting') {
-            this.sql.exec('UPDATE tenants SET status = ?, deleting_at = ? WHERE tenant_id = ?', status, new Date().toISOString(), tenantId);
-        }
-        else {
-            this.sql.exec('UPDATE tenants SET status = ?, deleting_at = NULL WHERE tenant_id = ?', status, tenantId);
-        }
+        this.ctx.storage.transactionSync(() => {
+            if (status === 'deleting') {
+                this.sql.exec('UPDATE tenants SET status = ?, deleting_at = ? WHERE tenant_id = ?', status, new Date().toISOString(), tenantId);
+            }
+            else {
+                this.sql.exec('UPDATE tenants SET status = ?, deleting_at = NULL WHERE tenant_id = ?', status, tenantId);
+            }
+            this.bumpLifecycleRevision(`tenant:${tenantId}`); // #1713, with the change it counts
+        });
         return before.status;
     }
     /**
@@ -1170,7 +1258,12 @@ export class ControlPlaneDO extends DurableObject {
         ]) {
             this.sql.exec(`DELETE FROM ${table} WHERE tenant_id = ?`, tenantId);
         }
-        this.sql.exec("UPDATE tenants SET status = 'reaped', deleting_at = NULL WHERE tenant_id = ?", tenantId);
+        this.ctx.storage.transactionSync(() => {
+            this.sql.exec("UPDATE tenants SET status = 'reaped', deleting_at = NULL WHERE tenant_id = ?", tenantId);
+            // Its scopes were reaped one by one, each forgetting its own; any left over go here.
+            this.forgetLifecycleDeliveries('scope_id IN (SELECT scope_id FROM scopes WHERE tenant_id = ?)', tenantId);
+            this.bumpLifecycleRevision(`tenant:${tenantId}`); // #1713
+        });
         return before.status;
     }
     /** Rename the DISPLAY name (never the slug); throw if absent; return the previous name. */
@@ -1496,8 +1589,11 @@ export class ControlPlaneDO extends DurableObject {
             else {
                 this.sql.exec('UPDATE scopes SET status = ? WHERE scope_id = ?', to, scopeId);
             }
-            if (to === 'reaped')
+            if (to === 'reaped') {
                 forgetSystemSwitchesOf(this.kernelSql, scopeId);
+                this.forgetLifecycleDeliveries('scope_id = ?', scopeId);
+            }
+            this.bumpLifecycleRevision(`scope:${scopeId}`); // #1713, with the status it counts
         });
         return { ok: true, status: row.status, vertical: row.vertical };
     }
@@ -1646,7 +1742,7 @@ export class ControlPlaneDO extends DurableObject {
         // for the target, and the router must not carry a second copy of "what is primary".
         const callerState = !caller || caller.tenant_id !== tenantId || caller.vertical !== callerVertical
             ? 'unknown'
-            : !isPrimaryScope({ kind: caller.kind ?? '', forkedFrom: caller.forked_from ?? null })
+            : !isPrimaryScopeRow(caller)
                 ? 'not-primary'
                 : caller.status !== 'active' || caller.tenant_status !== 'active'
                     ? 'inactive'
@@ -1680,24 +1776,19 @@ export class ControlPlaneDO extends DurableObject {
         // json_extract keeps the hot path from parsing whole manifests in JS: SQLite hands
         // back just the `outbound` array as JSON text, or NULL for a pre-#303 manifest —
         // which the egress worker treats as unenforced-but-metered.
-        return this.sql
-            .exec(`SELECT h.tenant_id, h.scope_id, h.vertical_slug, h.surface, h.region,
-                COALESCE(s.serving_ref, vv.deployment_ref) AS deployment_ref,
-                CASE WHEN s.serving_ref IS NOT NULL
-                     THEN json_extract(sv.manifest_json, '$.outbound')
-                     ELSE json_extract(vv.manifest_json, '$.outbound') END AS outbound_json,
-                CASE WHEN s.serving_ref IS NOT NULL
-                     THEN json_extract(sv.manifest_json, '$.calls')
-                     ELSE json_extract(vv.manifest_json, '$.calls') END AS calls_json
-           FROM hostnames h
-           JOIN scopes s ON s.scope_id = h.scope_id AND s.tenant_id = h.tenant_id
-           JOIN tenants t ON t.tenant_id = s.tenant_id
-           LEFT JOIN vertical_versions vv ON vv.id = s.vertical_version_id
-           LEFT JOIN verticals vr ON vr.slug = s.vertical
-           LEFT JOIN vertical_versions sv ON sv.id = vr.serving_version_id
-          WHERE h.hostname = ? AND h.status = 'active'
-            AND s.status = 'active' AND t.status = 'active'`, hostname)
+        const row = this.sql
+            .exec(READ_ROUTE_SQL, hostname)
             .toArray()[0];
+        if (!row)
+            return undefined;
+        // #2005: the predicate runs here, once, rather than travelling as two columns for the
+        // router to re-derive — every reader of the directory answers it the same way.
+        const { kind, forked_from, scope_hostnames_json, ...route } = row;
+        return {
+            ...route,
+            primary: isPrimaryScopeRow({ kind, forked_from }),
+            scope_hostnames: JSON.parse(scope_hostnames_json),
+        };
     }
     /** Demote any current canonical for this surface — exactly one may hold it. */
     demoteCanonical(scopeId, surface) {
@@ -1915,9 +2006,24 @@ export class ControlPlaneDO extends DurableObject {
      * wiping the scope DO's storage — the fork-only refusal lives there, not here.
      */
     deleteScopeDirectory(scopeId) {
-        this.sql.exec('DELETE FROM hostnames WHERE scope_id = ?', scopeId);
-        forgetSystemSwitchesOf(this.kernelSql, scopeId);
-        this.sql.exec('DELETE FROM scopes WHERE scope_id = ?', scopeId);
+        this.ctx.storage.transactionSync(() => {
+            this.sql.exec('DELETE FROM hostnames WHERE scope_id = ?', scopeId);
+            forgetSystemSwitchesOf(this.kernelSql, scopeId);
+            this.forgetLifecycleDeliveries('scope_id = ?', scopeId);
+            this.sql.exec('DELETE FROM scopes WHERE scope_id = ?', scopeId);
+        });
+    }
+    /**
+     * #1713 / #2016: drop the delivery bookkeeping of scopes that will never be delivered to again —
+     * what each deployment acknowledged (`scope_lifecycle_receipts`) and when the heal last asked it
+     * for its tenant record (`scope_tenant_asks`). A hard-deleted fork's rows would otherwise outlive
+     * it, and every preview created and deleted would grow both tables, and every directory dump, for
+     * good. A reaped scope keeps its tombstone row but leaves every delivery walk, so its go too.
+     */
+    forgetLifecycleDeliveries(where, ...params) {
+        for (const table of ['scope_lifecycle_receipts', 'scope_tenant_asks']) {
+            this.sql.exec(`DELETE FROM ${table} WHERE ${where}`, ...params);
+        }
     }
     readChannel(verticalSlug, channel) {
         return this.sql
@@ -2346,18 +2452,26 @@ export class ControlPlaneDO extends DurableObject {
        VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL)`, row.connectionId, row.tenantId, row.vertical, row.permission, row.scopeId, row.expiresAt, row.grantedBy, row.grantedAt);
     }
     // -- the schedule switch's record (#1674) — `system-switch-record.ts` is the whole rule ---
-    /** OFF's write, after the scope's switch held. */
+    /** OFF's write, before the scope's switch moves (#1823). Answers the row as it was. */
     recordSystemSwitchedOff(row) {
-        recordSystemSwitchedOff(this.kernelSql, row);
+        return recordSystemSwitchedOff(this.kernelSql, row);
     }
     /** ON's write, before the scope moves. Answers the row as it was, for a failed ON to restore. */
     recordSystemSwitchedOn(row) {
         return recordSystemSwitchedOn(this.kernelSql, row);
     }
-    /** Put a row back as `recordSystemSwitchedOn` found it — the ON that followed it failed. */
+    /** Put a row back as the switch call's record write found it — the move that followed it failed. */
     restoreSystemSwitchRecord(key, prior) {
         restoreSystemSwitchRecord(this.kernelSql, key, prior);
     }
+    /**
+     * #1823: which of these modules the tenant holds a live TENANT-level grant for — what the
+     * switch passes the scope as `tenantHeld`, so a module whose only authority there is a tenant
+     * tuple is still switchable. Read here, where the tenant tuples live.
+     */
+    tenantHeldSystemModules(tenantId, moduleIds, now) {
+        return moduleIds.filter((m) => tenantHoldsSystemGrant(this.kernelSql, tenantId, m, now));
+    }
     /** The fleet read. */
     listSystemSwitches(filter) {
         return listSystemSwitchRecords(this.kernelSql, filter);
@@ -2655,6 +2769,107 @@ export class ControlPlaneDO extends DurableObject {
             at: r.at,
         }));
     }
+    /**
+     * #1713: the scopes whose lifecycle a deployment must hold, with their lifecycle as this
+     * directory holds it. A scope with a vertical (only a deployment serves one), past provisioning
+     * and not reaped (a reaped store must not be recreated by a delivery), under a tenant that is
+     * not reaped. `scopeId` narrows it to one scope and `tenantId` to one tenant's scopes, the
+     * fan-out of a tenant transition. With `drift`, only the scopes the heal sweep must deliver to:
+     * a receipt that differs from the directory, and every scope held now. Re-delivering a held
+     * scope every pass is what puts a hold back on a store a carry or a restore landed without it,
+     * and held scopes are few. Drifted rows first, so held ones never starve them.
+     */
+    lifecycleTargets(filter) {
+        const where = [
+            's.vertical IS NOT NULL',
+            "s.status NOT IN ('provisioning', 'reaped')",
+            "t.status <> 'reaped'",
+        ];
+        const params = [];
+        if (filter.tenantId) {
+            where.push('s.tenant_id = ?');
+            params.push(filter.tenantId);
+        }
+        if (filter.scopeId) {
+            where.push('s.scope_id = ?');
+            params.push(filter.scopeId);
+        }
+        const epoch = this.lifecycleEpoch();
+        // What the deployment would acknowledge now (`lifecycleReceipt`): the statuses and the full
+        // revision. No receipt is drift only after a restore (epoch > 0), when what the deployment
+        // holds is unknown; before any, a live scope with no receipt runs as its deployment does.
+        const acked = "s.status || '/' || t.status || '@' || CAST(? AS INTEGER) || '.' || COALESCE(rs.revision, 0) || '.' || COALESCE(rt.revision, 0)";
+        // CAST: a bound JS number reaches DO SQLite as a REAL, and would concatenate as '0.0'.
+        const drifted = `(r.delivered IS NULL AND CAST(? AS INTEGER) > 0) OR r.delivered <> ${acked}`;
+        const driftParams = [epoch, epoch];
+        if (filter.drift) {
+            where.push(`(${drifted} OR s.status NOT IN ('active', 'archived') OR t.status <> 'active')`);
+            params.push(...driftParams);
+        }
+        if (filter.unrecorded)
+            where.push('COALESCE(r.tenant_recorded, 0) <> 1');
+        // The unrecorded walk rotates on the ask, not the answer: never asked first, then the one asked
+        // longest ago, so a scope that answers 0, or never answers at all, cannot starve the rest of a
+        // bounded pass.
+        const order = filter.unrecorded
+            ? 'a.asked_at IS NOT NULL, a.asked_at, s.scope_id'
+            : `CASE WHEN ${drifted} THEN 0 ELSE 1 END, s.scope_id`;
+        if (!filter.unrecorded)
+            params.push(...driftParams);
+        params.push(filter.limit ?? 1000);
+        return this.sql
+            .exec(`SELECT s.tenant_id, s.scope_id, s.status AS scope_status, t.status AS tenant_status, r.delivered,
+                ${epoch} AS epoch, COALESCE(rs.revision, 0) AS scope_rev, COALESCE(rt.revision, 0) AS tenant_rev
+           FROM scopes s
+           JOIN tenants t ON t.tenant_id = s.tenant_id
+           LEFT JOIN scope_lifecycle_receipts r ON r.scope_id = s.scope_id
+           LEFT JOIN scope_tenant_asks a ON a.scope_id = s.scope_id
+           LEFT JOIN lifecycle_revisions rs ON rs.subject = 'scope:' || s.scope_id
+           LEFT JOIN lifecycle_revisions rt ON rt.subject = 'tenant:' || s.tenant_id
+          WHERE ${where.join(' AND ')}
+          ORDER BY ${order}
+          LIMIT ?`, ...params)
+            .toArray();
+    }
+    /** #1713: the directory's epoch (`lifecycleRevision.epoch`): 0 until a restore mints one. */
+    lifecycleEpoch() {
+        const row = this.sql
+            .exec("SELECT revision FROM lifecycle_revisions WHERE subject = 'directory:epoch'")
+            .toArray()[0];
+        return row?.revision ?? 0;
+    }
+    /**
+     * #1713: raise the directory's epoch to at least `atLeast`, never lower it, and answer the epoch
+     * now. One statement, so concurrent heals that each learned a held epoch settle on the highest.
+     * Called when a scope refuses this directory's delivery while holding an epoch another history
+     * of the directory minted (a fresh-directory restore whose clock ran behind the old one's).
+     */
+    raiseLifecycleEpoch(atLeast) {
+        if (!Number.isSafeInteger(atLeast) || atLeast < 0)
+            throw new Error(`not an epoch: ${atLeast}`);
+        // An epoch is a mint time: one a day past the clock is forged or broken, and never raised to.
+        if (atLeast > Date.now() + 24 * 60 * 60 * 1000)
+            throw new Error(`epoch ${atLeast} is ahead of the clock; not raised`);
+        this.sql.exec(`INSERT INTO lifecycle_revisions (subject, revision) VALUES ('directory:epoch', ?)
+         ON CONFLICT (subject) DO UPDATE SET revision = MAX(revision, excluded.revision)`, atLeast);
+        return this.lifecycleEpoch();
+    }
+    /** #1713: count one lifecycle change of a scope or a tenant. Call inside the change's transaction. */
+    bumpLifecycleRevision(subject) {
+        this.sql.exec('INSERT INTO lifecycle_revisions (subject, revision) VALUES (?, 1) ON CONFLICT (subject) DO UPDATE SET revision = revision + 1', subject);
+    }
+    /** #2016: note that the heal asked these scopes for their tenant record at `at`, before it asks. */
+    recordTenantAsks(scopeIds, at) {
+        for (const scopeId of scopeIds) {
+            this.sql.exec(`INSERT OR REPLACE INTO scope_tenant_asks (scope_id, asked_at) SELECT ?, ? WHERE ${STILL_DELIVERED}`, scopeId, at, scopeId);
+        }
+    }
+    /** #1713: record what a scope's deployment acknowledged holding (`lifecycleReceipt`), and #2016:
+     *  whether it answered that the scope holds a record of its tenant. */
+    recordLifecycleReceipt(scopeId, delivered, at, tenantRecorded = false) {
+        this.sql.exec(`INSERT OR REPLACE INTO scope_lifecycle_receipts (scope_id, delivered, at, tenant_recorded)
+       SELECT ?, ?, ?, ? WHERE ${STILL_DELIVERED}`, scopeId, delivered, at, tenantRecorded ? 1 : 0, scopeId);
+    }
     recordOpsFailure(row) {
         this.sql.exec(`INSERT INTO _substrat_ops_failures
          (id, actor, operation, stage, tenant_id, scope_id, vertical, version, status, message, reference, origin, code, fingerprint, at)