@opengeni/db 0.28.9 → 0.36.0

package/drizzle/0214_session_activity_commit_gate.sql

@@ -0,0 +1,262 @@
+-- deployment-mode: maintenance
+-- Replace the hidden per-row activity-counter allocator with one explicit
+-- once-per-transaction commit gate. Readers become ordinary MVCC reads; writers
+-- settle every other constraint before briefly advancing the workspace clock.
+-- The durable ledger remains the source of truth; no application-local cache participates.
+
+SET lock_timeout = '5s';
+SET statement_timeout = '10min';
+
+ALTER TABLE "sessions"
+  ADD COLUMN "activity_revision_pending_xid" bigint;
+
+CREATE INDEX "sessions_workspace_activity_pending_idx"
+  ON "sessions" ("workspace_id", "activity_revision_pending_xid", "id")
+  WHERE "activity_revision_pending_xid" IS NOT NULL;
+
+CREATE OR REPLACE FUNCTION opengeni_private.ensure_workspace_session_activity_revision(
+  target_schema text,
+  target_workspace_id uuid,
+  target_account_id uuid
+)
+RETURNS void
+LANGUAGE plpgsql
+SECURITY DEFINER
+SET search_path = pg_catalog
+AS $function$
+DECLARE
+  previous_account_id text := current_setting('opengeni.account_id', true);
+  previous_workspace_id text := current_setting('opengeni.workspace_id', true);
+BEGIN
+  -- FORCE RLS applies to supported non-bypass table owners too. Establish only
+  -- the row's own trusted scope, then restore the caller's transaction-local
+  -- context so structural initialization cannot leak authority downstream.
+  PERFORM set_config('opengeni.account_id', target_account_id::text, true);
+  PERFORM set_config('opengeni.workspace_id', target_workspace_id::text, true);
+  EXECUTE format(
+    'INSERT INTO %I.workspace_session_activity_revisions '
+    || '(workspace_id, account_id, revision) VALUES ($1, $2, 0) '
+    || 'ON CONFLICT (workspace_id) DO NOTHING',
+    target_schema
+  )
+  USING target_workspace_id, target_account_id;
+  PERFORM set_config('opengeni.account_id', coalesce(previous_account_id, ''), true);
+  PERFORM set_config('opengeni.workspace_id', coalesce(previous_workspace_id, ''), true);
+EXCEPTION WHEN OTHERS THEN
+  PERFORM set_config('opengeni.account_id', coalesce(previous_account_id, ''), true);
+  PERFORM set_config('opengeni.workspace_id', coalesce(previous_workspace_id, ''), true);
+  RAISE;
+END;
+$function$;
+
+REVOKE ALL ON FUNCTION opengeni_private.ensure_workspace_session_activity_revision(
+  text, uuid, uuid
+) FROM PUBLIC;
+
+CREATE OR REPLACE FUNCTION opengeni_private.initialize_workspace_session_activity_revision()
+RETURNS trigger
+LANGUAGE plpgsql
+SECURITY DEFINER
+SET search_path = pg_catalog
+AS $function$
+BEGIN
+  PERFORM opengeni_private.ensure_workspace_session_activity_revision(
+    TG_TABLE_SCHEMA,
+    NEW.id,
+    NEW.account_id
+  );
+  RETURN NEW;
+END;
+$function$;
+
+REVOKE ALL ON FUNCTION opengeni_private.initialize_workspace_session_activity_revision()
+  FROM PUBLIC;
+
+CREATE TRIGGER workspaces_initialize_session_activity_revision
+AFTER INSERT ON "workspaces"
+FOR EACH ROW
+EXECUTE FUNCTION opengeni_private.initialize_workspace_session_activity_revision();
+
+-- Install the structural initializer before taking the existing-workspace
+-- snapshot. CREATE TRIGGER fences concurrent workspace writers; the subsequent
+-- conflict-safe backfill covers every row that committed before the trigger,
+-- while later rows are covered by the trigger itself. The same private helper
+-- keeps this path valid when the schema owner is subject to FORCE RLS.
+SELECT opengeni_private.ensure_workspace_session_activity_revision(
+  current_schema(),
+  "id",
+  "account_id"
+)
+FROM "workspaces";
+
+-- Counter lifecycle is now structural: the workspace trigger creates rows and
+-- the foreign-key cascade removes them. The application may only observe and
+-- advance an existing counter at the explicit commit gate.
+DO $grants$
+DECLARE target_schema text := current_schema();
+BEGIN
+  IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'opengeni_app') THEN
+    EXECUTE format(
+      'REVOKE INSERT, DELETE ON TABLE %I.workspace_session_activity_revisions FROM opengeni_app',
+      target_schema
+    );
+  END IF;
+END $grants$;
+
+-- UPDATE remains necessary for the runtime finalizer, but the table itself
+-- rejects every other write shape. This catches a forgotten wrapper even when
+-- application code reaches for the counter directly instead of a session row.
+CREATE OR REPLACE FUNCTION opengeni_private.guard_workspace_session_activity_revision()
+RETURNS trigger
+LANGUAGE plpgsql
+SET search_path = pg_catalog
+AS $function$
+DECLARE
+  gate_state text := nullif(current_setting('opengeni.session_activity_gate_state', true), '');
+  gate_workspace_id uuid := nullif(
+    current_setting('opengeni.session_activity_gate_workspace_id', true),
+    ''
+  )::uuid;
+BEGIN
+  IF gate_state <> 'finalizing'
+    OR gate_workspace_id IS DISTINCT FROM OLD.workspace_id
+    OR NEW.workspace_id IS DISTINCT FROM OLD.workspace_id
+    OR NEW.account_id IS DISTINCT FROM OLD.account_id
+    OR NEW.revision IS DISTINCT FROM OLD.revision + 1
+  THEN
+    RAISE EXCEPTION USING
+      ERRCODE = '55000',
+      MESSAGE = 'Workspace session activity revision requires the finalizing commit gate';
+  END IF;
+  RETURN NEW;
+END;
+$function$;
+
+REVOKE ALL ON FUNCTION opengeni_private.guard_workspace_session_activity_revision()
+  FROM PUBLIC;
+
+CREATE TRIGGER workspace_session_activity_revision_commit_gate
+BEFORE UPDATE ON "workspace_session_activity_revisions"
+FOR EACH ROW
+EXECUTE FUNCTION opengeni_private.guard_workspace_session_activity_revision();
+
+DROP TRIGGER sessions_assign_activity_revision ON "sessions";
+DROP FUNCTION opengeni_private.assign_session_activity_revision();
+
+CREATE OR REPLACE FUNCTION opengeni_private.mark_session_activity_pending()
+RETURNS trigger
+LANGUAGE plpgsql
+SET search_path = pg_catalog
+AS $function$
+DECLARE
+  gate_state text := nullif(current_setting('opengeni.session_activity_gate_state', true), '');
+  gate_workspace_id uuid := nullif(
+    current_setting('opengeni.session_activity_gate_workspace_id', true),
+    ''
+  )::uuid;
+  current_xid bigint := pg_current_xact_id()::text::bigint;
+BEGIN
+  IF gate_state = 'finalizing' THEN
+    IF TG_OP <> 'UPDATE'
+      OR NEW.workspace_id IS DISTINCT FROM OLD.workspace_id
+      OR NEW.updated_at IS DISTINCT FROM OLD.updated_at
+      OR OLD.activity_revision_pending_xid IS DISTINCT FROM current_xid
+      OR NEW.activity_revision_pending_xid IS NOT NULL
+      OR NEW.activity_revision <= OLD.activity_revision
+    THEN
+      RAISE EXCEPTION USING
+        ERRCODE = '55000',
+        MESSAGE = 'Invalid session activity finalization';
+    END IF;
+    RETURN NEW;
+  END IF;
+
+  IF gate_state <> 'open' OR gate_workspace_id IS DISTINCT FROM NEW.workspace_id THEN
+    RAISE EXCEPTION USING
+      ERRCODE = '55000',
+      MESSAGE = 'Session activity write requires an open workspace commit gate';
+  END IF;
+
+  IF TG_OP = 'INSERT' THEN
+    IF NEW.activity_revision <> 0 OR NEW.activity_revision_pending_xid IS NOT NULL THEN
+      RAISE EXCEPTION USING
+        ERRCODE = '55000',
+        MESSAGE = 'Session activity revision is managed by the commit gate';
+    END IF;
+  ELSIF NEW.activity_revision IS DISTINCT FROM OLD.activity_revision
+    OR NEW.activity_revision_pending_xid IS DISTINCT FROM OLD.activity_revision_pending_xid
+  THEN
+    RAISE EXCEPTION USING
+      ERRCODE = '55000',
+      MESSAGE = 'Session activity revision is managed by the commit gate';
+  END IF;
+
+  NEW.activity_revision_pending_xid := current_xid;
+  RETURN NEW;
+END;
+$function$;
+
+REVOKE ALL ON FUNCTION opengeni_private.mark_session_activity_pending()
+  FROM PUBLIC;
+
+CREATE TRIGGER sessions_mark_activity_pending
+BEFORE INSERT OR UPDATE OF "updated_at", "activity_revision", "activity_revision_pending_xid"
+ON "sessions"
+FOR EACH ROW
+EXECUTE FUNCTION opengeni_private.mark_session_activity_pending();
+
+-- A transaction that opens the gate manually but omits the finalizer still
+-- cannot commit. The application gate switches to `preparing` before forcing
+-- all deferred checks, then clears every pending marker in its final write.
+CREATE OR REPLACE FUNCTION opengeni_private.reject_unfinalized_session_activity()
+RETURNS trigger
+LANGUAGE plpgsql
+SET search_path = pg_catalog
+AS $function$
+DECLARE
+  gate_state text := nullif(current_setting('opengeni.session_activity_gate_state', true), '');
+  still_pending_xid bigint;
+BEGIN
+  IF gate_state = 'preparing' THEN
+    RETURN NULL;
+  END IF;
+  EXECUTE format(
+    'SELECT activity_revision_pending_xid FROM %I.sessions WHERE id = $1',
+    TG_TABLE_SCHEMA
+  )
+  INTO still_pending_xid
+  USING NEW.id;
+  IF still_pending_xid IS NOT NULL THEN
+    RAISE EXCEPTION USING
+      ERRCODE = '55000',
+      MESSAGE = 'Session activity transaction reached commit without finalization';
+  END IF;
+  RETURN NULL;
+END;
+$function$;
+
+REVOKE ALL ON FUNCTION opengeni_private.reject_unfinalized_session_activity()
+  FROM PUBLIC;
+
+CREATE CONSTRAINT TRIGGER sessions_activity_insert_commit_guard
+AFTER INSERT ON "sessions"
+DEFERRABLE INITIALLY DEFERRED
+FOR EACH ROW
+WHEN (NEW.activity_revision_pending_xid IS NOT NULL)
+EXECUTE FUNCTION opengeni_private.reject_unfinalized_session_activity();
+
+-- The clearing write must queue a second check as well. The application first
+-- flushes all existing deferred constraints, re-defers these two guards, stamps
+-- every pending row, and then forces both guards against the final row state.
+CREATE CONSTRAINT TRIGGER sessions_activity_update_commit_guard
+AFTER UPDATE ON "sessions"
+DEFERRABLE INITIALLY DEFERRED
+FOR EACH ROW
+WHEN (
+  NEW.activity_revision_pending_xid IS NOT NULL
+  OR OLD.activity_revision_pending_xid IS NOT NULL
+)
+EXECUTE FUNCTION opengeni_private.reject_unfinalized_session_activity();
+
+RESET statement_timeout;
+RESET lock_timeout;

package/drizzle/0215_browser_controller_host.sql

@@ -0,0 +1,385 @@
+-- deployment-mode: maintenance
+-- Separate a remote browser's logical placement from the sandbox that hosts
+-- its sole browserd mutation controller. Existing managed sessions preserve
+-- their current home placement; remote rows are backfilled from their immutable
+-- creation association.
+SET LOCAL lock_timeout = '5s';
+SET LOCAL statement_timeout = '30s';
+
+ALTER TABLE "browser_sessions"
+  ADD COLUMN "controller_host_sandbox_group_id" uuid;
+
+UPDATE "browser_sessions"
+SET "controller_host_sandbox_group_id" = "sandbox_group_id"
+WHERE "placement_kind" = 'sandbox_group';
+
+UPDATE "browser_sessions" browser
+SET "controller_host_sandbox_group_id" = source."sandbox_group_id"
+FROM "browser_session_associations" association
+JOIN "sessions" source
+  ON source."workspace_id" = association."workspace_id"
+ AND source."id" = association."session_id"
+WHERE browser."workspace_id" = association."workspace_id"
+  AND browser."id" = association."browser_session_id"
+  AND browser."placement_kind" = 'external_provider'
+  AND association."relationship" = 'created';
+
+ALTER TABLE "browser_sessions"
+  ADD CONSTRAINT "browser_sessions_controller_host_check" CHECK (
+    ("placement_kind" = 'sandbox_group'
+      AND "controller_host_sandbox_group_id" = "sandbox_group_id")
+    OR
+    ("placement_kind" = 'external_provider'
+      AND "controller_host_sandbox_group_id" IS NOT NULL)
+    OR
+    ("placement_kind" IN ('connected_machine', 'attached_device')
+      AND "controller_host_sandbox_group_id" IS NULL)
+  );
+
+CREATE INDEX "browser_sessions_controller_host_sandbox_group_idx"
+  ON "browser_sessions"
+  ("workspace_id", "controller_host_sandbox_group_id", "lifecycle");
+
+-- Reinstall the interaction-aware lease reaper against controller locality,
+-- not logical browser locality. This keeps a remote browser alive exactly while
+-- its home browserd lease and holder remain authoritative.
+DO $interaction_aware_reaper$
+DECLARE data_schema text := current_schema();
+BEGIN
+  EXECUTE format($create$
+    CREATE OR REPLACE FUNCTION opengeni_private.reap_sandbox_leases(
+      p_viewer_holder_ttl_ms bigint,
+      p_turn_holder_ttl_ms bigint,
+      p_interaction_holder_ttl_ms bigint,
+      p_idle_grace_ms bigint
+    )
+    RETURNS TABLE (
+      workspace_id uuid,
+      sandbox_group_id uuid,
+      instance_id text,
+      lease_epoch integer
+    )
+    LANGUAGE plpgsql
+    SECURITY DEFINER
+    SET search_path = pg_catalog
+    AS $function$
+    DECLARE
+      locked_ids uuid[];
+      stale_interaction_ids uuid[];
+      changed_interaction_workspaces uuid[];
+    BEGIN
+      PERFORM pg_catalog.set_config('opengeni.sandbox_recovery_protocol_v2', '1', true);
+
+      SELECT coalesce(pg_catalog.array_agg(candidate.id), ARRAY[]::uuid[])
+      INTO locked_ids
+      FROM (
+        SELECT lease.id
+        FROM %1$I.sandbox_leases lease
+        WHERE lease.liveness <> 'cold'
+          OR EXISTS (
+            SELECT 1
+            FROM %1$I.sandbox_lease_holders holder
+            WHERE holder.lease_id = lease.id
+          )
+        ORDER BY lease.updated_at, lease.id
+        LIMIT 500
+        FOR UPDATE OF lease SKIP LOCKED
+      ) candidate;
+
+      IF pg_catalog.cardinality(locked_ids) = 0 THEN
+        RETURN;
+      END IF;
+
+      DELETE FROM %1$I.sandbox_lease_holders holder
+      USING (
+        SELECT stale.id
+        FROM %1$I.sandbox_lease_holders stale
+        WHERE stale.lease_id = ANY(locked_ids)
+          AND stale.kind IN ('viewer', 'direct')
+          AND stale.last_heartbeat_at
+            < pg_catalog.now()
+              - pg_catalog.make_interval(secs => p_viewer_holder_ttl_ms / 1000.0)
+        FOR UPDATE OF stale SKIP LOCKED
+      ) victim
+      WHERE holder.id = victim.id;
+
+      IF p_turn_holder_ttl_ms > 0 THEN
+        DELETE FROM %1$I.sandbox_lease_holders holder
+        USING (
+          SELECT stale.id
+          FROM %1$I.sandbox_lease_holders stale
+          WHERE stale.lease_id = ANY(locked_ids)
+            AND stale.kind = 'turn'
+            AND (
+              stale.last_heartbeat_at
+                < pg_catalog.now()
+                  - pg_catalog.make_interval(secs => p_turn_holder_ttl_ms / 1000.0)
+              OR (
+                stale.holder_id ~* '^turn-attempt:[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$'
+                AND NOT EXISTS (
+                  SELECT 1
+                  FROM %1$I.session_turn_attempts attempt
+                  JOIN %1$I.session_turns turn
+                    ON turn.account_id = attempt.account_id
+                   AND turn.workspace_id = attempt.workspace_id
+                   AND turn.session_id = attempt.session_id
+                   AND turn.id = attempt.turn_id
+                   AND turn.execution_generation = attempt.execution_generation
+                   AND turn.active_attempt_id = attempt.id
+                  WHERE stale.holder_id = ('turn-attempt:' || attempt.id::text)
+                    AND attempt.account_id = stale.account_id
+                    AND attempt.workspace_id = stale.workspace_id
+                    AND attempt.session_id = stale.subject_id
+                    AND attempt.state IN ('claimed', 'running')
+                )
+              )
+            )
+          FOR UPDATE OF stale SKIP LOCKED
+        ) victim
+        WHERE holder.id = victim.id;
+      END IF;
+
+      IF p_interaction_holder_ttl_ms > 0 THEN
+        SELECT coalesce(pg_catalog.array_agg(candidate.id), ARRAY[]::uuid[])
+        INTO stale_interaction_ids
+        FROM (
+          SELECT stale.id
+          FROM %1$I.sandbox_lease_holders stale
+          JOIN %1$I.sandbox_leases lease ON lease.id = stale.lease_id
+          WHERE stale.lease_id = ANY(locked_ids)
+            AND stale.kind = 'interaction'
+            AND (
+              stale.last_heartbeat_at
+                < pg_catalog.now()
+                  - pg_catalog.make_interval(
+                      secs => p_interaction_holder_ttl_ms / 1000.0
+                    )
+              OR NOT (
+                EXISTS (
+                  SELECT 1
+                  FROM %1$I.browser_sessions browser
+                  WHERE stale.holder_id = ('browser-session:' || browser.id::text)
+                    AND browser.account_id = stale.account_id
+                    AND browser.workspace_id = stale.workspace_id
+                    AND browser.controller_host_sandbox_group_id = lease.sandbox_group_id
+                    AND browser.lifecycle IN (
+                      'starting', 'active', 'suspending', 'restoring', 'ending'
+                    )
+                )
+                OR EXISTS (
+                  SELECT 1
+                  FROM %1$I.computer_sessions computer
+                  WHERE stale.holder_id = ('computer-session:' || computer.id::text)
+                    AND computer.account_id = stale.account_id
+                    AND computer.workspace_id = stale.workspace_id
+                    AND computer.sandbox_group_id = lease.sandbox_group_id
+                    AND computer.lifecycle IN (
+                      'starting', 'active', 'suspending', 'restoring', 'ending'
+                    )
+                )
+              )
+            )
+          FOR UPDATE OF stale SKIP LOCKED
+        ) candidate;
+
+        IF pg_catalog.cardinality(stale_interaction_ids) > 0 THEN
+          WITH updated_browser AS (
+            UPDATE %1$I.browser_sessions browser
+            SET lifecycle = 'lost',
+                controller_id = null,
+                controller_generation = null,
+                placement_instance_id = null,
+                controller_heartbeat_at = null,
+                failure_code = 'controller_heartbeat_expired',
+                updated_at = pg_catalog.now()
+            FROM %1$I.sandbox_lease_holders holder
+            WHERE holder.id = ANY(stale_interaction_ids)
+              AND holder.holder_id = ('browser-session:' || browser.id::text)
+              AND holder.account_id = browser.account_id
+              AND holder.workspace_id = browser.workspace_id
+              AND browser.lifecycle IN (
+                'starting', 'active', 'suspending', 'restoring', 'ending'
+              )
+            RETURNING browser.workspace_id
+          ), updated_computer AS (
+            UPDATE %1$I.computer_sessions computer
+            SET lifecycle = 'lost',
+                controller_id = null,
+                controller_generation = null,
+                placement_instance_id = null,
+                controller_heartbeat_at = null,
+                failure_code = 'controller_heartbeat_expired',
+                updated_at = pg_catalog.now()
+            FROM %1$I.sandbox_lease_holders holder
+            WHERE holder.id = ANY(stale_interaction_ids)
+              AND holder.holder_id = ('computer-session:' || computer.id::text)
+              AND holder.account_id = computer.account_id
+              AND holder.workspace_id = computer.workspace_id
+              AND computer.lifecycle IN (
+                'starting', 'active', 'suspending', 'restoring', 'ending'
+              )
+            RETURNING computer.workspace_id
+          ), updated AS (
+            SELECT updated_browser.workspace_id AS interaction_workspace_id
+            FROM updated_browser
+            UNION
+            SELECT updated_computer.workspace_id AS interaction_workspace_id
+            FROM updated_computer
+          )
+          SELECT coalesce(
+            pg_catalog.array_agg(DISTINCT updated.interaction_workspace_id),
+            ARRAY[]::uuid[]
+          )
+          INTO changed_interaction_workspaces
+          FROM updated;
+
+          DELETE FROM %1$I.sandbox_lease_holders holder
+          WHERE holder.id = ANY(stale_interaction_ids);
+
+          UPDATE %1$I.workspace_interaction_revisions revision
+          SET revision = revision.revision + 1,
+              updated_at = pg_catalog.now()
+          WHERE revision.workspace_id = ANY(changed_interaction_workspaces);
+        END IF;
+      END IF;
+
+      UPDATE %1$I.sandbox_leases lease SET
+        refcount = counts.total,
+        turn_holders = counts.turns,
+        viewer_holders = counts.viewers,
+        liveness = CASE
+          WHEN lease.liveness = 'warm'
+            AND counts.total = 0
+            AND counts.turns = 0
+            AND (
+              lease.reaper_hold_id IS NULL
+              OR lease.reaper_hold_until <= pg_catalog.now()
+            )
+          THEN 'draining' ELSE lease.liveness END,
+        expires_at = CASE
+          WHEN lease.liveness = 'warm'
+            AND counts.total = 0
+            AND counts.turns = 0
+            AND (
+              lease.reaper_hold_id IS NULL
+              OR lease.reaper_hold_until <= pg_catalog.now()
+            )
+          THEN CASE
+            WHEN lease.rotation_requested_at IS NOT NULL
+              OR lease.archive_capture_id IS NOT NULL
+            THEN pg_catalog.now() - interval '1 millisecond'
+            ELSE pg_catalog.now()
+              + pg_catalog.make_interval(secs => p_idle_grace_ms / 1000.0)
+          END
+          ELSE lease.expires_at END,
+        updated_at = pg_catalog.now()
+      FROM (
+        SELECT candidate.id,
+          pg_catalog.count(holder.id)::int AS total,
+          pg_catalog.count(holder.id)
+            FILTER (WHERE holder.kind = 'turn')::int AS turns,
+          pg_catalog.count(holder.id)
+            FILTER (WHERE holder.kind = 'viewer')::int AS viewers
+        FROM pg_catalog.unnest(locked_ids) candidate(id)
+        LEFT JOIN %1$I.sandbox_lease_holders holder
+          ON holder.lease_id = candidate.id
+        GROUP BY candidate.id
+      ) counts
+      WHERE lease.id = counts.id;
+
+      UPDATE %1$I.sandbox_leases lease SET
+        liveness = 'cold',
+        instance_id = null,
+        lease_epoch = lease.lease_epoch + 1,
+        reaper_hold_id = null,
+        reaper_hold_until = null,
+        reaper_hold_reason = null,
+        resume_state = opengeni_private.warming_reset_resume_state_v2(
+          lease.backend,
+          lease.resume_backend_id,
+          lease.resume_state,
+          lease.workspace_generation,
+          lease.archive_generation,
+          pg_catalog.clock_timestamp()
+        ),
+        resume_backend_id = CASE
+          WHEN coalesce(
+            lease.resume_state #>> '{sessionState,workspaceArchive}', ''
+          ) <> ''
+            OR coalesce(
+              lease.resume_state #>> '{sessionState,workspaceArchivePrev}', ''
+            ) <> ''
+          THEN coalesce(lease.resume_backend_id, lease.backend)
+          ELSE null END,
+        data_plane_url = null,
+        terminal_data_plane_url = null,
+        provider_created_at = null,
+        provider_deadline_at = null,
+        rotation_requested_at = null,
+        rotation_reason = null,
+        updated_at = pg_catalog.now()
+      WHERE lease.id = ANY(locked_ids)
+        AND lease.liveness = 'warming'
+        AND lease.expires_at < pg_catalog.now()
+        AND lease.instance_id IS NULL
+        AND (
+          lease.reaper_hold_id IS NULL
+          OR lease.reaper_hold_until <= pg_catalog.now()
+        );
+
+      UPDATE %1$I.sandbox_leases lease SET
+        liveness = 'draining',
+        refcount = 0,
+        turn_holders = 0,
+        viewer_holders = 0,
+        data_plane_url = null,
+        terminal_data_plane_url = null,
+        lease_epoch = lease.lease_epoch + 1,
+        reaper_hold_id = null,
+        reaper_hold_until = null,
+        reaper_hold_reason = null,
+        expires_at = pg_catalog.now() - interval '1 millisecond',
+        updated_at = pg_catalog.now()
+      WHERE lease.id = ANY(locked_ids)
+        AND lease.liveness = 'warming'
+        AND lease.expires_at < pg_catalog.now()
+        AND lease.instance_id IS NOT NULL
+        AND (
+          lease.reaper_hold_id IS NULL
+          OR lease.reaper_hold_until <= pg_catalog.now()
+        );
+
+      RETURN QUERY
+        SELECT lease.workspace_id, lease.sandbox_group_id,
+          lease.instance_id, lease.lease_epoch
+        FROM %1$I.sandbox_leases lease
+        WHERE lease.id = ANY(locked_ids)
+          AND lease.liveness = 'draining'
+          AND lease.expires_at < pg_catalog.now()
+          AND lease.refcount = 0
+          AND (
+            lease.reaper_hold_id IS NULL
+            OR lease.reaper_hold_until <= pg_catalog.now()
+          );
+    END;
+    $function$;
+  $create$, data_schema);
+END
+$interaction_aware_reaper$;
+
+REVOKE ALL ON FUNCTION opengeni_private.reap_sandbox_leases(
+  bigint, bigint, bigint, bigint
+) FROM PUBLIC;
+
+DO $reaper_grant$
+BEGIN
+  IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'opengeni_app') THEN
+    GRANT EXECUTE ON FUNCTION opengeni_private.reap_sandbox_leases(
+      bigint, bigint, bigint, bigint
+    ) TO opengeni_app;
+  END IF;
+END
+$reaper_grant$;
+
+RESET statement_timeout;
+RESET lock_timeout;