@opengeni/db 0.9.3 → 0.12.0

package/drizzle/0107_host_export_lineage_contract.sql

@@ -0,0 +1,381 @@
+-- deployment-mode: rolling
+-- Forward-only repair for published 0097/0103/0104 histories. Immutable root
+-- capture already exists after 0103; this migration linearizes first-consumer
+-- enablement with deferred producers and validates the session/root invariant.
+-- It never derives or rewrites historical lineage. Rows that may have been
+-- populated by the old 0104 backfill require explicit evidence-backed
+-- disposition before retry.
+
+SET LOCAL lock_timeout = '5s';
+SET LOCAL statement_timeout = '5min';
+
+-- Replacing these SECURITY DEFINER functions preserves their identity, owner,
+-- and existing exporter ACL while making the config-row lock the commit-order
+-- boundary between first-consumer registration and deferred source capture.
+DO $migration$
+DECLARE target_schema text := current_schema();
+BEGIN
+  EXECUTE format($create$
+    CREATE OR REPLACE FUNCTION opengeni_private.enqueue_host_session_event_export()
+    RETURNS trigger
+    LANGUAGE plpgsql
+    SECURITY DEFINER
+    SET search_path = pg_catalog
+    AS $function$
+    DECLARE
+      v_enabled boolean;
+      v_initiator jsonb;
+      v_context jsonb := '{}'::jsonb;
+      v_origin text;
+      v_payload_bytes integer;
+    BEGIN
+      IF NEW.type IN (
+        'agent.message.delta', 'agent.reasoning.delta',
+        'sandbox.command.output.delta', 'terminal.pty.output.delta'
+      ) THEN
+        RETURN NEW;
+      END IF;
+
+      SELECT c.session_events_enabled INTO v_enabled
+      FROM %1$I.host_export_config c WHERE c.id = 1
+      FOR SHARE;
+      IF coalesce(v_enabled, false) = false THEN
+        RETURN NEW;
+      END IF;
+
+      IF NEW.turn_id IS NOT NULL THEN
+        SELECT
+          CASE WHEN octet_length(t.initiator_subject_id) <= 1024 THEN
+            jsonb_strip_nulls(jsonb_build_object(
+              'kind', t.initiator_kind,
+              'subjectId', t.initiator_subject_id,
+              'label', CASE
+                WHEN jsonb_typeof(t.initiator_context -> 'label') = 'string'
+                THEN left(t.initiator_context ->> 'label', 256)
+                ELSE NULL
+              END
+            ))
+          ELSE NULL END,
+          jsonb_strip_nulls(jsonb_build_object(
+            'label', CASE
+              WHEN jsonb_typeof(t.initiator_context -> 'label') = 'string'
+              THEN left(t.initiator_context ->> 'label', 256)
+              ELSE NULL
+            END,
+            'backfill', CASE
+              WHEN jsonb_typeof(t.initiator_context -> 'backfill') = 'boolean'
+              THEN t.initiator_context -> 'backfill'
+              ELSE NULL
+            END,
+            'attributionOmitted', CASE
+              WHEN octet_length(t.initiator_subject_id) > 1024 THEN 'subject_id_too_large'
+              ELSE NULL
+            END
+          )),
+          t.source
+        INTO v_initiator, v_context, v_origin
+        FROM %1$I.session_turns t
+        WHERE t.workspace_id = NEW.workspace_id AND t.id = NEW.turn_id;
+      ELSIF NEW.type = 'session.created' THEN
+        SELECT
+          CASE WHEN octet_length(s.created_by_subject_id) <= 1024 THEN
+            jsonb_strip_nulls(jsonb_build_object(
+              'kind', s.created_by_kind,
+              'subjectId', s.created_by_subject_id,
+              'label', CASE
+                WHEN jsonb_typeof(s.created_by_context -> 'label') = 'string'
+                THEN left(s.created_by_context ->> 'label', 256)
+                ELSE NULL
+              END
+            ))
+          ELSE NULL END,
+          jsonb_strip_nulls(jsonb_build_object(
+            'label', CASE
+              WHEN jsonb_typeof(s.created_by_context -> 'label') = 'string'
+              THEN left(s.created_by_context ->> 'label', 256)
+              ELSE NULL
+            END,
+            'backfill', CASE
+              WHEN jsonb_typeof(s.created_by_context -> 'backfill') = 'boolean'
+              THEN s.created_by_context -> 'backfill'
+              ELSE NULL
+            END,
+            'attributionOmitted', CASE
+              WHEN octet_length(s.created_by_subject_id) > 1024 THEN 'subject_id_too_large'
+              ELSE NULL
+            END
+          )),
+          NULL
+        INTO v_initiator, v_context, v_origin
+        FROM %1$I.sessions s
+        WHERE s.workspace_id = NEW.workspace_id AND s.id = NEW.session_id;
+      END IF;
+
+      v_payload_bytes := octet_length(NEW.payload::text)
+        + octet_length(NEW.type)
+        + coalesce(octet_length(NEW.client_event_id), 0)
+        + coalesce(octet_length(NEW.turn_association), 0)
+        + coalesce(octet_length(NEW.duplicate_reason), 0)
+        + octet_length(coalesce(v_initiator, 'null'::jsonb)::text)
+        + octet_length(v_context::text)
+        + 768;
+      INSERT INTO %1$I.host_export_outbox (
+        export_kind, source_id, account_id, workspace_id, session_id,
+        turn_id, turn_generation, turn_attempt_id, session_sequence,
+        client_event_id, turn_association, duplicate_of_event_id, duplicate_reason,
+        event_type, idempotency_key, initiator, initiator_context, origin,
+        payload, envelope_bytes, occurred_at, source_recorded_at, enqueued_at
+      ) VALUES (
+        'session_event', NEW.id, NEW.account_id, NEW.workspace_id, NEW.session_id,
+        NEW.turn_id, NEW.turn_generation, NEW.turn_attempt_id, NEW.sequence,
+        NEW.client_event_id, NEW.turn_association, NEW.duplicate_of_event_id,
+        NEW.duplicate_reason,
+        NEW.type, 'session_event:' || NEW.id::text, v_initiator,
+        coalesce(v_context, '{}'::jsonb), v_origin, NEW.payload,
+        greatest(1, v_payload_bytes), NEW.occurred_at,
+        NEW.created_at, clock_timestamp()
+      )
+      ON CONFLICT (export_kind, source_id) DO NOTHING;
+      RETURN NEW;
+    END $function$;
+  $create$, target_schema);
+
+  EXECUTE format($create$
+    CREATE OR REPLACE FUNCTION opengeni_private.enqueue_host_usage_event_export()
+    RETURNS trigger
+    LANGUAGE plpgsql
+    SECURITY DEFINER
+    SET search_path = pg_catalog
+    AS $function$
+    DECLARE
+      v_enabled boolean;
+      v_initiator jsonb := CASE
+        WHEN NEW.initiator_kind IS NOT NULL THEN jsonb_strip_nulls(jsonb_build_object(
+          'kind', NEW.initiator_kind,
+          'subjectId', NEW.initiator_subject_id,
+          'label', CASE
+            WHEN jsonb_typeof(NEW.initiator_context -> 'label') = 'string'
+            THEN left(NEW.initiator_context ->> 'label', 256)
+            ELSE NULL
+          END
+        ))
+        ELSE NULL
+      END;
+      v_context jsonb := jsonb_strip_nulls(jsonb_build_object(
+        'label', CASE
+          WHEN jsonb_typeof(NEW.initiator_context -> 'label') = 'string'
+          THEN left(NEW.initiator_context ->> 'label', 256)
+          ELSE NULL
+        END,
+        'backfill', CASE
+          WHEN jsonb_typeof(NEW.initiator_context -> 'backfill') = 'boolean'
+          THEN NEW.initiator_context -> 'backfill'
+          ELSE NULL
+        END
+      ));
+      v_origin text := NEW.origin;
+      v_payload jsonb;
+      v_payload_bytes integer;
+    BEGIN
+      SELECT c.usage_events_enabled INTO v_enabled
+      FROM %1$I.host_export_config c WHERE c.id = 1
+      FOR SHARE;
+      IF coalesce(v_enabled, false) = false THEN
+        RETURN NEW;
+      END IF;
+
+      -- Export-specific bounds apply only while the optional host stream is
+      -- enabled. Standalone/custom usage writers retain their historical
+      -- behavior; an embedded writer that cannot be represented fails its
+      -- source transaction visibly instead of committing a poison outbox row.
+      IF octet_length(NEW.event_type) NOT BETWEEN 1 AND 256
+        OR octet_length(NEW.unit) NOT BETWEEN 1 AND 128
+        OR (NEW.subject_id IS NOT NULL AND octet_length(NEW.subject_id) > 1024)
+        OR (NEW.source_resource_type IS NOT NULL
+          AND octet_length(NEW.source_resource_type) > 256)
+        OR (NEW.source_resource_id IS NOT NULL
+          AND octet_length(NEW.source_resource_id) > 2048)
+        OR octet_length(NEW.idempotency_key) NOT BETWEEN 1 AND 2048
+        OR (NEW.billing_provider_event_id IS NOT NULL
+          AND octet_length(NEW.billing_provider_event_id) > 2048) THEN
+        RAISE EXCEPTION 'usage event exceeds the enabled host-export wire bounds'
+          USING ERRCODE = '22001';
+      END IF;
+
+      IF NEW.turn_id IS NOT NULL THEN
+        SELECT
+          CASE WHEN octet_length(t.initiator_subject_id) <= 1024 THEN
+            jsonb_strip_nulls(jsonb_build_object(
+              'kind', t.initiator_kind,
+              'subjectId', t.initiator_subject_id,
+              'label', CASE
+                WHEN jsonb_typeof(t.initiator_context -> 'label') = 'string'
+                THEN left(t.initiator_context ->> 'label', 256)
+                ELSE NULL
+              END
+            ))
+          ELSE NULL END,
+          jsonb_strip_nulls(jsonb_build_object(
+            'label', CASE
+              WHEN jsonb_typeof(t.initiator_context -> 'label') = 'string'
+              THEN left(t.initiator_context ->> 'label', 256)
+              ELSE NULL
+            END,
+            'backfill', CASE
+              WHEN jsonb_typeof(t.initiator_context -> 'backfill') = 'boolean'
+              THEN t.initiator_context -> 'backfill'
+              ELSE NULL
+            END,
+            'attributionOmitted', CASE
+              WHEN octet_length(t.initiator_subject_id) > 1024 THEN 'subject_id_too_large'
+              ELSE NULL
+            END
+          )),
+          t.source
+        INTO v_initiator, v_context, v_origin
+        FROM %1$I.session_turns t
+        WHERE t.workspace_id = NEW.workspace_id AND t.id = NEW.turn_id;
+      ELSIF v_initiator IS NULL AND NEW.subject_id IS NOT NULL
+        AND octet_length(NEW.subject_id) <= 1024 THEN
+        v_initiator := jsonb_build_object('kind', 'subject', 'subjectId', NEW.subject_id);
+      END IF;
+
+      v_payload := jsonb_build_object(
+        'id', NEW.id,
+        'workspaceId', NEW.workspace_id,
+        'accountId', NEW.account_id,
+        'subjectId', NEW.subject_id,
+        'eventType', NEW.event_type,
+        'quantity', NEW.quantity,
+        'unit', NEW.unit,
+        'sourceResourceType', NEW.source_resource_type,
+        'sourceResourceId', NEW.source_resource_id,
+        'idempotencyKey', NEW.idempotency_key,
+        'occurredAt', NEW.occurred_at,
+        'recordedAt', NEW.recorded_at,
+        'exportedToBillingAt', NEW.exported_to_billing_at,
+        'billingProviderEventId', NEW.billing_provider_event_id
+      );
+      v_payload_bytes := octet_length(v_payload::text)
+        + octet_length(coalesce(v_initiator, 'null'::jsonb)::text)
+        + octet_length(v_context::text)
+        + 768;
+      INSERT INTO %1$I.host_export_outbox (
+        export_kind, source_id, account_id, workspace_id, session_id,
+        turn_id, turn_attempt_id, event_type, idempotency_key, initiator,
+        initiator_context, origin, payload, envelope_bytes, occurred_at,
+        source_recorded_at, enqueued_at
+      ) VALUES (
+        'usage_event', NEW.id, NEW.account_id, NEW.workspace_id, NEW.session_id,
+        NEW.turn_id, NEW.turn_attempt_id, NEW.event_type, NEW.idempotency_key,
+        v_initiator, coalesce(v_context, '{}'::jsonb), v_origin, v_payload,
+        greatest(1, v_payload_bytes), NEW.occurred_at,
+        NEW.recorded_at, clock_timestamp()
+      )
+      ON CONFLICT (export_kind, source_id) DO NOTHING;
+      RETURN NEW;
+    END $function$;
+  $create$, target_schema);
+END $migration$;
+
+REVOKE ALL ON FUNCTION
+  opengeni_private.enqueue_host_session_event_export() FROM PUBLIC;
+REVOKE ALL ON FUNCTION
+  opengeni_private.enqueue_host_usage_event_export() FROM PUBLIC;
+
+-- The old 0104 backfill resolved lineage from then-current session ancestry.
+-- A session-scoped row enqueued before 0103's durable ledger boundary therefore
+-- lacks immutable-capture provenance even when its root is non-null. Reject the
+-- candidate without touching data. The single-row probe is bounded by the
+-- migration statement timeout; operators must disposition suspect history
+-- separately and rerun the same migration.
+DO $migration$
+DECLARE
+  capture_applied_at timestamptz;
+  capture_ledger_rows integer;
+  suspect_source_id uuid;
+BEGIN
+  SELECT min(m.applied_at), count(*)::integer
+  INTO capture_applied_at, capture_ledger_rows
+  FROM "schema_migrations" m
+  WHERE m.name = '0103_host_export_root_session.sql';
+
+  IF capture_ledger_rows <> 1 OR capture_applied_at IS NULL THEN
+    RAISE EXCEPTION '0107 requires the exact 0103 schema-migration ledger boundary'
+      USING ERRCODE = '55000',
+        HINT = 'Apply migrations through immutable 0104 with the canonical runner before retrying 0107.';
+  END IF;
+
+  SELECT o.source_id
+  INTO suspect_source_id
+  FROM "host_export_outbox" o
+  WHERE o.session_id IS NOT NULL
+    AND (
+      o.root_session_id IS NULL
+      OR o.enqueued_at < capture_applied_at
+    )
+  ORDER BY o.enqueued_at, o.source_id
+  LIMIT 1;
+
+  IF suspect_source_id IS NOT NULL THEN
+    RAISE EXCEPTION '0107 found host-export lineage without immutable-capture provenance'
+      USING ERRCODE = '23514',
+        HINT = 'Do not derive lineage from current sessions; use an evidence-backed maintenance disposition, then retry 0107.';
+  END IF;
+END $migration$;
+
+DO $migration$
+DECLARE
+  existing_expression text;
+  normalized_expression text;
+BEGIN
+  SELECT pg_get_expr(c.conbin, c.conrelid, true)
+  INTO existing_expression
+  FROM pg_catalog.pg_constraint c
+  WHERE c.conname = 'host_export_outbox_root_session_check'
+    AND c.conrelid = 'host_export_outbox'::regclass
+    AND c.contype = 'c'
+    AND NOT c.connoinherit;
+
+  IF existing_expression IS NULL THEN
+    IF EXISTS (
+      SELECT 1
+      FROM pg_catalog.pg_constraint c
+      WHERE c.conname = 'host_export_outbox_root_session_check'
+        AND c.conrelid = 'host_export_outbox'::regclass
+    ) THEN
+      RAISE EXCEPTION 'host_export_outbox_root_session_check has an incompatible constraint type'
+        USING ERRCODE = '55000';
+    END IF;
+
+    ALTER TABLE "host_export_outbox"
+      ADD CONSTRAINT "host_export_outbox_root_session_check"
+      CHECK ("session_id" IS NULL OR "root_session_id" IS NOT NULL)
+      NOT VALID;
+  ELSE
+    normalized_expression := lower(regexp_replace(
+      existing_expression,
+      '["()[:space:]]',
+      '',
+      'g'
+    ));
+    IF normalized_expression <> 'session_idisnullorroot_session_idisnotnull' THEN
+      RAISE EXCEPTION 'host_export_outbox_root_session_check has an incompatible expression'
+        USING ERRCODE = '55000';
+    END IF;
+  END IF;
+END $migration$;
+
+-- VALIDATE takes SHARE UPDATE EXCLUSIVE, which permits ordinary reads/writes.
+-- Both lock acquisition and the read-only table scan are source-bounded above;
+-- timeout is a retryable operator disposition, never permission to rewrite data.
+DO $migration$
+BEGIN
+  ALTER TABLE "host_export_outbox"
+    VALIDATE CONSTRAINT "host_export_outbox_root_session_check";
+EXCEPTION
+  WHEN check_violation THEN
+    RAISE EXCEPTION '0107 cannot validate host-export lineage without historical provenance'
+      USING ERRCODE = '23514',
+        HINT = 'Do not derive lineage from current sessions; use an evidence-backed maintenance disposition, then retry 0107.';
+END $migration$;

package/drizzle/0108_fence_invalidated_warming_epochs.sql

@@ -0,0 +1,76 @@
+-- deployment-mode: rolling
+-- Fence provider creates that outlive a failed or expired warming acquisition.
+--
+-- A warming row can be invalidated while a provider create is still
+-- non-abortably in flight. Advancing lease_epoch in the same transaction that
+-- exposes cold/draining means a successor can never reuse the old acquisition
+-- epoch, so late record/commit/fail/cleanup callbacks fail their existing CAS.
+
+CREATE OR REPLACE FUNCTION opengeni_private.reap_sandbox_leases(
+  p_viewer_holder_ttl_ms bigint,
+  p_turn_holder_ttl_ms   bigint,
+  p_idle_grace_ms        bigint
+)
+RETURNS TABLE (workspace_id uuid, sandbox_group_id uuid, instance_id text, lease_epoch integer)
+LANGUAGE plpgsql
+SECURITY DEFINER
+AS $$
+BEGIN
+  DELETE FROM sandbox_lease_holders h
+  WHERE h.kind = 'viewer'
+    AND h.last_heartbeat_at < now() - make_interval(secs => p_viewer_holder_ttl_ms / 1000.0);
+
+  IF p_turn_holder_ttl_ms > 0 THEN
+    DELETE FROM sandbox_lease_holders h
+    WHERE h.kind = 'turn'
+      AND h.last_heartbeat_at < now() - make_interval(secs => p_turn_holder_ttl_ms / 1000.0);
+  END IF;
+
+  UPDATE sandbox_leases L SET
+    refcount       = c.total,
+    turn_holders   = c.turns,
+    viewer_holders = c.viewers,
+    liveness = CASE WHEN L.liveness = 'warm' AND c.total = 0 AND c.turns = 0
+                    THEN 'draining' ELSE L.liveness END,
+    expires_at = CASE WHEN L.liveness = 'warm' AND c.total = 0 AND c.turns = 0
+                    THEN now() + make_interval(secs => p_idle_grace_ms / 1000.0)
+                    ELSE L.expires_at END,
+    updated_at = now()
+  FROM (
+    SELECT L2.id,
+           (SELECT count(*) FROM sandbox_lease_holders h WHERE h.lease_id = L2.id)::int                       AS total,
+           (SELECT count(*) FROM sandbox_lease_holders h WHERE h.lease_id = L2.id AND h.kind = 'turn')::int   AS turns,
+           (SELECT count(*) FROM sandbox_lease_holders h WHERE h.lease_id = L2.id AND h.kind = 'viewer')::int AS viewers
+    FROM sandbox_leases L2
+  ) c
+  WHERE L.id = c.id;
+
+  -- The old warming epoch is permanently closed before a successor can acquire.
+  UPDATE sandbox_leases AS L SET
+    liveness = 'cold', instance_id = NULL,
+    resume_backend_id = NULL, resume_state = NULL,
+    data_plane_url = NULL, terminal_data_plane_url = NULL,
+    lease_epoch = L.lease_epoch + 1,
+    updated_at = now()
+  WHERE L.liveness = 'warming' AND L.expires_at < now() AND L.instance_id IS NULL;
+
+  -- Keep an attributed provider id for the drain/terminate path, but fence the
+  -- expired creator before exposing the row as drainable.
+  UPDATE sandbox_leases AS L SET
+    liveness = 'draining',
+    refcount = 0,
+    turn_holders = 0,
+    viewer_holders = 0,
+    data_plane_url = NULL,
+    terminal_data_plane_url = NULL,
+    lease_epoch = L.lease_epoch + 1,
+    expires_at = now() - interval '1 millisecond',
+    updated_at = now()
+  WHERE L.liveness = 'warming' AND L.expires_at < now() AND L.instance_id IS NOT NULL;
+
+  RETURN QUERY
+    SELECT L.workspace_id, L.sandbox_group_id, L.instance_id, L.lease_epoch
+    FROM sandbox_leases L
+    WHERE L.liveness = 'draining' AND L.expires_at < now() AND L.refcount = 0;
+END;
+$$;

package/drizzle/0109_nested_agent_depth_expand.sql

@@ -0,0 +1,42 @@
+-- deployment-mode: rolling
+-- Expand phase: persist deployment policy and add nullable lineage columns.
+SET lock_timeout = '5s';
+SET statement_timeout = '30s';
+
+CREATE TABLE "nested_agent_depth_configuration" (
+  "singleton" boolean PRIMARY KEY DEFAULT true NOT NULL,
+  "max_nested_agent_depth" integer NOT NULL,
+  "policy_source" text NOT NULL,
+  "updated_at" timestamptz NOT NULL DEFAULT now(),
+  CONSTRAINT "nested_agent_depth_configuration_singleton_check" CHECK ("singleton"),
+  CONSTRAINT "nested_agent_depth_configuration_max_check"
+    CHECK ("max_nested_agent_depth" >= 0 AND "max_nested_agent_depth" <= 2147483647),
+  CONSTRAINT "nested_agent_depth_configuration_source_check"
+    CHECK ("policy_source" IN ('deployment', 'default'))
+);
+
+DO $grants$
+BEGIN
+  IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'opengeni_app') THEN
+    EXECUTE format('GRANT SELECT ON %I."nested_agent_depth_configuration" TO opengeni_app', current_schema());
+  END IF;
+END $grants$;
+
+INSERT INTO "nested_agent_depth_configuration" (
+  "singleton", "max_nested_agent_depth", "policy_source"
+) VALUES (
+  true,
+  coalesce(nullif(current_setting('opengeni.max_nested_agent_depth', true), ''), '3')::integer,
+  coalesce(nullif(current_setting('opengeni.nested_agent_depth_policy_source', true), ''), 'default')
+) ON CONFLICT ("singleton") DO NOTHING;
+
+ALTER TABLE "sessions"
+  ADD COLUMN IF NOT EXISTS "root_session_id" uuid,
+  ADD COLUMN IF NOT EXISTS "nested_agent_depth" integer,
+  ADD COLUMN IF NOT EXISTS "max_nested_agent_depth_override" integer,
+  ADD COLUMN IF NOT EXISTS "effective_max_nested_agent_depth" integer,
+  ADD COLUMN IF NOT EXISTS "nested_agent_depth_policy_source" text,
+  ADD COLUMN IF NOT EXISTS "nested_agent_depth_policy_session_id" uuid;
+
+RESET statement_timeout;
+RESET lock_timeout;