@team-semicolon/semicolony-cli 4.18.51 → 4.18.53

package/migrations/160_hive_cron_failure_alert_outbox.sql

@@ -0,0 +1,406 @@
+-- @schema-retarget: off
+-- Durable, credential-free Hive cron failure-alert authority.
+-- The migration runner supplies the transaction boundary; do not add one here.
+
+DO $$
+DECLARE
+  role_state record;
+BEGIN
+  SELECT rolcanlogin, rolsuper, rolcreatedb, rolcreaterole,
+         rolinherit, rolreplication, rolbypassrls
+    INTO role_state
+    FROM pg_catalog.pg_roles
+   WHERE rolname = 'sc_provider';
+
+  IF NOT FOUND THEN
+    RAISE EXCEPTION 'sc_provider role missing; migration 135 required';
+  END IF;
+
+  IF NOT role_state.rolcanlogin
+     OR role_state.rolsuper
+     OR role_state.rolcreatedb
+     OR role_state.rolcreaterole
+     OR role_state.rolinherit
+     OR role_state.rolreplication
+     OR role_state.rolbypassrls THEN
+    RAISE EXCEPTION 'sc_provider role attributes are unsafe';
+  END IF;
+END $$;
+
+CREATE TABLE semicolony.cron_read_action_failure_alert_outbox (
+  occurrence_key   TEXT PRIMARY KEY,
+  failure_code     TEXT NOT NULL CHECK (
+    failure_code IN ('released_failed', 'retry_exhausted', 'stale_binding')
+  ),
+  state            TEXT NOT NULL DEFAULT 'pending' CHECK (
+    state IN ('pending', 'claimed', 'sent', 'failed', 'ambiguous')
+  ),
+  claim_id         TEXT UNIQUE CHECK (
+    claim_id IS NULL
+    OR (claim_id COLLATE "C") ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$'
+  ),
+  claimed_at       TIMESTAMPTZ,
+  claim_expires_at TIMESTAMPTZ,
+  resolved_at      TIMESTAMPTZ,
+  created_at       TIMESTAMPTZ NOT NULL DEFAULT pg_catalog.clock_timestamp(),
+  updated_at       TIMESTAMPTZ NOT NULL DEFAULT pg_catalog.clock_timestamp(),
+  FOREIGN KEY (occurrence_key)
+    REFERENCES semicolony.cron_read_action_dispatch_outbox (idempotency_key),
+  CHECK (
+    (
+      state = 'pending'
+      AND claim_id IS NULL
+      AND claimed_at IS NULL
+      AND claim_expires_at IS NULL
+      AND resolved_at IS NULL
+    )
+    OR
+    (
+      state = 'claimed'
+      AND claim_id IS NOT NULL
+      AND claimed_at IS NOT NULL
+      AND claim_expires_at IS NOT NULL
+      AND claim_expires_at > claimed_at
+      AND resolved_at IS NULL
+    )
+    OR
+    (
+      state IN ('sent', 'failed', 'ambiguous')
+      AND claim_id IS NOT NULL
+      AND claimed_at IS NOT NULL
+      AND claim_expires_at IS NOT NULL
+      AND claim_expires_at > claimed_at
+      AND resolved_at IS NOT NULL
+    )
+  )
+);
+
+CREATE INDEX idx_cron_read_action_failure_alert_pending
+  ON semicolony.cron_read_action_failure_alert_outbox (created_at, occurrence_key)
+  WHERE state = 'pending';
+
+CREATE INDEX idx_cron_read_action_failure_alert_expired
+  ON semicolony.cron_read_action_failure_alert_outbox (claim_expires_at, occurrence_key)
+  WHERE state = 'claimed';
+
+CREATE FUNCTION semicolony.hive_cron_protect_failure_alert()
+RETURNS trigger
+LANGUAGE plpgsql
+SECURITY DEFINER
+SET search_path = pg_catalog
+SET TimeZone = 'UTC'
+AS $$
+BEGIN
+  IF TG_OP = 'DELETE' THEN
+    RAISE EXCEPTION 'failure_alert_delete_denied' USING ERRCODE = '55000';
+  END IF;
+
+  IF NEW.occurrence_key IS DISTINCT FROM OLD.occurrence_key
+     OR NEW.failure_code IS DISTINCT FROM OLD.failure_code
+     OR NEW.created_at IS DISTINCT FROM OLD.created_at THEN
+    RAISE EXCEPTION 'failure_alert_identity_immutable' USING ERRCODE = '55000';
+  END IF;
+
+  IF OLD.state IN ('sent', 'failed', 'ambiguous') THEN
+    RAISE EXCEPTION 'failure_alert_terminal_immutable' USING ERRCODE = '55000';
+  END IF;
+
+  IF OLD.state = 'pending' AND NEW.state <> 'claimed' THEN
+    RAISE EXCEPTION 'invalid_failure_alert_transition' USING ERRCODE = '55000';
+  END IF;
+
+  IF OLD.state = 'claimed' AND NEW.state NOT IN ('sent', 'failed', 'ambiguous') THEN
+    RAISE EXCEPTION 'invalid_failure_alert_transition' USING ERRCODE = '55000';
+  END IF;
+
+  RETURN NEW;
+END;
+$$;
+
+CREATE TRIGGER protect_cron_read_action_failure_alert
+BEFORE UPDATE OR DELETE ON semicolony.cron_read_action_failure_alert_outbox
+FOR EACH ROW EXECUTE FUNCTION semicolony.hive_cron_protect_failure_alert();
+
+CREATE FUNCTION semicolony.hive_cron_record_failure_alert()
+RETURNS trigger
+LANGUAGE plpgsql
+SECURITY DEFINER
+SET search_path = pg_catalog
+SET TimeZone = 'UTC'
+AS $$
+DECLARE
+  failure_count integer;
+BEGIN
+  IF OLD.status IS DISTINCT FROM 'failed'
+     AND NEW.status = 'failed'
+     AND NEW.result_code IN ('released_failed', 'retry_exhausted', 'stale_binding') THEN
+    UPDATE semicolony.bot_cron_jobs AS job
+       SET consecutive_failures = job.consecutive_failures + 1,
+           last_status = 'failure',
+           last_error = NEW.result_code,
+           last_run = NEW.resolved_at
+     WHERE job.bot_id = 'semi'
+       AND job.job_id = NEW.job_id
+       AND job.runtime_source = 'hive-cron-v2'
+    RETURNING job.consecutive_failures INTO failure_count;
+
+    IF NOT FOUND THEN
+      RAISE EXCEPTION 'failure_alert_job_authority_missing' USING ERRCODE = 'P0001';
+    END IF;
+
+    IF failure_count = 3 THEN
+      INSERT INTO semicolony.cron_read_action_failure_alert_outbox (
+        occurrence_key, failure_code
+      ) VALUES (
+        NEW.idempotency_key, NEW.result_code
+      );
+    END IF;
+  END IF;
+
+  RETURN NEW;
+END;
+$$;
+
+CREATE TRIGGER record_hive_cron_failure_alert
+AFTER UPDATE ON semicolony.cron_read_action_dispatch_outbox
+FOR EACH ROW
+WHEN (
+  OLD.status IS DISTINCT FROM 'failed'
+  AND NEW.status = 'failed'
+  AND NEW.result_code IN ('released_failed', 'retry_exhausted', 'stale_binding')
+)
+EXECUTE FUNCTION semicolony.hive_cron_record_failure_alert();
+
+CREATE FUNCTION semicolony.hive_cron_claim_failure_alert(
+  p_claim_id text,
+  p_lease_seconds integer
+)
+RETURNS TABLE (
+  outcome text,
+  occurrence_key text,
+  failure_code text,
+  claim_id text,
+  claimed_at timestamptz,
+  claim_expires_at timestamptz,
+  resolved_at timestamptz
+)
+LANGUAGE plpgsql
+SECURITY DEFINER
+SET search_path = pg_catalog
+SET TimeZone = 'UTC'
+AS $$
+DECLARE
+  alert record;
+  db_now timestamptz;
+BEGIN
+  IF session_user <> 'sc_provider'
+     OR public.is_provider() IS DISTINCT FROM true THEN
+    RAISE EXCEPTION 'provider_mode_required' USING ERRCODE = '42501';
+  END IF;
+
+  IF p_claim_id IS NULL
+     OR (p_claim_id COLLATE "C") !~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$'
+     OR p_lease_seconds IS NULL
+     OR p_lease_seconds NOT BETWEEN 1 AND 3600 THEN
+    RAISE EXCEPTION 'invalid_failure_alert_claim' USING ERRCODE = '22023';
+  END IF;
+
+  PERFORM pg_catalog.pg_advisory_xact_lock(
+    pg_catalog.hashtextextended(p_claim_id, 0::bigint)
+  );
+  db_now := pg_catalog.clock_timestamp();
+
+  SELECT current_alert.occurrence_key,
+         current_alert.failure_code,
+         current_alert.state,
+         current_alert.claim_id,
+         current_alert.claimed_at,
+         current_alert.claim_expires_at,
+         current_alert.resolved_at
+    INTO alert
+    FROM semicolony.cron_read_action_failure_alert_outbox AS current_alert
+   WHERE current_alert.claim_id = p_claim_id
+   FOR UPDATE;
+
+  IF FOUND THEN
+    IF alert.state = 'claimed' AND alert.claim_expires_at <= db_now THEN
+      UPDATE semicolony.cron_read_action_failure_alert_outbox AS current_alert
+         SET state = 'ambiguous',
+             resolved_at = db_now,
+             updated_at = db_now
+       WHERE current_alert.occurrence_key = alert.occurrence_key;
+
+      RETURN QUERY SELECT
+        'recovered_ambiguous'::text,
+        alert.occurrence_key,
+        alert.failure_code,
+        alert.claim_id,
+        alert.claimed_at,
+        alert.claim_expires_at,
+        db_now;
+    ELSE
+      RETURN QUERY SELECT
+        'claim_exists'::text,
+        alert.occurrence_key,
+        alert.failure_code,
+        alert.claim_id,
+        alert.claimed_at,
+        alert.claim_expires_at,
+        alert.resolved_at;
+    END IF;
+    RETURN;
+  END IF;
+
+  SELECT current_alert.occurrence_key,
+         current_alert.failure_code,
+         current_alert.claim_id,
+         current_alert.claimed_at,
+         current_alert.claim_expires_at
+    INTO alert
+    FROM semicolony.cron_read_action_failure_alert_outbox AS current_alert
+   WHERE current_alert.state = 'claimed'
+     AND current_alert.claim_expires_at <= db_now
+   ORDER BY current_alert.claim_expires_at, current_alert.occurrence_key
+   LIMIT 1
+   FOR UPDATE;
+
+  IF FOUND THEN
+    UPDATE semicolony.cron_read_action_failure_alert_outbox AS current_alert
+       SET state = 'ambiguous',
+           resolved_at = db_now,
+           updated_at = db_now
+     WHERE current_alert.occurrence_key = alert.occurrence_key;
+
+    RETURN QUERY SELECT
+      'recovered_ambiguous'::text,
+      alert.occurrence_key,
+      alert.failure_code,
+      alert.claim_id,
+      alert.claimed_at,
+      alert.claim_expires_at,
+      db_now;
+    RETURN;
+  END IF;
+
+  SELECT current_alert.occurrence_key,
+         current_alert.failure_code
+    INTO alert
+    FROM semicolony.cron_read_action_failure_alert_outbox AS current_alert
+   WHERE current_alert.state = 'pending'
+   ORDER BY current_alert.created_at, current_alert.occurrence_key
+   LIMIT 1
+   FOR UPDATE SKIP LOCKED;
+
+  IF NOT FOUND THEN
+    RETURN;
+  END IF;
+
+  UPDATE semicolony.cron_read_action_failure_alert_outbox AS current_alert
+     SET state = 'claimed',
+         claim_id = p_claim_id,
+         claimed_at = db_now,
+         claim_expires_at = db_now + pg_catalog.make_interval(secs => p_lease_seconds),
+         updated_at = db_now
+   WHERE current_alert.occurrence_key = alert.occurrence_key;
+
+  RETURN QUERY SELECT
+    'claimed'::text,
+    alert.occurrence_key,
+    alert.failure_code,
+    p_claim_id,
+    db_now,
+    db_now + pg_catalog.make_interval(secs => p_lease_seconds),
+    NULL::timestamptz;
+END;
+$$;
+
+CREATE FUNCTION semicolony.hive_cron_complete_failure_alert(
+  p_occurrence_key text,
+  p_claim_id text,
+  p_outcome text
+)
+RETURNS TABLE (outcome text)
+LANGUAGE plpgsql
+SECURITY DEFINER
+SET search_path = pg_catalog
+SET TimeZone = 'UTC'
+AS $$
+DECLARE
+  alert record;
+  db_now timestamptz;
+BEGIN
+  IF session_user <> 'sc_provider'
+     OR public.is_provider() IS DISTINCT FROM true THEN
+    RAISE EXCEPTION 'provider_mode_required' USING ERRCODE = '42501';
+  END IF;
+
+  IF p_occurrence_key IS NULL
+     OR (p_occurrence_key COLLATE "C") !~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$'
+     OR p_claim_id IS NULL
+     OR (p_claim_id COLLATE "C") !~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$'
+     OR p_outcome IS NULL
+     OR p_outcome NOT IN ('sent', 'failed', 'ambiguous') THEN
+    RAISE EXCEPTION 'invalid_failure_alert_completion' USING ERRCODE = '22023';
+  END IF;
+
+  db_now := pg_catalog.clock_timestamp();
+
+  SELECT current_alert.state,
+         current_alert.claim_id
+    INTO alert
+    FROM semicolony.cron_read_action_failure_alert_outbox AS current_alert
+   WHERE current_alert.occurrence_key = p_occurrence_key
+   FOR UPDATE;
+
+  IF NOT FOUND THEN
+    outcome := 'not_found';
+    RETURN NEXT;
+    RETURN;
+  END IF;
+
+  IF alert.claim_id IS DISTINCT FROM p_claim_id THEN
+    outcome := 'claim_mismatch';
+    RETURN NEXT;
+    RETURN;
+  END IF;
+
+  IF alert.state IN ('sent', 'failed', 'ambiguous') THEN
+    outcome := CASE WHEN alert.state = p_outcome THEN 'existing' ELSE 'outcome_mismatch' END;
+    RETURN NEXT;
+    RETURN;
+  END IF;
+
+  IF alert.state <> 'claimed' THEN
+    outcome := 'claim_mismatch';
+    RETURN NEXT;
+    RETURN;
+  END IF;
+
+  UPDATE semicolony.cron_read_action_failure_alert_outbox AS current_alert
+     SET state = p_outcome,
+         resolved_at = db_now,
+         updated_at = db_now
+   WHERE current_alert.occurrence_key = p_occurrence_key;
+
+  outcome := 'completed';
+  RETURN NEXT;
+END;
+$$;
+
+REVOKE ALL PRIVILEGES ON TABLE
+  semicolony.cron_read_action_failure_alert_outbox
+FROM PUBLIC, sc_app, sc_provider;
+
+REVOKE ALL ON FUNCTION semicolony.hive_cron_protect_failure_alert()
+FROM PUBLIC, sc_app, sc_provider;
+REVOKE ALL ON FUNCTION semicolony.hive_cron_record_failure_alert()
+FROM PUBLIC, sc_app, sc_provider;
+REVOKE ALL ON FUNCTION semicolony.hive_cron_claim_failure_alert(text, integer)
+FROM PUBLIC, sc_app, sc_provider;
+REVOKE ALL ON FUNCTION semicolony.hive_cron_complete_failure_alert(text, text, text)
+FROM PUBLIC, sc_app, sc_provider;
+
+GRANT EXECUTE ON FUNCTION semicolony.hive_cron_claim_failure_alert(text, integer)
+TO sc_provider;
+GRANT EXECUTE ON FUNCTION semicolony.hive_cron_complete_failure_alert(text, text, text)
+TO sc_provider;