@bluefields/db 0.2.0

package/src/migrations/0001_initial.sql

@@ -0,0 +1,398 @@
+-- BlueFields v1 initial schema.
+--
+-- Source: docs/03_implementation_plan_detailed.md "The data model" section
+-- (lines ~100-480) + the `host_records` block at line 1271. Per-column
+-- decisions are traced to Reviews #2-#11 in docs/05_decision_log.md.
+--
+-- SPEC DEVIATIONS (intentional, required for migration to actually run):
+--   - `snapshots` and `events` use composite primary keys (id, fetched_at)
+--     and (id, created_at). The spec wrote `id UUID PRIMARY KEY` for these,
+--     but Postgres requires the partition column in every unique constraint
+--     on a partitioned table. The composite-PK pattern matches what
+--     `cost_records` and `audit_log` already use in the spec.
+--   - FOREIGN KEY constraints from non-partitioned tables to
+--     `snapshots(id)` and `events(id)` are removed: a DB-level FK to a
+--     partitioned table's `id` alone is not expressible. Affected:
+--       * `extractions.snapshot_id`     — column kept, FK dropped
+--       * `event_feedback.event_id`     — column kept, FK dropped
+--     Referential integrity is enforced by application code only.
+
+-- ============================================================================
+-- Extensions
+-- ============================================================================
+
+CREATE EXTENSION IF NOT EXISTS pgcrypto;
+
+-- pg_partman + pg_cron are OPTIONAL. They give us automatic partition
+-- pre-creation and hourly maintenance on a self-managed Postgres. Managed
+-- Postgres providers (Neon, etc.) often don't ship them. We wrap the
+-- extension creation and all calls into DO blocks that warn-and-skip on
+-- absence — partitioned tables themselves still work, you just have to
+-- manage partitions externally (e.g. a cron job that calls partman from a
+-- self-managed Postgres, or pre-create partitions manually).
+DO $$
+BEGIN
+  EXECUTE 'CREATE EXTENSION IF NOT EXISTS pg_partman';
+EXCEPTION WHEN OTHERS THEN
+  RAISE NOTICE 'pg_partman not available on this Postgres — partition pre-creation skipped';
+END $$;
+
+DO $$
+BEGIN
+  EXECUTE 'CREATE EXTENSION IF NOT EXISTS pg_cron';
+EXCEPTION WHEN OTHERS THEN
+  RAISE NOTICE 'pg_cron not available on this Postgres — partman maintenance must be scheduled externally';
+END $$;
+
+-- ============================================================================
+-- Tables
+-- ============================================================================
+
+-- Customers and auth ---------------------------------------------------------
+CREATE TABLE customers (
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  email TEXT UNIQUE NOT NULL,
+  org_name TEXT,
+  plan TEXT NOT NULL DEFAULT 'free',
+  stripe_customer_id TEXT,
+  created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+CREATE TABLE api_keys (
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  customer_id UUID NOT NULL REFERENCES customers(id),
+  key_hash TEXT NOT NULL UNIQUE,
+  key_prefix TEXT NOT NULL,
+  name TEXT,
+  last_used_at TIMESTAMPTZ,
+  created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  revoked_at TIMESTAMPTZ
+);
+
+-- Subscriptions (watch mode) -------------------------------------------------
+CREATE TABLE subscriptions (
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  customer_id UUID NOT NULL REFERENCES customers(id),
+  url TEXT NOT NULL,
+  url_canonical TEXT NOT NULL,
+  canon_version SMALLINT NOT NULL DEFAULT 1,
+  schema JSONB,
+  webhook_url TEXT,
+  -- Dual-key webhook secret scheme (Review #7)
+  webhook_secret_current TEXT,
+  webhook_secret_previous TEXT,
+  webhook_secret_rotated_at TIMESTAMPTZ,
+  webhook_url_state TEXT NOT NULL DEFAULT 'unverified',
+  webhook_consecutive_failures INT NOT NULL DEFAULT 0,
+  min_interval_seconds INT NOT NULL DEFAULT 3600,
+  max_interval_seconds INT NOT NULL DEFAULT 86400,
+  current_interval_seconds INT NOT NULL DEFAULT 3600,
+  status TEXT NOT NULL DEFAULT 'active',
+  next_poll_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  inngest_event_id TEXT,
+  created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  last_polled_at TIMESTAMPTZ,
+  last_change_at TIMESTAMPTZ,
+  poll_count INT NOT NULL DEFAULT 0,
+  change_count INT NOT NULL DEFAULT 0,
+  intent TEXT,
+  diff_rules JSONB NOT NULL DEFAULT '{"rules":[]}'::jsonb,
+  last_meaningful_extraction_id UUID,
+  -- Adaptive polling (Review #9)
+  detection_latency_target_seconds INT,
+  vertical_hint TEXT,
+  ewma_change_interval_seconds INT,
+  last_change_observed_at TIMESTAMPTZ,
+  consecutive_no_change_polls INT NOT NULL DEFAULT 0,
+  average_observed_interval_seconds INT,
+  http_validator JSONB,
+  cyclic_pattern_hint JSONB
+);
+
+CREATE INDEX idx_subs_customer ON subscriptions(customer_id);
+CREATE INDEX idx_subs_canonical ON subscriptions(url_canonical) WHERE status = 'active';
+CREATE INDEX idx_subs_status ON subscriptions(status) WHERE status != 'expired';
+-- Critical: pg_cron dispatcher hits this every minute (Review #1)
+CREATE INDEX idx_subs_due ON subscriptions(next_poll_at) WHERE status = 'active';
+
+-- Snapshots (immutable history) — partitioned by month on fetched_at ---------
+CREATE TABLE snapshots (
+  id UUID NOT NULL DEFAULT gen_random_uuid(),
+  url_canonical TEXT NOT NULL,
+  url_surt TEXT NOT NULL,
+  canon_version SMALLINT NOT NULL,
+  content_hash TEXT NOT NULL,
+  fetched_at TIMESTAMPTZ NOT NULL,
+  fetcher_id TEXT NOT NULL,
+  proxy_egress_ip INET NOT NULL,
+  user_agent TEXT NOT NULL,
+  response_status INT NOT NULL,
+  response_headers JSONB NOT NULL,
+  raw_html_r2_key TEXT NOT NULL,
+  raw_html_size_bytes BIGINT NOT NULL,
+  fetch_duration_ms INT NOT NULL,
+  attestation_signature TEXT NOT NULL,
+  attestation_key_id TEXT NOT NULL,
+  attestation_algorithm TEXT NOT NULL DEFAULT 'EdDSA',
+  manifest_version SMALLINT NOT NULL DEFAULT 1,
+  encryption_key_id TEXT,
+  cost_micro_usd INT NOT NULL,
+  created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  -- Composite PK required for partitioned tables (see header note).
+  PRIMARY KEY (id, fetched_at)
+) PARTITION BY RANGE (fetched_at);
+
+CREATE INDEX idx_snap_url_time ON snapshots(url_canonical, fetched_at DESC);
+CREATE INDEX idx_snap_surt_time ON snapshots(url_surt, fetched_at DESC);
+CREATE INDEX idx_snap_hash ON snapshots(content_hash);
+
+-- Host-specific extractor overrides (Review #6; v2-reserved, empty in v1) ---
+CREATE TABLE host_extractors (
+  host TEXT PRIMARY KEY,
+  extractor_id TEXT NOT NULL,
+  enabled BOOLEAN NOT NULL DEFAULT true,
+  schema_compatibility TEXT[],
+  notes TEXT,
+  created_at TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+-- Extractions (interpreted layer) -------------------------------------------
+CREATE TABLE extractions (
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  -- snapshot_id: column kept, FK dropped (snapshots is partitioned; see header).
+  snapshot_id UUID NOT NULL,
+  subscription_id UUID REFERENCES subscriptions(id),
+  customer_id UUID NOT NULL REFERENCES customers(id),
+  content_hash TEXT NOT NULL,
+  schema_hash TEXT,
+  extractor_version TEXT NOT NULL,
+  markdown TEXT,
+  structured_data JSONB,
+  extraction_status TEXT NOT NULL,
+  status_detail TEXT,
+  encryption_key_id TEXT,
+  llm_model TEXT,
+  llm_input_tokens INT,
+  llm_output_tokens INT,
+  cost_micro_usd INT NOT NULL,
+  duration_ms INT NOT NULL DEFAULT 0,
+  created_at TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+CREATE INDEX idx_ext_sub_time ON extractions(subscription_id, created_at DESC);
+CREATE INDEX idx_ext_customer ON extractions(customer_id);
+CREATE INDEX idx_ext_snap ON extractions(snapshot_id);
+-- Cache lookup index: hot path. Cross-snapshot dedup keys on content (Review #6).
+CREATE UNIQUE INDEX idx_ext_cache
+  ON extractions(content_hash, schema_hash, extractor_version);
+
+-- Events — partitioned by month on created_at -------------------------------
+CREATE TABLE events (
+  id UUID NOT NULL DEFAULT gen_random_uuid(),
+  subscription_id UUID NOT NULL REFERENCES subscriptions(id),
+  customer_id UUID NOT NULL REFERENCES customers(id),
+  event_type TEXT NOT NULL,
+  previous_extraction_id UUID REFERENCES extractions(id),
+  current_extraction_id UUID REFERENCES extractions(id),
+  diff JSONB,
+  semantic_classification TEXT,
+  -- Delivery state machine (Review #7)
+  delivery_state TEXT NOT NULL DEFAULT 'pending',
+  delivery_attempts INT NOT NULL DEFAULT 0,
+  next_retry_at TIMESTAMPTZ,
+  last_attempt_at TIMESTAMPTZ,
+  last_response_status INT,
+  last_response_body_excerpt TEXT,
+  delivered_at TIMESTAMPTZ,
+  created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  -- Composite PK required for partitioned tables.
+  PRIMARY KEY (id, created_at)
+) PARTITION BY RANGE (created_at);
+
+CREATE INDEX idx_events_sub_time ON events(subscription_id, created_at DESC);
+CREATE INDEX idx_events_customer_time ON events(customer_id, created_at DESC);
+CREATE INDEX idx_events_undelivered ON events(created_at) WHERE delivered_at IS NULL;
+-- Retry-due index: hot path for delivery loop (Review #7)
+CREATE INDEX idx_events_due_for_retry ON events(next_retry_at)
+  WHERE delivery_state = 'pending' AND next_retry_at IS NOT NULL;
+CREATE INDEX idx_events_dead_lettered ON events(customer_id, created_at DESC)
+  WHERE delivery_state = 'dead_lettered';
+
+-- Customer feedback on event classification (Review #5) ----------------------
+CREATE TABLE event_feedback (
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  -- event_id: column kept, FK dropped (events is partitioned; see header).
+  event_id UUID,
+  subscription_id UUID NOT NULL REFERENCES subscriptions(id),
+  customer_id UUID NOT NULL REFERENCES customers(id),
+  verdict TEXT NOT NULL,
+  notes TEXT,
+  reviewed_by_us BOOLEAN NOT NULL DEFAULT false,
+  follow_up_action TEXT,
+  created_at TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+CREATE INDEX idx_feedback_customer ON event_feedback(customer_id);
+CREATE INDEX idx_feedback_unreviewed ON event_feedback(created_at) WHERE reviewed_by_us = false;
+
+-- Cost tracking — partitioned by month on recorded_at (Review #11) ----------
+CREATE TABLE cost_records (
+  id BIGSERIAL,
+  customer_id UUID NOT NULL REFERENCES customers(id),
+  subscription_id UUID REFERENCES subscriptions(id),
+  url_canonical TEXT,
+  cost_type TEXT NOT NULL,
+  cost_micro_usd INT NOT NULL,
+  units NUMERIC,
+  cost_subtype TEXT,
+  recorded_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  PRIMARY KEY (id, recorded_at)
+) PARTITION BY RANGE (recorded_at);
+
+CREATE INDEX idx_cost_customer_time ON cost_records(customer_id, recorded_at);
+CREATE INDEX idx_cost_url ON cost_records(url_canonical, recorded_at);
+CREATE INDEX idx_cost_type_time ON cost_records(cost_type, recorded_at);
+
+-- Unified audit log — partitioned by month on recorded_at (Review #11) ------
+CREATE TABLE audit_log (
+  id BIGSERIAL,
+  customer_id UUID REFERENCES customers(id),
+  actor_type TEXT NOT NULL,
+  actor_id TEXT,
+  category TEXT NOT NULL,
+  action TEXT NOT NULL,
+  resource_type TEXT,
+  resource_id TEXT,
+  metadata JSONB,
+  ip_address INET,
+  user_agent TEXT,
+  recorded_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  PRIMARY KEY (id, recorded_at)
+) PARTITION BY RANGE (recorded_at);
+
+CREATE INDEX idx_audit_customer ON audit_log(customer_id, recorded_at DESC);
+CREATE INDEX idx_audit_security ON audit_log(category, recorded_at DESC)
+  WHERE category IN ('security', 'auth');
+CREATE INDEX idx_audit_actor ON audit_log(actor_type, actor_id, recorded_at DESC);
+
+-- Per-customer per-billing-period usage counters (Reviews #8, #10) ----------
+CREATE TABLE usage_periods (
+  customer_id UUID NOT NULL REFERENCES customers(id),
+  period_start DATE NOT NULL,
+  period_end DATE NOT NULL,
+  plan TEXT NOT NULL,
+  -- Quotas (snapshot at period start)
+  events_quota INT NOT NULL,
+  scrapes_quota INT NOT NULL,
+  active_subs_quota INT NOT NULL,
+  overage_cap_events INT NOT NULL,
+  -- Counters
+  events_billable INT NOT NULL DEFAULT 0,
+  events_nonbillable INT NOT NULL DEFAULT 0,
+  scrapes_consumed INT NOT NULL DEFAULT 0,
+  active_subs_max INT NOT NULL DEFAULT 0,
+  -- MCP credits (Review #10)
+  credits_quota INT NOT NULL DEFAULT 0,
+  credits_consumed INT NOT NULL DEFAULT 0,
+  -- Alert state
+  quota_alert_50_sent BOOLEAN NOT NULL DEFAULT false,
+  quota_alert_75_sent BOOLEAN NOT NULL DEFAULT false,
+  quota_alert_90_sent BOOLEAN NOT NULL DEFAULT false,
+  quota_alert_100_sent BOOLEAN NOT NULL DEFAULT false,
+  credits_alert_50_sent BOOLEAN NOT NULL DEFAULT false,
+  credits_alert_75_sent BOOLEAN NOT NULL DEFAULT false,
+  credits_alert_90_sent BOOLEAN NOT NULL DEFAULT false,
+  credits_alert_100_sent BOOLEAN NOT NULL DEFAULT false,
+  polling_paused_at TIMESTAMPTZ,
+  mcp_paused_at TIMESTAMPTZ,
+  spend_cap_micro_usd BIGINT,
+  PRIMARY KEY (customer_id, period_start)
+);
+
+CREATE INDEX idx_usage_unalerted ON usage_periods(customer_id)
+  WHERE quota_alert_100_sent = false;
+
+-- Per-host config (Review #4) -----------------------------------------------
+CREATE TABLE host_records (
+  host TEXT PRIMARY KEY,
+  fetch_tier SMALLINT NOT NULL DEFAULT 1,
+  fetch_tier_updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  preferred_proxy_provider TEXT,
+  max_concurrent_fetches SMALLINT NOT NULL DEFAULT 2,
+  observed_anti_bot TEXT[],
+  robots_status TEXT NOT NULL DEFAULT 'unknown',
+  takedown_status TEXT NOT NULL DEFAULT 'active',
+  country_hint CHAR(2),
+  notes TEXT,
+  created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+CREATE INDEX idx_host_tier ON host_records(fetch_tier);
+CREATE INDEX idx_host_takedown ON host_records(takedown_status) WHERE takedown_status != 'active';
+
+-- ============================================================================
+-- pg_partman bootstrap (skipped if pg_partman not installed)
+-- ============================================================================
+-- 1-month range partitions, 4 months pre-created. Each block is independent
+-- so a partial install (extension partially loaded, one CALL fails) still
+-- lets the others succeed.
+
+DO $$
+BEGIN
+  -- Check for the partman schema (not just pg_extension) because some managed
+  -- Postgres providers (Neon) register a pg_extension row without actually
+  -- creating the schema, which would make CALL partman.* fail mid-transaction.
+  IF EXISTS (SELECT 1 FROM information_schema.schemata WHERE schema_name = 'partman') THEN
+    CALL partman.create_parent(
+      p_parent_table => 'public.snapshots',
+      p_control      => 'fetched_at',
+      p_type         => 'range',
+      p_interval     => '1 month',
+      p_premake      => 4
+    );
+    CALL partman.create_parent(
+      p_parent_table => 'public.events',
+      p_control      => 'created_at',
+      p_type         => 'range',
+      p_interval     => '1 month',
+      p_premake      => 4
+    );
+    CALL partman.create_parent(
+      p_parent_table => 'public.cost_records',
+      p_control      => 'recorded_at',
+      p_type         => 'range',
+      p_interval     => '1 month',
+      p_premake      => 4
+    );
+    CALL partman.create_parent(
+      p_parent_table => 'public.audit_log',
+      p_control      => 'recorded_at',
+      p_type         => 'range',
+      p_interval     => '1 month',
+      p_premake      => 4
+    );
+  ELSE
+    RAISE NOTICE 'partman schema not present — skipping partition pre-creation. Tables exist and accept rows; partitions must be created manually or after migrating to a partman-capable Postgres.';
+  END IF;
+END $$;
+
+-- ============================================================================
+-- pg_cron — hourly partman maintenance (skipped if pg_cron not installed)
+-- ============================================================================
+
+DO $$
+BEGIN
+  -- Same schema-presence check pattern as the partman block above.
+  IF EXISTS (SELECT 1 FROM information_schema.schemata WHERE schema_name = 'cron')
+     AND EXISTS (SELECT 1 FROM information_schema.schemata WHERE schema_name = 'partman') THEN
+    PERFORM cron.schedule(
+      'partman-maintenance',
+      '0 * * * *',
+      $cron$CALL partman.run_maintenance_proc()$cron$
+    );
+  ELSE
+    RAISE NOTICE 'cron and/or partman schema not present — partman maintenance not scheduled.';
+  END IF;
+END $$;

package/src/migrations/0002_pg_cron.sql

@@ -0,0 +1,109 @@
+-- BlueFields chunk 6 — dispatch infrastructure.
+--
+-- Per Review #1, the architecture splits "when to poll" (pg_cron + a
+-- claim-then-enqueue SQL function) from "what happens during a poll"
+-- (Inngest step functions). This migration creates the SQL function
+-- and the queue table that the API worker drains.
+--
+-- The cron.schedule() call is wrapped in a DO block because not every
+-- Postgres host has pg_cron (Neon free tier doesn't). When pg_cron is
+-- missing, the operator must trigger dispatch_due_subscriptions()
+-- externally — Inngest's own cron, a Fly scheduled machine, or a manual
+-- SELECT in psql. The function itself works regardless.
+
+-- ============================================================================
+-- Queue table — API worker drains this
+-- ============================================================================
+
+CREATE TABLE IF NOT EXISTS inngest_dispatch_queue (
+  id BIGSERIAL PRIMARY KEY,
+  subscription_id UUID NOT NULL REFERENCES subscriptions(id),
+  -- The Inngest event name to send (lets the dispatch function discriminate
+  -- between subscription.poll / webhook.ping / webhook.deliver later)
+  event_name TEXT NOT NULL,
+  -- Payload to ship to Inngest as the event data
+  event_data JSONB NOT NULL DEFAULT '{}'::jsonb,
+  -- When this row was created (the API worker SLO is to drain within ~5s)
+  enqueued_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  -- Optimistic lock — workers SET claimed_at and claimed_by to mark in-flight rows
+  claimed_at TIMESTAMPTZ,
+  claimed_by TEXT
+);
+
+CREATE INDEX IF NOT EXISTS idx_dispatch_queue_unclaimed
+  ON inngest_dispatch_queue (enqueued_at)
+  WHERE claimed_at IS NULL;
+CREATE INDEX IF NOT EXISTS idx_dispatch_queue_subscription
+  ON inngest_dispatch_queue (subscription_id);
+
+-- ============================================================================
+-- dispatch_due_subscriptions(): SELECT + claim-sentinel + enqueue
+-- ============================================================================
+--
+-- Runs every minute (via pg_cron OR an external scheduler). Selects all
+-- subscriptions where next_poll_at is due AND polling isn't paused AND
+-- the webhook isn't disabled. For each, advances next_poll_at by a 5-minute
+-- claim sentinel (so this same row can't be re-claimed by a concurrent
+-- dispatch), and enqueues a `subscription.poll` event for the API worker
+-- to relay to Inngest.
+--
+-- Returns the count of subscriptions enqueued for observability.
+
+CREATE OR REPLACE FUNCTION dispatch_due_subscriptions()
+RETURNS INTEGER
+LANGUAGE plpgsql
+AS $$
+DECLARE
+  claimed_count INTEGER := 0;
+BEGIN
+  WITH due AS (
+    SELECT s.id
+      FROM subscriptions s
+     WHERE s.next_poll_at <= now()
+       AND s.status = 'active'
+       AND (s.webhook_url_state IS DISTINCT FROM 'disabled')
+       -- Don't redispatch rows already mid-claim. The 5-minute sentinel
+       -- (set below) is the lock; this guard is belt-and-suspenders.
+       FOR UPDATE SKIP LOCKED
+  ),
+  -- Advance next_poll_at by the claim-sentinel duration so the row
+  -- doesn't get re-selected by the next dispatch run before the worker
+  -- has had a chance to update it.
+  claimed AS (
+    UPDATE subscriptions
+       SET next_poll_at = now() + INTERVAL '5 minutes'
+     WHERE id IN (SELECT id FROM due)
+     RETURNING id
+  )
+  INSERT INTO inngest_dispatch_queue (subscription_id, event_name, event_data)
+  SELECT c.id, 'subscription.poll', jsonb_build_object('subscription_id', c.id)
+    FROM claimed c;
+
+  GET DIAGNOSTICS claimed_count = ROW_COUNT;
+  RETURN claimed_count;
+END;
+$$;
+
+-- ============================================================================
+-- pg_cron — schedule dispatch every minute (skipped if pg_cron absent)
+-- ============================================================================
+
+DO $$
+BEGIN
+  IF EXISTS (SELECT 1 FROM information_schema.schemata WHERE schema_name = 'cron') THEN
+    -- Unschedule the previous version if any (idempotent re-runs).
+    BEGIN
+      PERFORM cron.unschedule('dispatch-subscriptions');
+    EXCEPTION WHEN OTHERS THEN
+      -- First run — nothing to unschedule.
+      NULL;
+    END;
+    PERFORM cron.schedule(
+      'dispatch-subscriptions',
+      '* * * * *',
+      $cron$SELECT dispatch_due_subscriptions()$cron$
+    );
+  ELSE
+    RAISE NOTICE 'pg_cron not installed — schedule dispatch_due_subscriptions() externally (Inngest cron or Fly scheduled machine).';
+  END IF;
+END $$;

package/src/migrations/0003_webhook_secret_cleanup.sql

@@ -0,0 +1,55 @@
+-- BlueFields chunk 16 — webhook secret rotation cleanup.
+--
+-- Per Review #7, the previous webhook secret is kept for 24 hours after
+-- rotation so the customer has time to deploy the new secret. This
+-- migration creates a SQL function that nulls expired _previous values
+-- and schedules it nightly via pg_cron.
+--
+-- The function is the same shape as the JS-side `pruneExpiredPreviousSecrets`
+-- in the application layer — either one works in
+-- isolation. The SQL version lets pg_cron drive it without a worker; the
+-- JS version is the fallback when pg_cron is not installed (Neon free tier).
+
+-- ============================================================================
+-- prune_expired_previous_webhook_secrets(): scrub the previous-secret column
+-- ============================================================================
+
+CREATE OR REPLACE FUNCTION prune_expired_previous_webhook_secrets()
+RETURNS INTEGER
+LANGUAGE plpgsql
+AS $$
+DECLARE
+  affected INTEGER := 0;
+BEGIN
+  UPDATE subscriptions
+     SET webhook_secret_previous = NULL
+   WHERE webhook_secret_previous IS NOT NULL
+     AND webhook_secret_rotated_at IS NOT NULL
+     AND webhook_secret_rotated_at + INTERVAL '24 hours' < now();
+
+  GET DIAGNOSTICS affected = ROW_COUNT;
+  RETURN affected;
+END;
+$$;
+
+-- ============================================================================
+-- pg_cron — run nightly at 03:17 UTC (off-peak; staggered from other jobs)
+-- ============================================================================
+
+DO $$
+BEGIN
+  IF EXISTS (SELECT 1 FROM information_schema.schemata WHERE schema_name = 'cron') THEN
+    BEGIN
+      PERFORM cron.unschedule('prune-expired-previous-webhook-secrets');
+    EXCEPTION WHEN OTHERS THEN
+      NULL;
+    END;
+    PERFORM cron.schedule(
+      'prune-expired-previous-webhook-secrets',
+      '17 3 * * *',
+      $cron$SELECT prune_expired_previous_webhook_secrets()$cron$
+    );
+  ELSE
+    RAISE NOTICE 'pg_cron not installed — drive prune_expired_previous_webhook_secrets() externally (Inngest scheduled function or worker).';
+  END IF;
+END $$;

package/src/migrations/0004_events_pg_notify.sql

@@ -0,0 +1,39 @@
+-- BlueFields chunk 20 — pg_notify on events INSERT for SSE streaming.
+--
+-- Per Review #7, customers can subscribe to events via Server-Sent Events
+-- in addition to webhooks. The SSE endpoint listens on a per-subscription
+-- channel (`subscription_event_<subscription_id>`) via Postgres LISTEN.
+-- This migration installs the trigger that NOTIFY's whenever a row hits
+-- the events table — regardless of which code path inserted it
+-- (auto-disable's subscription.error, chunk 21's subscription.change, etc.).
+--
+-- The payload is minimal — just the event id + type + creation timestamp.
+-- The SSE handler fetches the full row when it dispatches; this keeps the
+-- NOTIFY payload under the 8KB Postgres limit even on huge events.
+
+CREATE OR REPLACE FUNCTION events_notify_on_insert()
+RETURNS TRIGGER
+LANGUAGE plpgsql
+AS $$
+DECLARE
+  channel TEXT;
+  payload JSONB;
+BEGIN
+  channel := 'subscription_event_' || NEW.subscription_id::text;
+  payload := jsonb_build_object(
+    'event_id', NEW.id,
+    'event_type', NEW.event_type,
+    'customer_id', NEW.customer_id,
+    'created_at', to_char(NEW.created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')
+  );
+  PERFORM pg_notify(channel, payload::text);
+  RETURN NEW;
+END;
+$$;
+
+-- DROP first so re-runs are idempotent. The trigger fires on every INSERT.
+DROP TRIGGER IF EXISTS events_notify_trigger ON events;
+CREATE TRIGGER events_notify_trigger
+  AFTER INSERT ON events
+  FOR EACH ROW
+  EXECUTE FUNCTION events_notify_on_insert();

package/src/migrations/0005_subscription_consecutive_failures.sql

@@ -0,0 +1,19 @@
+-- BlueFields chunk-21 follow-up — subscription poll auto-pause column.
+--
+-- Per chunk 21 spec: after N consecutive subscription failures (fetch
+-- exceptions, extractor exceptions, etc. — NOT webhook deliveries, which
+-- already have webhook_consecutive_failures), the subscription is
+-- auto-paused. The customer must reactivate via PATCH /subscriptions/:id
+-- with status='active'.
+--
+-- This column is SEPARATE from webhook_consecutive_failures (Review #7)
+-- — webhook failures and poll failures have different meanings and
+-- different recovery semantics.
+
+ALTER TABLE subscriptions
+  ADD COLUMN IF NOT EXISTS consecutive_failures INTEGER NOT NULL DEFAULT 0;
+
+-- Index supports the dashboard "subscriptions with active alerts" view.
+CREATE INDEX IF NOT EXISTS idx_subs_consecutive_failures
+  ON subscriptions (consecutive_failures)
+  WHERE consecutive_failures > 0;

package/src/migrations/0006_notification_preferences.sql

@@ -0,0 +1,27 @@
+-- BlueFields — notification preferences (billing-trust alerts).
+--
+-- One row per customer. Absence of a row = "default opt-in" — the
+-- notification-check queries LEFT JOIN this table and COALESCE the
+-- enabled flags to true. No backfill needed for existing customers.
+
+CREATE TABLE IF NOT EXISTS notification_preferences (
+  customer_id                       UUID PRIMARY KEY REFERENCES customers(id),
+  cost_anomaly_enabled              BOOLEAN     NOT NULL DEFAULT TRUE,
+  cost_anomaly_threshold_multiplier NUMERIC(4,2) NOT NULL DEFAULT 2.00,
+  delivery_health_enabled           BOOLEAN     NOT NULL DEFAULT TRUE,
+  delivery_health_threshold_pct     SMALLINT    NOT NULL DEFAULT 25,
+  email_override                    TEXT,
+  last_cost_alert_at                TIMESTAMPTZ,
+  last_delivery_alert_at            TIMESTAMPTZ,
+  created_at                        TIMESTAMPTZ NOT NULL DEFAULT now(),
+  updated_at                        TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+-- Partial indices for the cron's "who's eligible" scan.
+CREATE INDEX IF NOT EXISTS idx_notif_prefs_cost_eligible
+  ON notification_preferences (customer_id)
+  WHERE cost_anomaly_enabled = TRUE;
+
+CREATE INDEX IF NOT EXISTS idx_notif_prefs_delivery_eligible
+  ON notification_preferences (customer_id)
+  WHERE delivery_health_enabled = TRUE;