@bluefields/db 0.2.0

package/src/migrations/0007_crawl_jobs.sql

@@ -0,0 +1,65 @@
+-- BlueFields Phase C — crawl jobs.
+--
+-- A "crawl job" is a long-running BFS over a domain. The route returns
+-- a job_id immediately; an Inngest function does the actual BFS,
+-- writing per-page rows as it goes. Customers poll GET /crawl/:id or
+-- subscribe to GET /crawl/:id/stream (SSE) for progress.
+--
+-- The same schema serves /batch/scrape + /extract in Phase D — both
+-- are also "jobs with per-URL results."
+
+CREATE TABLE IF NOT EXISTS crawl_jobs (
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  customer_id UUID NOT NULL REFERENCES customers(id),
+  -- Discriminator: 'crawl' | 'batch_scrape' | 'extract'. One table for
+  -- all three keeps the polling/stream/webhook paths uniform.
+  job_kind TEXT NOT NULL,
+  -- For crawl: the seed URL. For batch_scrape/extract: null (per-URL
+  -- list lives in `params.urls`).
+  start_url TEXT,
+  url_canonical TEXT,
+  status TEXT NOT NULL DEFAULT 'pending',
+  -- All request-time params (maxDepth, maxPages, includes, excludes,
+  -- scrapeOptions, urls list, schema, ...) — kept opaque here so the
+  -- worker can evolve without schema migrations.
+  params JSONB NOT NULL DEFAULT '{}'::jsonb,
+  total_pages_discovered INT NOT NULL DEFAULT 0,
+  total_pages_scraped INT NOT NULL DEFAULT 0,
+  -- Optional: webhook to call when status transitions to terminal.
+  webhook_url TEXT,
+  webhook_secret TEXT,
+  created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  started_at TIMESTAMPTZ,
+  completed_at TIMESTAMPTZ,
+  -- On failure: short human-readable reason.
+  error TEXT
+);
+
+CREATE INDEX IF NOT EXISTS idx_crawl_jobs_customer_time
+  ON crawl_jobs (customer_id, created_at DESC);
+CREATE INDEX IF NOT EXISTS idx_crawl_jobs_status_pending
+  ON crawl_jobs (status, created_at)
+  WHERE status IN ('pending', 'running');
+
+-- Per-page results. Composite PK on (job_id, url_canonical) so a
+-- job can't double-record the same URL (BFS dedup safety net).
+CREATE TABLE IF NOT EXISTS crawl_job_pages (
+  job_id UUID NOT NULL REFERENCES crawl_jobs(id) ON DELETE CASCADE,
+  url_canonical TEXT NOT NULL,
+  -- BFS depth from start_url. 0 for the seed; null for batch_scrape/extract.
+  depth INT,
+  -- Result fields: subset of FetchResult + extraction metadata.
+  content_hash TEXT,
+  response_status INT,
+  fetched_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  extraction_id UUID,
+  -- 'ok' | 'fetch_failed' | 'extract_failed' | 'excluded' | 'depth_limit'
+  status TEXT NOT NULL DEFAULT 'ok',
+  status_detail TEXT,
+  PRIMARY KEY (job_id, url_canonical)
+);
+
+CREATE INDEX IF NOT EXISTS idx_crawl_pages_job_time
+  ON crawl_job_pages (job_id, fetched_at DESC);
+CREATE INDEX IF NOT EXISTS idx_crawl_pages_job_status
+  ON crawl_job_pages (job_id, status);

package/src/migrations/0008_sessions.sql

@@ -0,0 +1,38 @@
+-- BlueFields Phase G — named sessions for cookie/header reuse across scrapes.
+--
+-- One row per (customer_id, name). The cookies + headers blobs are
+-- AES-256-GCM ciphertext (key from SESSION_ENCRYPTION_KEY env). Each
+-- payload is a versioned JSON envelope: `{ v: 1, iv, tag, ciphertext }`
+-- — the encryption layer lives in @bluefields/control/sessions.ts.
+--
+-- Expiry: caller-supplied or default 30 days. A row past expires_at is
+-- treated as missing by `loadSession`; a daily cleanup cron is wired in
+-- chunk-22-style if/when needed (volume should be low — names are
+-- intentional, not request-scoped).
+
+CREATE TABLE IF NOT EXISTS sessions (
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  customer_id UUID NOT NULL REFERENCES customers(id) ON DELETE CASCADE,
+  -- Human-friendly identifier scoped per customer. NOT globally unique;
+  -- two customers can both have a session named 'github-prod'.
+  name TEXT NOT NULL,
+  -- AES-256-GCM ciphertext (versioned JSON envelope, stored as TEXT for
+  -- easy inspection in psql; not a BYTEA because Drizzle's text() type
+  -- works cleanly here).
+  cookies_ciphertext TEXT NOT NULL,
+  -- Optional default headers (Authorization, custom UA) — same envelope.
+  headers_ciphertext TEXT,
+  -- When this session stops being usable. NULL == no expiry (rare).
+  expires_at TIMESTAMPTZ,
+  created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  -- Per-customer uniqueness so save/use lookups are O(1).
+  UNIQUE (customer_id, name)
+);
+
+CREATE INDEX IF NOT EXISTS idx_sessions_customer_name
+  ON sessions (customer_id, name);
+-- Partial index for the cleanup job — only scan rows that have an expiry.
+CREATE INDEX IF NOT EXISTS idx_sessions_expires_at
+  ON sessions (expires_at)
+  WHERE expires_at IS NOT NULL;

package/src/migrations/0009_scrape_cache.sql

@@ -0,0 +1,34 @@
+-- BlueFields Phase A.5 — per-customer scrape cache for /scrape maxAge.
+--
+-- One row per (customer_id, url_canonical). Holds enough to short-circuit
+-- the next /scrape for the same URL without re-running the fetch +
+-- extractor pipeline.
+--
+-- We store an extraction_id pointer rather than denormalising markdown /
+-- structured_data — extractions can be 100KB+ and the cache is a small,
+-- hot lookup. The actual data is one extra SELECT on the extractions
+-- table when the cache hits.
+--
+-- Cross-customer sharing happens at the extractor layer (Review #6 cache
+-- on content_hash + schema_hash + extractor_version). This table is a
+-- per-customer URL-level layer on top of that.
+
+CREATE TABLE IF NOT EXISTS scrape_cache (
+  customer_id UUID NOT NULL REFERENCES customers(id) ON DELETE CASCADE,
+  url_canonical TEXT NOT NULL,
+  -- The fresh-fetched extraction. NULL only when the call was html-only
+  -- (no extractor invoked) — those cache rows can still answer html
+  -- requests when wired to a snapshot. v1: store NULL when no extractor
+  -- ran; serve only html-format requests from cache.
+  extraction_id UUID,
+  content_hash TEXT NOT NULL,
+  final_url TEXT NOT NULL,
+  response_status INT NOT NULL,
+  fetched_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  PRIMARY KEY (customer_id, url_canonical)
+);
+
+-- Lookup by url across customers is rare — main read is the PK lookup
+-- per /scrape call. A by-time index would help if we add a cleanup cron
+-- later, but for v1 we keep the table lean and rely on customer-scoped
+-- TTL behavior (rows are overwritten by upsert on each fresh scrape).

package/src/migrations/0010_neon_defaults.sql

@@ -0,0 +1,99 @@
+-- 0010 — Neon-friendly default partitions + missing usage_period seed.
+--
+-- Why:
+--   Migration 0001 creates partitioned parent tables (snapshots, events,
+--   cost_records, audit_log) and calls partman.create_parent() to spin
+--   up the per-month child partitions. Neon doesn't have pg_partman, so
+--   the partman call is wrapped in a DO-block that no-ops when the
+--   extension is missing. Result: the parent tables exist, but no child
+--   partitions — any INSERT errors out with:
+--     ERROR: no partition of relation "cost_records" found for row
+--
+--   For Neon (and any pg-partman-less Postgres), add a single DEFAULT
+--   partition per parent. New rows route there until/unless the
+--   operator later layers in a real partitioning strategy.
+--
+--   Also seeds a usage_periods row for any customer that lacks one in
+--   the current month — covers the bootstrap admin case where the
+--   customer was inserted via raw SQL without going through the Stripe
+--   checkout webhook that normally creates the period.
+--
+-- Safe to re-run: all statements are IF NOT EXISTS / ON CONFLICT.
+
+-- ── Default partitions ─────────────────────────────────────────────────
+
+CREATE TABLE IF NOT EXISTS snapshots_default
+  PARTITION OF snapshots DEFAULT;
+
+CREATE TABLE IF NOT EXISTS events_default
+  PARTITION OF events DEFAULT;
+
+CREATE TABLE IF NOT EXISTS cost_records_default
+  PARTITION OF cost_records DEFAULT;
+
+CREATE TABLE IF NOT EXISTS audit_log_default
+  PARTITION OF audit_log DEFAULT;
+
+-- ── Backfill usage_periods for customers without an active row ─────────
+--
+-- For each customer whose latest period_end is before today (or who has
+-- no period at all), insert a period covering the current calendar
+-- month with the customer's plan-derived quotas. Numbers below match
+-- the application layer's free-tier plan defaults; customers
+-- on higher tiers should still get a usable period (overstating their
+-- quota slightly is harmless — the Stripe checkout flow re-snapshots
+-- at the next month rollover).
+
+INSERT INTO usage_periods (
+  customer_id, period_start, period_end, plan,
+  events_quota, scrapes_quota, active_subs_quota, overage_cap_events,
+  credits_quota
+)
+SELECT
+  c.id,
+  date_trunc('month', now())::date AS period_start,
+  (date_trunc('month', now()) + interval '1 month - 1 day')::date AS period_end,
+  c.plan,
+  CASE c.plan
+    WHEN 'free' THEN 100
+    WHEN 'starter' THEN 5000
+    WHEN 'growth' THEN 50000
+    WHEN 'scale' THEN 500000
+    ELSE 100
+  END AS events_quota,
+  CASE c.plan
+    WHEN 'free' THEN 100
+    WHEN 'starter' THEN 5000
+    WHEN 'growth' THEN 50000
+    WHEN 'scale' THEN 500000
+    ELSE 100
+  END AS scrapes_quota,
+  CASE c.plan
+    WHEN 'free' THEN 10
+    WHEN 'starter' THEN 50
+    WHEN 'growth' THEN 500
+    WHEN 'scale' THEN 5000
+    ELSE 10
+  END AS active_subs_quota,
+  CASE c.plan
+    WHEN 'free' THEN 100
+    WHEN 'starter' THEN 7500
+    WHEN 'growth' THEN 75000
+    WHEN 'scale' THEN 750000
+    ELSE 100
+  END AS overage_cap_events,
+  CASE c.plan
+    WHEN 'free' THEN 500
+    WHEN 'starter' THEN 25000
+    WHEN 'growth' THEN 100000
+    WHEN 'scale' THEN 500000
+    ELSE 500
+  END AS credits_quota
+FROM customers c
+WHERE NOT EXISTS (
+  SELECT 1 FROM usage_periods up
+   WHERE up.customer_id = c.id
+     AND up.period_start <= now()::date
+     AND up.period_end >= now()::date
+)
+ON CONFLICT (customer_id, period_start) DO NOTHING;

package/src/migrations/0011_credits_consumed_numeric.sql

@@ -0,0 +1,12 @@
+-- 0011_credits_consumed_numeric.sql
+--
+-- Cache-hit fractional charge (0.2 credit — Option A pricing, 2026-06-16)
+-- needs a sub-integer credits column. Widen usage_periods.credits_consumed from
+-- INTEGER to NUMERIC(14,4): up to 10 integer digits (>> any plan's credit total)
+-- plus 4 decimal places. Lossless for existing integer values; not destructive.
+
+ALTER TABLE usage_periods
+  ALTER COLUMN credits_consumed TYPE NUMERIC(14, 4) USING credits_consumed::numeric(14, 4);
+
+ALTER TABLE usage_periods
+  ALTER COLUMN credits_consumed SET DEFAULT 0;

package/src/migrations/0012_crawl_page_attestation.sql

@@ -0,0 +1,10 @@
+-- 0012_crawl_page_attestation.sql
+--
+-- Per-page attestation for crawl / batch / extract (2026-06-21).
+-- Stores the signed v1 manifest ({ manifest, signature }) for each crawled page
+-- when ATTEST_SYNC_ROUTES is enabled, so GET /crawl/:id can return it. Nullable
+-- + additive: existing rows stay NULL and the default-off path is unaffected.
+-- Idempotent (IF NOT EXISTS) so the runner can safely re-apply it.
+
+ALTER TABLE crawl_job_pages
+  ADD COLUMN IF NOT EXISTS attestation jsonb;

package/src/migrations/0013_customer_deleted_at.sql

@@ -0,0 +1,5 @@
+-- Account deletion (GDPR right to erasure): mark a customer as deleted/anonymized.
+-- The row is kept (anonymized in place by @bluefields/control deleteAccount) for
+-- billing/audit integrity; NULL = active. Additive + idempotent (the migrate
+-- runner re-applies every .sql file, so this MUST stay re-runnable).
+ALTER TABLE customers ADD COLUMN IF NOT EXISTS deleted_at timestamptz;