openship 0.5.0 → 0.6.1

package/dist/server/lua/pipe_stream.lua

@@ -2,6 +2,14 @@
 -- content_by_lua: SSE endpoint for real-time request log streaming.
 -- GET /logs/stream?domain=example.com
 -- Internal only - 127.0.0.1:9145
+--
+-- Reads the SAME ring buffer site_logger.lua writes (rlog:{domain}:seq +
+-- rlog:{domain}:{slot}), by a PER-CONNECTION cursor. Nothing is consumed: N
+-- concurrent watchers each walk the ring independently and none steals another's
+-- frames. This is what replaced the old single-subscriber log_pipe:q queue —
+-- a containerized edge tails this over `docker exec curl`, and a curl that
+-- outlives its client (the daemon buffers its stdout, so no EPIPE) can no longer
+-- drain the queue out from under the next reader.
 
 local sh = ngx.shared.request_data
 if not sh then
@@ -20,12 +28,10 @@ end
 domain = domain:lower()
 if domain:sub(1, 4) == "www." then domain = domain:sub(5) end
 
-local SUB_KEY   = "log_pipe:sub:" .. domain
-local QUEUE_KEY = "log_pipe:q:"   .. domain
-
--- Clear stale queue entries, mark subscriber active
-sh:delete(QUEUE_KEY)
-sh:set(SUB_KEY, true, 30)
+-- MUST match site_logger.lua's RING — same slot arithmetic on both ends.
+local RING    = 1000
+local SEQ_KEY = "rlog:" .. domain .. ":seq"
+local slot_key = function(seq) return "rlog:" .. domain .. ":" .. (seq % RING) end
 
 ngx.header["Content-Type"]      = "text/event-stream"
 ngx.header["Cache-Control"]     = "no-cache, no-store"
@@ -36,55 +42,62 @@ ngx.header["X-Accel-Buffering"] = "no"
 -- Without this, ngx.flush on an empty buffer sends chunked-EOF (0\r\n\r\n)
 -- and terminates the response immediately.
 ngx.print(": connected\n\n")
-if not ngx.flush(true) then
-    sh:delete(SUB_KEY)
-    return
-end
+if not ngx.flush(true) then return end
 
-local started  = ngx.now()
-local last_hb  = started
-local last_ref = started
+-- Start at the current head: stream only requests that arrive AFTER connect.
+-- History up to this point is served separately by /logs/recent, and the client
+-- dedups the two by the deterministic `{host}:{seq}` id, so a row seen by both
+-- collapses to one.
+local cursor  = tonumber(sh:get(SEQ_KEY)) or 0
+local started = ngx.now()
+local last_hb = started
 
 while true do
-    -- Max 1 hour per connection
-    if ngx.now() - started > 3600 then
-        sh:delete(SUB_KEY)
-        return
-    end
+    -- Max 1 hour per connection.
+    if ngx.now() - started > 3600 then return end
+
+    local head = tonumber(sh:get(SEQ_KEY)) or 0
+
+    -- If we've fallen more than a full ring behind, the oldest unread slots have
+    -- already been overwritten. Jump to the oldest slot still holding its own
+    -- entry (head-RING maps to head's slot, so head-RING+1 is the oldest live one)
+    -- and skip the lost range — same bound the old 2000-entry queue had.
+    if cursor < head - RING then cursor = head - RING end
 
-    -- Drain up to 100 queued entries per cycle
     local sent = 0
-    while sent < 100 do
-        local entry = sh:rpop(QUEUE_KEY)
-        if not entry then break end
-        ngx.print("event: request\ndata: ", entry, "\n\n")
-        sent = sent + 1
+    while cursor < head and sent < 100 do
+        local next_seq = cursor + 1
+        local entry = sh:get(slot_key(next_seq))
+        if entry then
+            ngx.print("event: request\ndata: ", entry, "\n\n")
+            sent = sent + 1
+            cursor = next_seq
+        elseif next_seq < head then
+            -- A later seq already exists, so this can't be the incr(seq)→set(slot)
+            -- write window: the slot is genuinely lost (expired or wrapped). Skip it.
+            cursor = next_seq
+        else
+            -- next_seq == head: the newest entry's slot may not be written yet
+            -- (the microsecond window between incr(seq) and set(slot)). Retry it
+            -- next cycle without advancing.
+            break
+        end
     end
 
     if sent > 0 then
-        if not ngx.flush(true) then
-            sh:delete(SUB_KEY)
-            return
-        end
+        if not ngx.flush(true) then return end
     end
 
     local now = ngx.now()
 
-    -- Heartbeat every 15s
+    -- Heartbeat every 15s. Its flush is also how a dead client is detected: once
+    -- the reader (a `docker exec curl`) goes away and its socket closes, this
+    -- flush fails and the connection tears down here.
     if now - last_hb > 15 then
         ngx.print(": ping\n\n")
-        if not ngx.flush(true) then
-            sh:delete(SUB_KEY)
-            return
-        end
+        if not ngx.flush(true) then return end
         last_hb = now
     end
 
-    -- Refresh subscriber TTL every 10s
-    if now - last_ref > 10 then
-        sh:set(SUB_KEY, true, 30)
-        last_ref = now
-    end
-
     ngx.sleep(0.05)
 end

package/dist/server/lua/site_logger.lua

@@ -8,17 +8,33 @@
 --   s:{domain}:{epoch_min}:i        bandwidth in  bytes         (TTL 24h)
 --   s:{domain}:{epoch_min}:o        bandwidth out bytes         (TTL 24h)
 --   s:{domain}:{epoch_min}:t        response time sum (seconds) (TTL 24h)
---   s:{domain}:{epoch_min}:u        unique (non-static) reqs    (TTL 24h)
---   g:{domain}:{YYYYMMDD}:{CC}      country hit count           (TTL 48h)
+--   s:{domain}:{epoch_min}:u        page (non-static) requests  (TTL 24h)
 --   c:{domain}:{epoch_min}:{CC}     country per minute          (TTL 24h)
 --   t:{domain}:r / :i / :o          lifetime totals             (no TTL)
 --   d:{domain}                      domain index marker         (no TTL)
 --
+--   Daily rollup, all under one `g:{domain}:{YYYYMMDD}:` prefix so mgmt_api's
+--   /analytics/geo serves the whole day in a SINGLE dict scan:
+--   g:{domain}:{YYYYMMDD}:{CC}      country hit count           (TTL 48h)
+--   g:{domain}:{YYYYMMDD}:v         distinct visitors           (TTL 48h)
+--   g:{domain}:{YYYYMMDD}:p:{path}  path hit count              (TTL 48h)
+--   g:{domain}:{YYYYMMDD}:s:{code}  status-code count           (TTL 48h)
+--
+-- Shared dict key schema (visitors zone):
+--   vsalt:{YYYYMMDD}                per-day hash salt           (TTL 48h)
+--   vd:{domain}:{YYYYMMDD}:{hash}   distinct-visitor marker     (TTL 48h)
+--
 -- Shared dict key schema (request_data zone):
 --   rlog:{domain}:seq               monotonic write pointer
 --   rlog:{domain}:{slot}            JSON entry (ring buf)       (TTL 1h)
---   log_pipe:sub:{domain}           live subscriber flag        (TTL 30s)
---   log_pipe:q:{domain}             live log queue entries
+--
+-- The ring is ALSO the live source: pipe_stream.lua's SSE endpoint reads these
+-- slots by a per-connection cursor, so there is no separate live queue or
+-- subscriber flag — every watcher sees the full stream and steals from none.
+--
+-- NOTE on `:u` — it counts non-static REQUESTS, not people. It was surfaced as
+-- "unique IPs" all the way to the dashboard, which it never was. Real distinct
+-- visitors are the `vd:`/`:v` pair below.
 
 local cjson = require "cjson.safe"
 
@@ -29,12 +45,17 @@ if not geo_ok then
     ngx.log(ngx.WARN, "[site_logger] geo_country module not available. GeoIP disabled.")
 end
 
--- SAFE LOAD: Pipe module (per-worker, cached by require)
-local pipe_ok, pipe_log = pcall(require, "openship.pipe_log")
-if not pipe_ok then pipe_log = nil end
-
 local analytics    = ngx.shared.analytics
 local request_data = ngx.shared.request_data
+-- Per-host edge config, pushed by the API (see mgmt_api's /analytics/config).
+--
+-- The `rules` zone, NOT `analytics`: this holds a handful of small config values and is
+-- never under eviction pressure, whereas the analytics zone is a 256 MB churn of counters.
+-- An evicted flag there would silently stop path collection with nothing to explain why.
+local rules_dict   = ngx.shared.rules
+-- Optional: a box whose nginx.conf predates this zone keeps working, just without
+-- distinct-visitor counts. Never assume a dict exists.
+local visitors     = ngx.shared.visitors
 if not analytics or not request_data then return end
 
 -- ── Helpers ──────────────────────────────────────────────────────────────────
@@ -89,11 +110,71 @@ local function is_static_asset(u)
     return false
 end
 
+-- Collapse a request URI into a low-cardinality bucket suitable as a dict key.
+-- Without this, "top paths" is unbounded: every ?token=, every /orders/48219, and
+-- every 404 a scanner probes mints a permanent key, and the analytics zone
+-- LRU-evicts real counters to store junk.
+local function normalize_path(u)
+    -- Query string carries ids/tokens and never identifies the route.
+    local path = u:match("^([^?#]*)") or u
+    if path == "" then return "/" end
+    -- Numeric and UUID/hex segments are ids, not routes: /orders/48219 and
+    -- /orders/48220 are one path.
+    path = path:gsub("/%d+", "/:id")
+    path = path:gsub("/%x%x%x%x%x%x%x%x%-?%x*%-?%x*%-?%x*%-?%x*", "/:id")
+    -- Bound the key itself — a long URI must not become a long key.
+    if #path > 120 then path = path:sub(1, 120) .. "…" end
+    return path
+end
+
+-- ── Enumerable sets, without a zone scan ─────────────────────────────────────
+--
+-- Reading these counters back used to need key DISCOVERY, and a shared dict offers exactly
+-- one mechanism for that: `get_keys`, which walks EVERY key in the zone holding its lock.
+-- Cost scaled with how much was in the 256 MB zone, not with what was being asked for, and
+-- it ran on the mgmt path four different ways.
+--
+-- So every unbounded set now carries its own index: a counter plus numbered slots.
+--
+--     <counter>        how many distinct members
+--     <prefix><n>      the n-th member
+--
+-- A reader reads the counter and then reads slots 1..n. Bounded by the answer.
+--
+-- The write only happens on a member's FIRST sighting, so steady state costs nothing: the
+-- millionth request from Germany today takes the plain `incr` path and touches no index.
+local function index_add(counter_key, slot_prefix, member, ttl)
+    local n = analytics:incr(counter_key, 1, 0, ttl)
+    if n then analytics:set(slot_prefix .. n, member, ttl) end
+end
+
+--- Add 1 to a counter, creating it (with TTL) and indexing it on first sighting.
+--
+-- `incr` with no `init` returns nil for an absent key, which doubles as the existence
+-- probe — so the common case is ONE hash operation. TTL is set by the creating `safe_add`
+-- rather than passed to `incr`: shdict rejects `init_ttl` without `init`, and that raises a
+-- Lua error which aborts the whole log handler (it has silently zeroed every counter below
+-- it before).
+local function bump_indexed(key, ttl, counter_key, slot_prefix, member)
+    if analytics:incr(key, 1) then return end
+    if analytics:safe_add(key, 1, ttl) then
+        index_add(counter_key, slot_prefix, member, ttl)
+    else
+        -- Another worker created it between our probe and our add; our +1 still counts.
+        analytics:incr(key, 1)
+    end
+end
+
 local RING   = 1000
 local D24H   = 86400
 local D48H   = 172800
 local D1H    = 3600
 
+-- Distinct paths tracked per domain per day. Past this the tail folds into
+-- "other" rather than growing the zone — a URL-fuzzing bot is capped at one
+-- extra key, not thousands.
+local PATH_CARDINALITY_CAP = 2000
+
 -- ── Capture ──────────────────────────────────────────────────────────────────
 
 local host = normalize(ngx.var.host)
@@ -109,6 +190,23 @@ local rt      = tonumber(ngx.var.request_time)   or 0  -- seconds (float)
 local method  = ngx.var.request_method or "GET"
 local status  = ngx.var.status         or "0"
 
+-- Client country, resolved ONCE per request.
+--
+-- Hoisted out of the geo block because both the daily/per-minute rollups and the raw
+-- ring buffer need it. The ring is the single source for the request log — /logs/recent
+-- reads it directly and the live SSE stream (pipe_stream.lua) reads the same slots by
+-- cursor — so resolving the country here means every row carries its flag no matter
+-- which path delivered it. An earlier split (looked up in the rollup, again in a
+-- separate live pipe) left the ring with no country at all.
+--
+-- pcall'd because a corrupt or partially-written mmdb makes the lookup raise, and an
+-- error here would abort every counter below it (see the section header).
+local cc = nil
+if geo and geo.get_country_code then
+    local ok_cc, res = pcall(geo.get_country_code, ip)
+    if ok_cc and type(res) == "string" and res ~= "" then cc = res end
+end
+
 -- ── 1. Minute-bucket counters ────────────────────────────────────────────────
 
 local minute = math.floor(ts / 60)
@@ -136,26 +234,145 @@ analytics:incr("t:" .. host .. ":r", 1,       0)
 analytics:incr("t:" .. host .. ":i", req_len, 0)
 analytics:incr("t:" .. host .. ":o", bytes,   0)
 
--- Domain index (set once, ignore subsequent "exists" errors)
-analytics:safe_add("d:" .. host, 1)
+-- Domain index. `safe_add` returning true means this is the first request this box has
+-- ever seen for the host, which is exactly when the enumerable index needs appending —
+-- so domain discovery no longer costs a zone scan either.
+if analytics:safe_add("d:" .. host, 1) then
+    index_add("dn", "di:", host)
+end
 
--- ── 3. GeoIP ─────────────────────────────────────────────────────────────────
+-- ── 3. Daily rollup: geo, visitors, paths, statuses ──────────────────────────
+--
+-- Each sub-block is pcall'd SEPARATELY, and that is load-bearing rather than
+-- defensive habit. A raised error in log_by_lua aborts the remainder of the
+-- handler, so one bad shdict call silently zeroes every counter written after it —
+-- which is exactly what an `incr(..., nil, ttl)` here did: paths AND statuses read
+-- as "no traffic" while requests and bandwidth looked perfect. Isolating them
+-- means a future edit can lose at most its own metric.
+
+local day  = today()
+local gpfx = "g:" .. host .. ":" .. day .. ":"
 
-if geo then
-    local cc = geo.get_country_code(ip)
-    if cc and cc ~= "" then
+-- 3a. GeoIP. `cc` was resolved once above and is reused by the ring buffer and the
+-- live pipe, so a request costs exactly one mmdb lookup no matter how many consumers
+-- report its country.
+if cc then
+    pcall(function()
         -- Daily geo (for /analytics/geo endpoint)
-        analytics:incr("g:" .. host .. ":" .. today() .. ":" .. cc, 1, 0, D48H)
-        -- Per-minute geo (for time-series country breakdown)
-        analytics:incr("c:" .. host .. ":" .. minute .. ":" .. cc, 1, 0, D24H)
+        bump_indexed(gpfx .. cc, D48H, gpfx .. "cn", gpfx .. "ci:", cc)
+        -- Per-minute geo (for the time-series country breakdown). Indexed per MINUTE, so
+        -- the scraper reads only the minutes in its window.
+        local mpfx = "c:" .. host .. ":" .. minute .. ":"
+        bump_indexed(mpfx .. cc, D24H, mpfx .. "n", mpfx .. "i:", cc)
+    end)
+end
+
+-- 3b. Distinct visitors — a COUNT, never an identity.
+--
+-- The address is hashed with a salt that is generated on this box, rotates daily,
+-- and is never written anywhere but this shared dict, so the marker keys are not
+-- reversible to an IP and are worthless the next day. Only the resulting counter
+-- is ever flushed to Postgres; no per-visitor row exists at any layer. That is
+-- what lets us report real visitor numbers while still not collecting behavioural
+-- analytics on anyone's end users.
+--
+-- Per DAY, not per minute: a per-minute set is ~1440x the keys for a number
+-- nobody reads, and would evict the counters it exists to annotate.
+pcall(function()
+if visitors then
+    local salt_key = "vsalt:" .. day
+    local salt = visitors:get(salt_key)
+    if not salt then
+        -- First writer wins; a racing worker's safe_add fails and it re-reads. The
+        -- salt only has to be unpredictable and stable for the day.
+        salt = tostring(ngx.now()) .. tostring(math.random(1, 2147483647))
+        local ok = visitors:safe_add(salt_key, salt, D48H)
+        if not ok then salt = visitors:get(salt_key) or salt end
+    end
+    -- safe_add is the dedup: it succeeds ONLY on this visitor's first request
+    -- today, so the counter below increments exactly once per distinct visitor.
+    local marker = "vd:" .. host .. ":" .. day .. ":" .. ngx.crc32_long(salt .. ip)
+    if visitors:safe_add(marker, 1, D48H) then
+        analytics:incr(gpfx .. "v", 1, 0, D48H)
+    end
+end
+end)
+
+-- 3c. Top paths — OPT-IN, non-static only, normalized and cardinality-capped.
+--
+-- Off unless the project turned it on, because this one block is the most expensive thing
+-- the log handler does. Measured on the shipped edge image, per request:
+--
+--     paths (whole block)        1.72 us      <- 57% of the counter path
+--       of which string work     1.38 us      normalize_path + is_static_asset
+--     minute buckets (5 incr)    0.61 us
+--     country (2 incr)           0.24 us
+--     status (1 incr)            0.14 us
+--     this flag read (1 get)     0.07 us
+--
+-- So the gate costs 0.07 us and saves 1.72 us — the counter path drops from ~3.0 us to
+-- ~1.35 us with paths off. It is also the highest-cardinality dimension (up to
+-- PATH_CARDINALITY_CAP keys per domain per day, against ~200 countries and a few dozen
+-- statuses) and the largest column in the daily rollup.
+--
+-- Absent flag = off. A box whose dict was just restarted therefore collects nothing until
+-- the API re-pushes, which is the safe direction: no data beats data nobody asked to pay
+-- for. The DB is the source of truth and re-pushes on every route apply.
+local collect_paths = rules_dict and rules_dict:get("cfg:" .. host .. ":paths")
+
+pcall(function()
+if collect_paths and not is_static_asset(uri) then
+    -- Hoisted: normalize_path is the single most expensive call in this handler (~1.4 us
+    -- of string work), so it runs exactly once even though both the counter key and the
+    -- index slot need its result.
+    local norm = normalize_path(uri)
+    local path_key = gpfx .. "p:" .. norm
+    -- incr-then-add: the hot path is ONE hash hit on an existing key. `incr` with
+    -- no init returns nil for an absent key, which doubles as the existence probe,
+    -- so only a path unseen today reaches the cap check below.
+    --
+    -- Deliberately no init_ttl here: shdict rejects init_ttl unless init is also
+    -- given ("'init_ttl' must be used with 'init'"), and that raises a Lua error
+    -- which aborts the whole log handler — taking the status counters below down
+    -- with it. The TTL is set by the safe_add on the creating request instead.
+    if not analytics:incr(path_key, 1) then
+        local distinct = analytics:incr(gpfx .. "pn", 1, 0, D48H) or 0
+        if distinct > PATH_CARDINALITY_CAP then
+            analytics:incr(gpfx .. "p:other", 1, 0, D48H)
+        else
+            -- `pn` was already the distinct-path counter, so the slots just give it an
+            -- enumerable side. Stored WITHOUT the "p:" prefix; the reader adds it back.
+            if analytics:safe_add(path_key, 1, D48H) then
+                analytics:set(gpfx .. "pi:" .. distinct, norm, D48H)
+            end
+        end
     end
 end
+end)
 
--- ── 4. Raw request ring buffer ──────────────────────────────────────────────
+-- 3d. Status-code mix. Bounded by construction (a few dozen codes).
+pcall(function()
+    bump_indexed(gpfx .. "s:" .. status, D48H, gpfx .. "sn", gpfx .. "si:", status)
+end)
+
+-- ── 4. Raw request ring buffer (also the live SSE source) ────────────────────
+--
+-- `seq` is claimed BEFORE encoding so the stored JSON can carry a deterministic
+-- id (`{host}:{seq}`). That id is the dedup key everywhere downstream: /logs/recent
+-- returns these entries verbatim, and pipe_stream.lua streams the same slots live,
+-- so the same request reaches the client with one stable id instead of a fresh
+-- random one per source. `incr` with init 0 never returns nil here; guard anyway
+-- so a shdict hiccup can't index a nil into the slot key.
 
 pcall(function()
+    local seq = request_data:incr("rlog:" .. host .. ":seq", 1, 0)
+    if not seq then return end
     local ok_j, j = pcall(cjson.encode, {
+        id     = host .. ":" .. seq,
         ip     = ip,
+        -- nil is simply omitted by cjson, so a box with no GeoIP writes the same shape
+        -- minus this key rather than a null the reader has to special-case.
+        country = cc,
         ts     = ts,
         method = method,
         status = status,
@@ -166,17 +383,6 @@ pcall(function()
         rt     = rt,
     })
     if ok_j and j then
-        local seq  = request_data:incr("rlog:" .. host .. ":seq", 1, 0)
-        local slot = seq % RING
-        request_data:set("rlog:" .. host .. ":" .. slot, j, D1H)
+        request_data:set("rlog:" .. host .. ":" .. (seq % RING), j, D1H)
     end
 end)
-
--- ── 5. Live-log pipe (only when a subscriber is watching) ───────────────────
--- Fire via timer to avoid blocking the log phase
-
-if pipe_log and pipe_log.pipe_request_log
-   and request_data:get("log_pipe:sub:" .. host) then
-    ngx.timer.at(0, pipe_log.pipe_request_log,
-                 host, ip, ts, ua, uri, req_len, bytes, rt, method, status)
-end

package/dist/server/migrations/0082_edge_target_verification.sql

@@ -0,0 +1,39 @@
+-- Proof that a routing TARGET is ours, as Openship Cloud's shared edge now requires
+-- before forwarding `<slug>.opsh.io` to it.
+--
+-- Keyed on (organization_id, target) because that is what the upstream keys it on:
+-- one target serves every free domain on that box, so this is target-level state,
+-- not domain-level. One install holds several rows — its own address plus one per
+-- remote server it deploys to.
+--
+-- The token is persisted HERE, not only on the edge, because the upstream's list
+-- endpoint returns id/target/status/expiry but NOT the token. Verification lasts 90
+-- days and the upstream re-probes THE SAME token within 7 days of expiry; a 404 then
+-- expires it and the free domain stops resolving ~83 days after a green deploy with
+-- nothing in our logs. So a token that lived only on a box's disk would be
+-- unrecoverable after a rebuild, and rows here are never deleted when a free domain
+-- is dropped.
+--
+-- `server_id` is ON DELETE SET NULL, not cascade: losing the server row must not
+-- take the token with it.
+CREATE TABLE IF NOT EXISTS "edge_target_verification" (
+  "id" text PRIMARY KEY NOT NULL,
+  "organization_id" text NOT NULL REFERENCES "organization"("id") ON DELETE CASCADE,
+  "target" text NOT NULL,
+  "host" text NOT NULL,
+  "server_id" text REFERENCES "servers"("id") ON DELETE SET NULL,
+  "verification_id" integer,
+  "token" text,
+  "retired_tokens" jsonb,
+  "challenge_path" text,
+  "status" text NOT NULL DEFAULT 'pending',
+  "validated_ip" text,
+  "expires_at" timestamp,
+  "last_checked_at" timestamp,
+  "last_error" text,
+  "created_at" timestamp NOT NULL DEFAULT now(),
+  "updated_at" timestamp NOT NULL DEFAULT now()
+);
+--> statement-breakpoint
+CREATE UNIQUE INDEX IF NOT EXISTS "uq_edge_target_verification"
+  ON "edge_target_verification" ("organization_id", "target");

package/dist/server/migrations/0083_service_incident.sql

@@ -0,0 +1,84 @@
+-- Continuous container health monitoring: the durable memory behind the alerts.
+--
+-- Openship knew a container had crashed at exactly ONE moment — the ~15s post-deploy
+-- stabilization watch. After that nobody looked again, so an OOM-kill at 3am or a
+-- Postgres that starts bouncing after a host reboot reached the operator only when a
+-- user complained. The health watch closes that, but a poller that alerts on "the
+-- container isn't running" gets muted within a day: a redeploy recreating containers,
+-- an operator's `docker stop`, an unreachable host, and a crash loop all look exactly
+-- like a failure at a point in time.
+--
+-- So the unit of alerting is an INCIDENT, not a state reading: opened once, escalated
+-- only when it gets worse, resolved once with a downtime duration. This table is that
+-- memory — it survives the control-plane restart that in-process state would not (a box
+-- down for three days must not re-page on every API restart), and it doubles as the
+-- project Health tab's history.
+CREATE TABLE IF NOT EXISTS "service_incident" (
+  "id" text PRIMARY KEY NOT NULL,
+  "organization_id" text NOT NULL REFERENCES "organization"("id") ON DELETE CASCADE,
+  -- Null for SERVER-scoped incidents (an unreachable box gets ONE incident, not one
+  -- per service on it — that fan-out is the single loudest source of alert fatigue).
+  "project_id" text REFERENCES "project"("id") ON DELETE CASCADE,
+  -- The resolved service row, when there is one. SET NULL, not cascade: deleting a
+  -- service must not erase the record of the outage it had (`service_key` still
+  -- identifies it, and `service_name` is snapshotted below).
+  "service_id" text REFERENCES "service"("id") ON DELETE SET NULL,
+  -- Stable identity across a container RECREATE, which is the whole point: every
+  -- deploy mints a new container id, so keying incidents on container id would open a
+  -- fresh incident for the same workload after each redeploy. Service id when we
+  -- resolved one, else the container name (single-app deploys have no service row);
+  -- `server:<id>` for server-scoped rows.
+  "service_key" text NOT NULL,
+  -- Snapshot of the display name at incident time — the alert text must still read
+  -- correctly after a rename, and history rows must not silently change meaning.
+  "service_name" text NOT NULL,
+  "server_id" text REFERENCES "servers"("id") ON DELETE SET NULL,
+  "container_id" text,
+  -- down | crash_loop | unhealthy | server_unreachable. Ordered by severity in the
+  -- watcher: an incident escalates upward (and re-notifies), never downward.
+  "kind" text NOT NULL,
+  "status" text NOT NULL DEFAULT 'open', -- open | resolved
+  "reason" text,
+  "exit_code" integer,
+  "restart_count" integer NOT NULL DEFAULT 0,
+  "oom_killed" boolean NOT NULL DEFAULT false,
+  -- Consecutive ticks that agreed on this verdict. An incident is only opened (and
+  -- only notified) at >= 2, so a snapshot taken mid-restart or during a slow
+  -- healthcheck start never pages anyone.
+  "confirmations" integer NOT NULL DEFAULT 0,
+  -- Notification bookkeeping. Deliberately a count, not a flag: open, escalation and
+  -- resolve each notify exactly once, and nothing else ever does.
+  "notify_count" integer NOT NULL DEFAULT 0,
+  "notified_at" timestamp,
+  "log_excerpt" text,
+  "opened_at" timestamp NOT NULL DEFAULT now(),
+  "resolved_at" timestamp,
+  -- Last tick that actually OBSERVED this workload. Not advanced when the host is
+  -- unreachable — during a connectivity loss we know nothing, so nothing is claimed.
+  "last_seen_at" timestamp NOT NULL DEFAULT now(),
+  "created_at" timestamp NOT NULL DEFAULT now(),
+  "updated_at" timestamp NOT NULL DEFAULT now()
+);
+--> statement-breakpoint
+-- One open incident per workload — the dedup primitive. As a partial unique index,
+-- "we are already alerting about this" is a DB invariant rather than sweep
+-- bookkeeping: two overlapping ticks (a manual Run now landing on a cron tick) cannot
+-- both open one and double-notify.
+CREATE UNIQUE INDEX IF NOT EXISTS "uq_service_incident_open"
+  ON "service_incident" ("project_id", "service_key")
+  WHERE "status" = 'open' AND "project_id" IS NOT NULL;
+--> statement-breakpoint
+-- Same invariant for server-scoped rows. Needs its own index because Postgres treats
+-- NULLs as distinct in a unique index, so the one above would happily allow a second
+-- open row for the same box.
+CREATE UNIQUE INDEX IF NOT EXISTS "uq_service_incident_open_server"
+  ON "service_incident" ("server_id")
+  WHERE "status" = 'open' AND "project_id" IS NULL;
+--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "idx_service_incident_project"
+  ON "service_incident" ("project_id", "opened_at" DESC);
+--> statement-breakpoint
+-- Intent marker. `disableProject` stopped the container and recorded NOTHING, so
+-- "the operator turned this off" was unknowable — the health watch would page about
+-- every deliberately-disabled project, forever. Set on disable, cleared on enable.
+ALTER TABLE "project" ADD COLUMN IF NOT EXISTS "disabled_at" timestamp;

package/dist/server/migrations/0084_analytics_daily_rollup.sql

@@ -0,0 +1,72 @@
+-- Widen the daily analytics rollup: distinct visitors, top paths, status mix.
+--
+-- The visitor pipeline already persisted per-country counts, and the dashboard
+-- already rendered cards for "unique IPs" and a top-paths table. Neither was real:
+-- `top_paths` was hardcoded `[]` in the service layer, and "unique IPs" was
+-- `unique_requests` — the count of non-static REQUESTS — so five page views from
+-- one browser read as five visitors. These columns are what makes those two
+-- numbers mean what the UI has been claiming.
+--
+-- All three land HERE, on the daily table, rather than on the per-minute one:
+-- visitors and paths are the high-cardinality metrics, and holding them per-minute
+-- would multiply the edge's shared-dict key count by ~1440 to feed a chart nobody
+-- asked for — evicting, in the process, the request counters they annotate.
+-- Countries already live here for the same reason, and the edge keeps all four
+-- under one `g:{domain}:{day}:` key prefix so a reader gets the whole day in a
+-- single dict scan.
+--
+-- Additive with defaults on purpose: `migrations-additive.test.ts` rejects a
+-- NOT NULL column without a DEFAULT, because an older cross-version dump omits the
+-- column entirely and Drizzle then emits DEFAULT for it — with no default that's a
+-- NULL into NOT NULL on the newer receiver, i.e. a broken cloud/project transfer.
+
+-- Distinct visitors for the day, deduplicated AT THE EDGE and stored as a count.
+--
+-- Privacy is a property of where the dedup happens: the edge hashes each address
+-- with a salt that is generated on that box, rotates daily, and never leaves its
+-- shared memory. Only the cardinality of that set is written here. No address and
+-- no per-visitor row exists at any layer, which is what keeps the published
+-- "we don't collect behavioural analytics on your end users" true while still
+-- reporting a real visitor number.
+--
+-- Understates on a very busy domain: the edge's `visitors` zone LRU-evicts past
+-- roughly 1M distinct/day. mgmt_api `GET /status` reports that zone's free space so
+-- a reader can label the number approximate instead of presenting an eviction
+-- artifact as a measurement.
+ALTER TABLE "server_analytics_geo" ADD COLUMN IF NOT EXISTS "visitors" integer DEFAULT 0 NOT NULL;
+--> statement-breakpoint
+
+-- Top paths: { "/": 900, "/orders/:id": 120, "other": 40 }.
+--
+-- Normalized at the edge before it ever becomes a key: query string stripped (so a
+-- ?token= or ?session= can never be persisted here), numeric and UUID segments
+-- collapsed to `:id`, key length capped, and the tail past a cardinality cap folded
+-- into "other" — otherwise a scanner walking /wp-admin variants mints thousands of
+-- permanent keys and evicts the real counters.
+ALTER TABLE "server_analytics_geo" ADD COLUMN IF NOT EXISTS "paths" jsonb;
+--> statement-breakpoint
+
+-- Status-code mix: { "200": 4210, "404": 17, "502": 3 }. Status was captured into
+-- the edge's raw-request ring buffer (RAM, 1h) but never aggregated, so error rate
+-- was unanswerable from any persisted data. Bounded by construction.
+ALTER TABLE "server_analytics_geo" ADD COLUMN IF NOT EXISTS "statuses" jsonb;
+--> statement-breakpoint
+
+-- Widen the per-minute bandwidth counters from int4 to int8.
+--
+-- These are bytes for ONE MINUTE, and int4 caps at 2,147,483,647 — about 2.1 GB,
+-- which a single minute reaches at roughly 286 Mbps. That is unremarkable for a
+-- site serving video or large downloads. Past it Postgres raises "integer out of
+-- range", the upsert for that minute fails, and the scrape for the whole domain
+-- dies with it — so the busiest domains on a box would be precisely the ones with
+-- no analytics, and it would look like the feature simply didn't work for them.
+--
+-- Now that collection runs unattended on a schedule rather than only when someone
+-- opens the tab, that failure would happen in a job with nobody reading the error.
+--
+-- Widening is lossless and there is no application change: the edge already
+-- accumulates these as Lua numbers (doubles, exact to 2^53) and Drizzle reads them
+-- back with mode:"number", which is exact over the same range.
+ALTER TABLE "server_analytics" ALTER COLUMN "bandwidth_in" TYPE bigint;
+--> statement-breakpoint
+ALTER TABLE "server_analytics" ALTER COLUMN "bandwidth_out" TYPE bigint;