openship 0.5.0 → 0.6.0

package/dist/server/lua/site_logger.lua

@@ -8,17 +8,31 @@
 --   s:{domain}:{epoch_min}:i        bandwidth in  bytes         (TTL 24h)
 --   s:{domain}:{epoch_min}:o        bandwidth out bytes         (TTL 24h)
 --   s:{domain}:{epoch_min}:t        response time sum (seconds) (TTL 24h)
---   s:{domain}:{epoch_min}:u        unique (non-static) reqs    (TTL 24h)
---   g:{domain}:{YYYYMMDD}:{CC}      country hit count           (TTL 48h)
+--   s:{domain}:{epoch_min}:u        page (non-static) requests  (TTL 24h)
 --   c:{domain}:{epoch_min}:{CC}     country per minute          (TTL 24h)
 --   t:{domain}:r / :i / :o          lifetime totals             (no TTL)
 --   d:{domain}                      domain index marker         (no TTL)
 --
+--   Daily rollup, all under one `g:{domain}:{YYYYMMDD}:` prefix so mgmt_api's
+--   /analytics/geo serves the whole day in a SINGLE dict scan:
+--   g:{domain}:{YYYYMMDD}:{CC}      country hit count           (TTL 48h)
+--   g:{domain}:{YYYYMMDD}:v         distinct visitors           (TTL 48h)
+--   g:{domain}:{YYYYMMDD}:p:{path}  path hit count              (TTL 48h)
+--   g:{domain}:{YYYYMMDD}:s:{code}  status-code count           (TTL 48h)
+--
+-- Shared dict key schema (visitors zone):
+--   vsalt:{YYYYMMDD}                per-day hash salt           (TTL 48h)
+--   vd:{domain}:{YYYYMMDD}:{hash}   distinct-visitor marker     (TTL 48h)
+--
 -- Shared dict key schema (request_data zone):
 --   rlog:{domain}:seq               monotonic write pointer
 --   rlog:{domain}:{slot}            JSON entry (ring buf)       (TTL 1h)
 --   log_pipe:sub:{domain}           live subscriber flag        (TTL 30s)
 --   log_pipe:q:{domain}             live log queue entries
+--
+-- NOTE on `:u` — it counts non-static REQUESTS, not people. It was surfaced as
+-- "unique IPs" all the way to the dashboard, which it never was. Real distinct
+-- visitors are the `vd:`/`:v` pair below.
 
 local cjson = require "cjson.safe"
 
@@ -35,6 +49,15 @@ if not pipe_ok then pipe_log = nil end
 
 local analytics    = ngx.shared.analytics
 local request_data = ngx.shared.request_data
+-- Per-host edge config, pushed by the API (see mgmt_api's /analytics/config).
+--
+-- The `rules` zone, NOT `analytics`: this holds a handful of small config values and is
+-- never under eviction pressure, whereas the analytics zone is a 256 MB churn of counters.
+-- An evicted flag there would silently stop path collection with nothing to explain why.
+local rules_dict   = ngx.shared.rules
+-- Optional: a box whose nginx.conf predates this zone keeps working, just without
+-- distinct-visitor counts. Never assume a dict exists.
+local visitors     = ngx.shared.visitors
 if not analytics or not request_data then return end
 
 -- ── Helpers ──────────────────────────────────────────────────────────────────
@@ -89,11 +112,71 @@ local function is_static_asset(u)
     return false
 end
 
+-- Collapse a request URI into a low-cardinality bucket suitable as a dict key.
+-- Without this, "top paths" is unbounded: every ?token=, every /orders/48219, and
+-- every 404 a scanner probes mints a permanent key, and the analytics zone
+-- LRU-evicts real counters to store junk.
+local function normalize_path(u)
+    -- Query string carries ids/tokens and never identifies the route.
+    local path = u:match("^([^?#]*)") or u
+    if path == "" then return "/" end
+    -- Numeric and UUID/hex segments are ids, not routes: /orders/48219 and
+    -- /orders/48220 are one path.
+    path = path:gsub("/%d+", "/:id")
+    path = path:gsub("/%x%x%x%x%x%x%x%x%-?%x*%-?%x*%-?%x*%-?%x*", "/:id")
+    -- Bound the key itself — a long URI must not become a long key.
+    if #path > 120 then path = path:sub(1, 120) .. "…" end
+    return path
+end
+
+-- ── Enumerable sets, without a zone scan ─────────────────────────────────────
+--
+-- Reading these counters back used to need key DISCOVERY, and a shared dict offers exactly
+-- one mechanism for that: `get_keys`, which walks EVERY key in the zone holding its lock.
+-- Cost scaled with how much was in the 256 MB zone, not with what was being asked for, and
+-- it ran on the mgmt path four different ways.
+--
+-- So every unbounded set now carries its own index: a counter plus numbered slots.
+--
+--     <counter>        how many distinct members
+--     <prefix><n>      the n-th member
+--
+-- A reader reads the counter and then reads slots 1..n. Bounded by the answer.
+--
+-- The write only happens on a member's FIRST sighting, so steady state costs nothing: the
+-- millionth request from Germany today takes the plain `incr` path and touches no index.
+local function index_add(counter_key, slot_prefix, member, ttl)
+    local n = analytics:incr(counter_key, 1, 0, ttl)
+    if n then analytics:set(slot_prefix .. n, member, ttl) end
+end
+
+--- Add 1 to a counter, creating it (with TTL) and indexing it on first sighting.
+--
+-- `incr` with no `init` returns nil for an absent key, which doubles as the existence
+-- probe — so the common case is ONE hash operation. TTL is set by the creating `safe_add`
+-- rather than passed to `incr`: shdict rejects `init_ttl` without `init`, and that raises a
+-- Lua error which aborts the whole log handler (it has silently zeroed every counter below
+-- it before).
+local function bump_indexed(key, ttl, counter_key, slot_prefix, member)
+    if analytics:incr(key, 1) then return end
+    if analytics:safe_add(key, 1, ttl) then
+        index_add(counter_key, slot_prefix, member, ttl)
+    else
+        -- Another worker created it between our probe and our add; our +1 still counts.
+        analytics:incr(key, 1)
+    end
+end
+
 local RING   = 1000
 local D24H   = 86400
 local D48H   = 172800
 local D1H    = 3600
 
+-- Distinct paths tracked per domain per day. Past this the tail folds into
+-- "other" rather than growing the zone — a URL-fuzzing bot is capped at one
+-- extra key, not thousands.
+local PATH_CARDINALITY_CAP = 2000
+
 -- ── Capture ──────────────────────────────────────────────────────────────────
 
 local host = normalize(ngx.var.host)
@@ -109,6 +192,22 @@ local rt      = tonumber(ngx.var.request_time)   or 0  -- seconds (float)
 local method  = ngx.var.request_method or "GET"
 local status  = ngx.var.status         or "0"
 
+-- Client country, resolved ONCE per request.
+--
+-- Hoisted out of the geo block because THREE consumers need it — the daily/per-minute
+-- rollups, the raw ring buffer behind /logs/recent, and the live SSE pipe. It used to be
+-- looked up inside the rollup and again inside pipe_log, so the ring buffer had no
+-- country at all: a request-log list showed flags on rows that arrived live and none on
+-- rows backfilled from /logs/recent, for the same traffic.
+--
+-- pcall'd because a corrupt or partially-written mmdb makes the lookup raise, and an
+-- error here would abort every counter below it (see the section header).
+local cc = nil
+if geo and geo.get_country_code then
+    local ok_cc, res = pcall(geo.get_country_code, ip)
+    if ok_cc and type(res) == "string" and res ~= "" then cc = res end
+end
+
 -- ── 1. Minute-bucket counters ────────────────────────────────────────────────
 
 local minute = math.floor(ts / 60)
@@ -136,26 +235,135 @@ analytics:incr("t:" .. host .. ":r", 1,       0)
 analytics:incr("t:" .. host .. ":i", req_len, 0)
 analytics:incr("t:" .. host .. ":o", bytes,   0)
 
--- Domain index (set once, ignore subsequent "exists" errors)
-analytics:safe_add("d:" .. host, 1)
+-- Domain index. `safe_add` returning true means this is the first request this box has
+-- ever seen for the host, which is exactly when the enumerable index needs appending —
+-- so domain discovery no longer costs a zone scan either.
+if analytics:safe_add("d:" .. host, 1) then
+    index_add("dn", "di:", host)
+end
+
+-- ── 3. Daily rollup: geo, visitors, paths, statuses ──────────────────────────
+--
+-- Each sub-block is pcall'd SEPARATELY, and that is load-bearing rather than
+-- defensive habit. A raised error in log_by_lua aborts the remainder of the
+-- handler, so one bad shdict call silently zeroes every counter written after it —
+-- which is exactly what an `incr(..., nil, ttl)` here did: paths AND statuses read
+-- as "no traffic" while requests and bandwidth looked perfect. Isolating them
+-- means a future edit can lose at most its own metric.
 
--- ── 3. GeoIP ─────────────────────────────────────────────────────────────────
+local day  = today()
+local gpfx = "g:" .. host .. ":" .. day .. ":"
 
-if geo then
-    local cc = geo.get_country_code(ip)
-    if cc and cc ~= "" then
+-- 3a. GeoIP. `cc` was resolved once above and is reused by the ring buffer and the
+-- live pipe, so a request costs exactly one mmdb lookup no matter how many consumers
+-- report its country.
+if cc then
+    pcall(function()
         -- Daily geo (for /analytics/geo endpoint)
-        analytics:incr("g:" .. host .. ":" .. today() .. ":" .. cc, 1, 0, D48H)
-        -- Per-minute geo (for time-series country breakdown)
-        analytics:incr("c:" .. host .. ":" .. minute .. ":" .. cc, 1, 0, D24H)
+        bump_indexed(gpfx .. cc, D48H, gpfx .. "cn", gpfx .. "ci:", cc)
+        -- Per-minute geo (for the time-series country breakdown). Indexed per MINUTE, so
+        -- the scraper reads only the minutes in its window.
+        local mpfx = "c:" .. host .. ":" .. minute .. ":"
+        bump_indexed(mpfx .. cc, D24H, mpfx .. "n", mpfx .. "i:", cc)
+    end)
+end
+
+-- 3b. Distinct visitors — a COUNT, never an identity.
+--
+-- The address is hashed with a salt that is generated on this box, rotates daily,
+-- and is never written anywhere but this shared dict, so the marker keys are not
+-- reversible to an IP and are worthless the next day. Only the resulting counter
+-- is ever flushed to Postgres; no per-visitor row exists at any layer. That is
+-- what lets us report real visitor numbers while still not collecting behavioural
+-- analytics on anyone's end users.
+--
+-- Per DAY, not per minute: a per-minute set is ~1440x the keys for a number
+-- nobody reads, and would evict the counters it exists to annotate.
+pcall(function()
+if visitors then
+    local salt_key = "vsalt:" .. day
+    local salt = visitors:get(salt_key)
+    if not salt then
+        -- First writer wins; a racing worker's safe_add fails and it re-reads. The
+        -- salt only has to be unpredictable and stable for the day.
+        salt = tostring(ngx.now()) .. tostring(math.random(1, 2147483647))
+        local ok = visitors:safe_add(salt_key, salt, D48H)
+        if not ok then salt = visitors:get(salt_key) or salt end
+    end
+    -- safe_add is the dedup: it succeeds ONLY on this visitor's first request
+    -- today, so the counter below increments exactly once per distinct visitor.
+    local marker = "vd:" .. host .. ":" .. day .. ":" .. ngx.crc32_long(salt .. ip)
+    if visitors:safe_add(marker, 1, D48H) then
+        analytics:incr(gpfx .. "v", 1, 0, D48H)
     end
 end
+end)
+
+-- 3c. Top paths — OPT-IN, non-static only, normalized and cardinality-capped.
+--
+-- Off unless the project turned it on, because this one block is the most expensive thing
+-- the log handler does. Measured on the shipped edge image, per request:
+--
+--     paths (whole block)        1.72 us      <- 57% of the counter path
+--       of which string work     1.38 us      normalize_path + is_static_asset
+--     minute buckets (5 incr)    0.61 us
+--     country (2 incr)           0.24 us
+--     status (1 incr)            0.14 us
+--     this flag read (1 get)     0.07 us
+--
+-- So the gate costs 0.07 us and saves 1.72 us — the counter path drops from ~3.0 us to
+-- ~1.35 us with paths off. It is also the highest-cardinality dimension (up to
+-- PATH_CARDINALITY_CAP keys per domain per day, against ~200 countries and a few dozen
+-- statuses) and the largest column in the daily rollup.
+--
+-- Absent flag = off. A box whose dict was just restarted therefore collects nothing until
+-- the API re-pushes, which is the safe direction: no data beats data nobody asked to pay
+-- for. The DB is the source of truth and re-pushes on every route apply.
+local collect_paths = rules_dict and rules_dict:get("cfg:" .. host .. ":paths")
+
+pcall(function()
+if collect_paths and not is_static_asset(uri) then
+    -- Hoisted: normalize_path is the single most expensive call in this handler (~1.4 us
+    -- of string work), so it runs exactly once even though both the counter key and the
+    -- index slot need its result.
+    local norm = normalize_path(uri)
+    local path_key = gpfx .. "p:" .. norm
+    -- incr-then-add: the hot path is ONE hash hit on an existing key. `incr` with
+    -- no init returns nil for an absent key, which doubles as the existence probe,
+    -- so only a path unseen today reaches the cap check below.
+    --
+    -- Deliberately no init_ttl here: shdict rejects init_ttl unless init is also
+    -- given ("'init_ttl' must be used with 'init'"), and that raises a Lua error
+    -- which aborts the whole log handler — taking the status counters below down
+    -- with it. The TTL is set by the safe_add on the creating request instead.
+    if not analytics:incr(path_key, 1) then
+        local distinct = analytics:incr(gpfx .. "pn", 1, 0, D48H) or 0
+        if distinct > PATH_CARDINALITY_CAP then
+            analytics:incr(gpfx .. "p:other", 1, 0, D48H)
+        else
+            -- `pn` was already the distinct-path counter, so the slots just give it an
+            -- enumerable side. Stored WITHOUT the "p:" prefix; the reader adds it back.
+            if analytics:safe_add(path_key, 1, D48H) then
+                analytics:set(gpfx .. "pi:" .. distinct, norm, D48H)
+            end
+        end
+    end
+end
+end)
+
+-- 3d. Status-code mix. Bounded by construction (a few dozen codes).
+pcall(function()
+    bump_indexed(gpfx .. "s:" .. status, D48H, gpfx .. "sn", gpfx .. "si:", status)
+end)
 
 -- ── 4. Raw request ring buffer ──────────────────────────────────────────────
 
 pcall(function()
     local ok_j, j = pcall(cjson.encode, {
         ip     = ip,
+        -- nil is simply omitted by cjson, so a box with no GeoIP writes the same shape
+        -- minus this key rather than a null the reader has to special-case.
+        country = cc,
         ts     = ts,
         method = method,
         status = status,
@@ -178,5 +386,5 @@ end)
 if pipe_log and pipe_log.pipe_request_log
    and request_data:get("log_pipe:sub:" .. host) then
     ngx.timer.at(0, pipe_log.pipe_request_log,
-                 host, ip, ts, ua, uri, req_len, bytes, rt, method, status)
+                 host, ip, ts, ua, uri, req_len, bytes, rt, method, status, cc)
 end

package/dist/server/migrations/0082_edge_target_verification.sql

@@ -0,0 +1,39 @@
+-- Proof that a routing TARGET is ours, as Openship Cloud's shared edge now requires
+-- before forwarding `<slug>.opsh.io` to it.
+--
+-- Keyed on (organization_id, target) because that is what the upstream keys it on:
+-- one target serves every free domain on that box, so this is target-level state,
+-- not domain-level. One install holds several rows — its own address plus one per
+-- remote server it deploys to.
+--
+-- The token is persisted HERE, not only on the edge, because the upstream's list
+-- endpoint returns id/target/status/expiry but NOT the token. Verification lasts 90
+-- days and the upstream re-probes THE SAME token within 7 days of expiry; a 404 then
+-- expires it and the free domain stops resolving ~83 days after a green deploy with
+-- nothing in our logs. So a token that lived only on a box's disk would be
+-- unrecoverable after a rebuild, and rows here are never deleted when a free domain
+-- is dropped.
+--
+-- `server_id` is ON DELETE SET NULL, not cascade: losing the server row must not
+-- take the token with it.
+CREATE TABLE IF NOT EXISTS "edge_target_verification" (
+  "id" text PRIMARY KEY NOT NULL,
+  "organization_id" text NOT NULL REFERENCES "organization"("id") ON DELETE CASCADE,
+  "target" text NOT NULL,
+  "host" text NOT NULL,
+  "server_id" text REFERENCES "servers"("id") ON DELETE SET NULL,
+  "verification_id" integer,
+  "token" text,
+  "retired_tokens" jsonb,
+  "challenge_path" text,
+  "status" text NOT NULL DEFAULT 'pending',
+  "validated_ip" text,
+  "expires_at" timestamp,
+  "last_checked_at" timestamp,
+  "last_error" text,
+  "created_at" timestamp NOT NULL DEFAULT now(),
+  "updated_at" timestamp NOT NULL DEFAULT now()
+);
+--> statement-breakpoint
+CREATE UNIQUE INDEX IF NOT EXISTS "uq_edge_target_verification"
+  ON "edge_target_verification" ("organization_id", "target");

package/dist/server/migrations/0083_service_incident.sql

@@ -0,0 +1,84 @@
+-- Continuous container health monitoring: the durable memory behind the alerts.
+--
+-- Openship knew a container had crashed at exactly ONE moment — the ~15s post-deploy
+-- stabilization watch. After that nobody looked again, so an OOM-kill at 3am or a
+-- Postgres that starts bouncing after a host reboot reached the operator only when a
+-- user complained. The health watch closes that, but a poller that alerts on "the
+-- container isn't running" gets muted within a day: a redeploy recreating containers,
+-- an operator's `docker stop`, an unreachable host, and a crash loop all look exactly
+-- like a failure at a point in time.
+--
+-- So the unit of alerting is an INCIDENT, not a state reading: opened once, escalated
+-- only when it gets worse, resolved once with a downtime duration. This table is that
+-- memory — it survives the control-plane restart that in-process state would not (a box
+-- down for three days must not re-page on every API restart), and it doubles as the
+-- project Health tab's history.
+CREATE TABLE IF NOT EXISTS "service_incident" (
+  "id" text PRIMARY KEY NOT NULL,
+  "organization_id" text NOT NULL REFERENCES "organization"("id") ON DELETE CASCADE,
+  -- Null for SERVER-scoped incidents (an unreachable box gets ONE incident, not one
+  -- per service on it — that fan-out is the single loudest source of alert fatigue).
+  "project_id" text REFERENCES "project"("id") ON DELETE CASCADE,
+  -- The resolved service row, when there is one. SET NULL, not cascade: deleting a
+  -- service must not erase the record of the outage it had (`service_key` still
+  -- identifies it, and `service_name` is snapshotted below).
+  "service_id" text REFERENCES "service"("id") ON DELETE SET NULL,
+  -- Stable identity across a container RECREATE, which is the whole point: every
+  -- deploy mints a new container id, so keying incidents on container id would open a
+  -- fresh incident for the same workload after each redeploy. Service id when we
+  -- resolved one, else the container name (single-app deploys have no service row);
+  -- `server:<id>` for server-scoped rows.
+  "service_key" text NOT NULL,
+  -- Snapshot of the display name at incident time — the alert text must still read
+  -- correctly after a rename, and history rows must not silently change meaning.
+  "service_name" text NOT NULL,
+  "server_id" text REFERENCES "servers"("id") ON DELETE SET NULL,
+  "container_id" text,
+  -- down | crash_loop | unhealthy | server_unreachable. Ordered by severity in the
+  -- watcher: an incident escalates upward (and re-notifies), never downward.
+  "kind" text NOT NULL,
+  "status" text NOT NULL DEFAULT 'open', -- open | resolved
+  "reason" text,
+  "exit_code" integer,
+  "restart_count" integer NOT NULL DEFAULT 0,
+  "oom_killed" boolean NOT NULL DEFAULT false,
+  -- Consecutive ticks that agreed on this verdict. An incident is only opened (and
+  -- only notified) at >= 2, so a snapshot taken mid-restart or during a slow
+  -- healthcheck start never pages anyone.
+  "confirmations" integer NOT NULL DEFAULT 0,
+  -- Notification bookkeeping. Deliberately a count, not a flag: open, escalation and
+  -- resolve each notify exactly once, and nothing else ever does.
+  "notify_count" integer NOT NULL DEFAULT 0,
+  "notified_at" timestamp,
+  "log_excerpt" text,
+  "opened_at" timestamp NOT NULL DEFAULT now(),
+  "resolved_at" timestamp,
+  -- Last tick that actually OBSERVED this workload. Not advanced when the host is
+  -- unreachable — during a connectivity loss we know nothing, so nothing is claimed.
+  "last_seen_at" timestamp NOT NULL DEFAULT now(),
+  "created_at" timestamp NOT NULL DEFAULT now(),
+  "updated_at" timestamp NOT NULL DEFAULT now()
+);
+--> statement-breakpoint
+-- One open incident per workload — the dedup primitive. As a partial unique index,
+-- "we are already alerting about this" is a DB invariant rather than sweep
+-- bookkeeping: two overlapping ticks (a manual Run now landing on a cron tick) cannot
+-- both open one and double-notify.
+CREATE UNIQUE INDEX IF NOT EXISTS "uq_service_incident_open"
+  ON "service_incident" ("project_id", "service_key")
+  WHERE "status" = 'open' AND "project_id" IS NOT NULL;
+--> statement-breakpoint
+-- Same invariant for server-scoped rows. Needs its own index because Postgres treats
+-- NULLs as distinct in a unique index, so the one above would happily allow a second
+-- open row for the same box.
+CREATE UNIQUE INDEX IF NOT EXISTS "uq_service_incident_open_server"
+  ON "service_incident" ("server_id")
+  WHERE "status" = 'open' AND "project_id" IS NULL;
+--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "idx_service_incident_project"
+  ON "service_incident" ("project_id", "opened_at" DESC);
+--> statement-breakpoint
+-- Intent marker. `disableProject` stopped the container and recorded NOTHING, so
+-- "the operator turned this off" was unknowable — the health watch would page about
+-- every deliberately-disabled project, forever. Set on disable, cleared on enable.
+ALTER TABLE "project" ADD COLUMN IF NOT EXISTS "disabled_at" timestamp;

package/dist/server/migrations/0084_analytics_daily_rollup.sql

@@ -0,0 +1,72 @@
+-- Widen the daily analytics rollup: distinct visitors, top paths, status mix.
+--
+-- The visitor pipeline already persisted per-country counts, and the dashboard
+-- already rendered cards for "unique IPs" and a top-paths table. Neither was real:
+-- `top_paths` was hardcoded `[]` in the service layer, and "unique IPs" was
+-- `unique_requests` — the count of non-static REQUESTS — so five page views from
+-- one browser read as five visitors. These columns are what makes those two
+-- numbers mean what the UI has been claiming.
+--
+-- All three land HERE, on the daily table, rather than on the per-minute one:
+-- visitors and paths are the high-cardinality metrics, and holding them per-minute
+-- would multiply the edge's shared-dict key count by ~1440 to feed a chart nobody
+-- asked for — evicting, in the process, the request counters they annotate.
+-- Countries already live here for the same reason, and the edge keeps all four
+-- under one `g:{domain}:{day}:` key prefix so a reader gets the whole day in a
+-- single dict scan.
+--
+-- Additive with defaults on purpose: `migrations-additive.test.ts` rejects a
+-- NOT NULL column without a DEFAULT, because an older cross-version dump omits the
+-- column entirely and Drizzle then emits DEFAULT for it — with no default that's a
+-- NULL into NOT NULL on the newer receiver, i.e. a broken cloud/project transfer.
+
+-- Distinct visitors for the day, deduplicated AT THE EDGE and stored as a count.
+--
+-- Privacy is a property of where the dedup happens: the edge hashes each address
+-- with a salt that is generated on that box, rotates daily, and never leaves its
+-- shared memory. Only the cardinality of that set is written here. No address and
+-- no per-visitor row exists at any layer, which is what keeps the published
+-- "we don't collect behavioural analytics on your end users" true while still
+-- reporting a real visitor number.
+--
+-- Understates on a very busy domain: the edge's `visitors` zone LRU-evicts past
+-- roughly 1M distinct/day. mgmt_api `GET /status` reports that zone's free space so
+-- a reader can label the number approximate instead of presenting an eviction
+-- artifact as a measurement.
+ALTER TABLE "server_analytics_geo" ADD COLUMN IF NOT EXISTS "visitors" integer DEFAULT 0 NOT NULL;
+--> statement-breakpoint
+
+-- Top paths: { "/": 900, "/orders/:id": 120, "other": 40 }.
+--
+-- Normalized at the edge before it ever becomes a key: query string stripped (so a
+-- ?token= or ?session= can never be persisted here), numeric and UUID segments
+-- collapsed to `:id`, key length capped, and the tail past a cardinality cap folded
+-- into "other" — otherwise a scanner walking /wp-admin variants mints thousands of
+-- permanent keys and evicts the real counters.
+ALTER TABLE "server_analytics_geo" ADD COLUMN IF NOT EXISTS "paths" jsonb;
+--> statement-breakpoint
+
+-- Status-code mix: { "200": 4210, "404": 17, "502": 3 }. Status was captured into
+-- the edge's raw-request ring buffer (RAM, 1h) but never aggregated, so error rate
+-- was unanswerable from any persisted data. Bounded by construction.
+ALTER TABLE "server_analytics_geo" ADD COLUMN IF NOT EXISTS "statuses" jsonb;
+--> statement-breakpoint
+
+-- Widen the per-minute bandwidth counters from int4 to int8.
+--
+-- These are bytes for ONE MINUTE, and int4 caps at 2,147,483,647 — about 2.1 GB,
+-- which a single minute reaches at roughly 286 Mbps. That is unremarkable for a
+-- site serving video or large downloads. Past it Postgres raises "integer out of
+-- range", the upsert for that minute fails, and the scrape for the whole domain
+-- dies with it — so the busiest domains on a box would be precisely the ones with
+-- no analytics, and it would look like the feature simply didn't work for them.
+--
+-- Now that collection runs unattended on a schedule rather than only when someone
+-- opens the tab, that failure would happen in a job with nobody reading the error.
+--
+-- Widening is lossless and there is no application change: the edge already
+-- accumulates these as Lua numbers (doubles, exact to 2^53) and Drizzle reads them
+-- back with mode:"number", which is exact over the same range.
+ALTER TABLE "server_analytics" ALTER COLUMN "bandwidth_in" TYPE bigint;
+--> statement-breakpoint
+ALTER TABLE "server_analytics" ALTER COLUMN "bandwidth_out" TYPE bigint;

package/dist/server/migrations/0085_resource_usage.sql

@@ -0,0 +1,68 @@
+-- Resource usage history: the persisted series behind the Monitoring tab's
+-- CPU/memory chart.
+--
+-- Usage existed only as a live 5-second SSE stream, so closing the tab discarded
+-- everything. "Was memory climbing before the OOM at 3am" was unanswerable, which is
+-- the question the data exists to answer.
+--
+-- ── Why this is a SAMPLED series, not a counter series like server_analytics ──
+--
+-- Traffic is counted for free: the edge increments a shared-dict key on every
+-- request, so per-minute granularity costs nothing and is exact. Resource usage has
+-- to be PROBED, and `docker stats` occupies the daemon for roughly a second per
+-- container because it must collect two CPU samples to compute a delta — about 500x
+-- a `docker inspect`. The sampling cadence therefore IS the resolution, and the cost
+-- scales with container count. 5-minute buckets are the compromise between catching
+-- a spike and not pinning every daemon on the estate; ~288 rows/day/service.
+--
+-- ── serviceKey is NOT NULL text, not a nullable service_id FK ────────────────
+--
+-- Postgres treats NULLs as DISTINCT in a unique index, so a nullable column cannot
+-- carry the "this row is the project itself" case without the partial-index
+-- workaround migration 0083 needed for service_incident. A sentinel ('__app__',
+-- matching the health watch's own constant so both features name the same workload
+-- identically) keeps one plain unique index correct.
+--
+-- Accepted consequence: deleting a service leaves its samples behind until retention
+-- prunes them. That is the right trade — history should outlive the thing it
+-- describes — and the project cascade still bounds the growth.
+--
+-- ── What is deliberately absent ─────────────────────────────────────────────
+--
+-- No project-total row. For a compose project the total is the per-bucket SUM of its
+-- services, computed at read. Storing it too would duplicate rows that are free to
+-- drift from their own sources, and would be wrong the moment a service is added or
+-- removed mid-window.
+CREATE TABLE IF NOT EXISTS "resource_usage" (
+  "id" text PRIMARY KEY NOT NULL,
+  "project_id" text NOT NULL REFERENCES "project"("id") ON DELETE CASCADE,
+  -- service.id, or '__app__' for a single-container project.
+  "service_key" text NOT NULL,
+  -- Bucket start in epoch MINUTES (not a timestamp), floored to a multiple of 5 —
+  -- same integer-bucket convention as server_analytics.
+  "minute" integer NOT NULL,
+  -- Per-CORE percent, as the runtimes report it: >100 is correct on a multi-core box
+  -- and must not be clamped. Readers divide by core count for "share of this host".
+  "cpu_percent" real DEFAULT 0 NOT NULL,
+  "memory_mb" real DEFAULT 0 NOT NULL,
+  -- CUMULATIVE since container start, not per-bucket deltas — that is what the
+  -- runtimes expose. Readers difference consecutive buckets and clamp negatives to
+  -- zero, because a restart resets the counter and would otherwise plot as a large
+  -- negative spike. bigint: a long-lived busy container passes int4 in days.
+  -- Always 0 for bare deploys (per-process accounting needs eBPF or a netns), so a
+  -- reader must not present 0 here as "no network traffic".
+  "network_rx_bytes" bigint DEFAULT 0 NOT NULL,
+  "network_tx_bytes" bigint DEFAULT 0 NOT NULL,
+  "created_at" timestamp DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+
+-- Makes a repeated sample for the same bucket a no-op (insert ... on conflict do
+-- nothing) rather than a double count, so the whole sweep is safely re-runnable.
+CREATE UNIQUE INDEX IF NOT EXISTS "uq_resource_usage_project_service_minute"
+  ON "resource_usage" ("project_id", "service_key", "minute");
+--> statement-breakpoint
+
+-- Every read is "this project over this window".
+CREATE INDEX IF NOT EXISTS "idx_resource_usage_project_minute"
+  ON "resource_usage" ("project_id", "minute");

package/dist/server/migrations/0086_remote_infra_updates.sql

@@ -0,0 +1,41 @@
+-- Remote infra (edge / mail container) update tracking.
+--
+-- When the control plane's APP_VERSION moves forward, the edge/mail containers
+-- already deployed on remote servers stay on their old image tag. Two additions
+-- drive the "monitor → advise → apply" loop for that drift:
+--
+--   instance_settings.auto_update_infra / last_seen_version — the instance-wide
+--     auto-update toggle and the version the infra reconcile last ran for (so the
+--     boot hook fires exactly once per upgrade). Both additive: the boolean
+--     carries a DEFAULT, last_seen_version is nullable.
+--
+--   server_container_status — the container sibling of server_module_status.
+--     One row per (server, component); "behind" is a plain tag comparison
+--     (running image ref != pinned image ref), since infra images are pinned to
+--     APP_VERSION. Mirrors the module table's org+behind index and unique key.
+ALTER TABLE "instance_settings" ADD COLUMN IF NOT EXISTS "auto_update_infra" boolean DEFAULT false NOT NULL;
+--> statement-breakpoint
+ALTER TABLE "instance_settings" ADD COLUMN IF NOT EXISTS "last_seen_version" text;
+--> statement-breakpoint
+CREATE TABLE IF NOT EXISTS "server_container_status" (
+  "id" text PRIMARY KEY NOT NULL,
+  "organization_id" text REFERENCES "organization"("id") ON DELETE CASCADE,
+  "server_id" text NOT NULL REFERENCES "servers"("id") ON DELETE CASCADE,
+  "component" text NOT NULL,
+  "running_label" text,
+  "pinned_label" text,
+  "running_version" text,
+  "pinned_version" text,
+  "behind" boolean DEFAULT false NOT NULL,
+  "latest_in_progress" boolean DEFAULT false NOT NULL,
+  "detail" jsonb,
+  "checked_at" timestamp DEFAULT now() NOT NULL,
+  "created_at" timestamp DEFAULT now() NOT NULL,
+  "updated_at" timestamp DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE UNIQUE INDEX IF NOT EXISTS "uq_server_container_status"
+  ON "server_container_status" ("server_id", "component");
+--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "idx_server_container_status_org_behind"
+  ON "server_container_status" ("organization_id", "behind");

package/dist/server/migrations/0087_server_container_version_columns.sql

@@ -0,0 +1,17 @@
+-- Repair: add `server_container_status.running_version` / `pinned_version` on a
+-- database that ran an earlier form of 0086.
+--
+-- 0086 was edited in place after it had already been applied: the version columns
+-- were added to its CREATE TABLE, but that create is `IF NOT EXISTS`, so on any
+-- database that had already run the older 0086 the table exists and the amended
+-- create is a silent no-op — the columns never appear. drizzle's migrator then
+-- refuses to re-run 0086 (its `when` equals the last-applied stamp, and the
+-- comparison is strict `<`; see 0081_project_readiness_repair for the same trap),
+-- so nothing else patches it either. Every `listByOrg` then dies with
+-- `column "running_version" does not exist` (42703).
+--
+-- Idempotent and stamped after 0086, so it sorts as a fresh migration: a database
+-- created cleanly gets the columns at 0086 and this is a no-op; a database that
+-- ran the stale 0086 gets them here. Safe to keep in history permanently.
+ALTER TABLE "server_container_status" ADD COLUMN IF NOT EXISTS "running_version" text;--> statement-breakpoint
+ALTER TABLE "server_container_status" ADD COLUMN IF NOT EXISTS "pinned_version" text;

package/dist/server/migrations/0088_audit_source_and_settings.sql

@@ -0,0 +1,27 @@
+-- Audit log: call source + per-org recording switch.
+--
+--   audit_event.source — WHERE the action came in from: "dashboard", "mcp",
+--     "cli", "api", "webhook", "system". Nullable on purpose: rows written
+--     before this migration have no knowable source and must not be
+--     mislabelled as any particular one, so they render as "Unknown". This is
+--     what makes "show me only what the AI assistant did" answerable at all —
+--     MCP tool calls re-enter the app through app.fetch() with nothing but an
+--     Authorization header, so until now an MCP-driven write and a CLI write
+--     produced identical rows.
+--
+--   audit_settings — recording on/off + retention, PER ORGANIZATION. Not
+--     instance_settings (one row would let a single tenant disable auditing for
+--     every org on a CLOUD_MODE instance) and not organization.metadata (Better
+--     Auth owns those writes, which is why the auditRetentionDays documented
+--     there never got a writer). Default enabled = true: auditing stays on
+--     unless an admin turns it off.
+ALTER TABLE "audit_event" ADD COLUMN IF NOT EXISTS "source" text;
+--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "audit_event_org_source_idx" ON "audit_event" ("organization_id","source","created_at" DESC);
+--> statement-breakpoint
+CREATE TABLE IF NOT EXISTS "audit_settings" (
+  "organization_id" text PRIMARY KEY NOT NULL REFERENCES "organization"("id") ON DELETE CASCADE,
+  "enabled" boolean DEFAULT true NOT NULL,
+  "retention_days" integer DEFAULT 90 NOT NULL,
+  "updated_at" timestamp DEFAULT now() NOT NULL
+);