openship 0.4.8 → 0.6.0

package/dist/server/lua/site_logger.lua

@@ -8,17 +8,31 @@
 --   s:{domain}:{epoch_min}:i        bandwidth in  bytes         (TTL 24h)
 --   s:{domain}:{epoch_min}:o        bandwidth out bytes         (TTL 24h)
 --   s:{domain}:{epoch_min}:t        response time sum (seconds) (TTL 24h)
---   s:{domain}:{epoch_min}:u        unique (non-static) reqs    (TTL 24h)
---   g:{domain}:{YYYYMMDD}:{CC}      country hit count           (TTL 48h)
+--   s:{domain}:{epoch_min}:u        page (non-static) requests  (TTL 24h)
 --   c:{domain}:{epoch_min}:{CC}     country per minute          (TTL 24h)
 --   t:{domain}:r / :i / :o          lifetime totals             (no TTL)
 --   d:{domain}                      domain index marker         (no TTL)
 --
+--   Daily rollup, all under one `g:{domain}:{YYYYMMDD}:` prefix so mgmt_api's
+--   /analytics/geo serves the whole day in a SINGLE dict scan:
+--   g:{domain}:{YYYYMMDD}:{CC}      country hit count           (TTL 48h)
+--   g:{domain}:{YYYYMMDD}:v         distinct visitors           (TTL 48h)
+--   g:{domain}:{YYYYMMDD}:p:{path}  path hit count              (TTL 48h)
+--   g:{domain}:{YYYYMMDD}:s:{code}  status-code count           (TTL 48h)
+--
+-- Shared dict key schema (visitors zone):
+--   vsalt:{YYYYMMDD}                per-day hash salt           (TTL 48h)
+--   vd:{domain}:{YYYYMMDD}:{hash}   distinct-visitor marker     (TTL 48h)
+--
 -- Shared dict key schema (request_data zone):
 --   rlog:{domain}:seq               monotonic write pointer
 --   rlog:{domain}:{slot}            JSON entry (ring buf)       (TTL 1h)
 --   log_pipe:sub:{domain}           live subscriber flag        (TTL 30s)
 --   log_pipe:q:{domain}             live log queue entries
+--
+-- NOTE on `:u` — it counts non-static REQUESTS, not people. It was surfaced as
+-- "unique IPs" all the way to the dashboard, which it never was. Real distinct
+-- visitors are the `vd:`/`:v` pair below.
 
 local cjson = require "cjson.safe"
 
@@ -35,6 +49,15 @@ if not pipe_ok then pipe_log = nil end
 
 local analytics    = ngx.shared.analytics
 local request_data = ngx.shared.request_data
+-- Per-host edge config, pushed by the API (see mgmt_api's /analytics/config).
+--
+-- The `rules` zone, NOT `analytics`: this holds a handful of small config values and is
+-- never under eviction pressure, whereas the analytics zone is a 256 MB churn of counters.
+-- An evicted flag there would silently stop path collection with nothing to explain why.
+local rules_dict   = ngx.shared.rules
+-- Optional: a box whose nginx.conf predates this zone keeps working, just without
+-- distinct-visitor counts. Never assume a dict exists.
+local visitors     = ngx.shared.visitors
 if not analytics or not request_data then return end
 
 -- ── Helpers ──────────────────────────────────────────────────────────────────
@@ -89,11 +112,71 @@ local function is_static_asset(u)
     return false
 end
 
+-- Collapse a request URI into a low-cardinality bucket suitable as a dict key.
+-- Without this, "top paths" is unbounded: every ?token=, every /orders/48219, and
+-- every 404 a scanner probes mints a permanent key, and the analytics zone
+-- LRU-evicts real counters to store junk.
+local function normalize_path(u)
+    -- Query string carries ids/tokens and never identifies the route.
+    local path = u:match("^([^?#]*)") or u
+    if path == "" then return "/" end
+    -- Numeric and UUID/hex segments are ids, not routes: /orders/48219 and
+    -- /orders/48220 are one path.
+    path = path:gsub("/%d+", "/:id")
+    path = path:gsub("/%x%x%x%x%x%x%x%x%-?%x*%-?%x*%-?%x*%-?%x*", "/:id")
+    -- Bound the key itself — a long URI must not become a long key.
+    if #path > 120 then path = path:sub(1, 120) .. "…" end
+    return path
+end
+
+-- ── Enumerable sets, without a zone scan ─────────────────────────────────────
+--
+-- Reading these counters back used to need key DISCOVERY, and a shared dict offers exactly
+-- one mechanism for that: `get_keys`, which walks EVERY key in the zone holding its lock.
+-- Cost scaled with how much was in the 256 MB zone, not with what was being asked for, and
+-- it ran on the mgmt path four different ways.
+--
+-- So every unbounded set now carries its own index: a counter plus numbered slots.
+--
+--     <counter>        how many distinct members
+--     <prefix><n>      the n-th member
+--
+-- A reader reads the counter and then reads slots 1..n. Bounded by the answer.
+--
+-- The write only happens on a member's FIRST sighting, so steady state costs nothing: the
+-- millionth request from Germany today takes the plain `incr` path and touches no index.
+local function index_add(counter_key, slot_prefix, member, ttl)
+    local n = analytics:incr(counter_key, 1, 0, ttl)
+    if n then analytics:set(slot_prefix .. n, member, ttl) end
+end
+
+--- Add 1 to a counter, creating it (with TTL) and indexing it on first sighting.
+--
+-- `incr` with no `init` returns nil for an absent key, which doubles as the existence
+-- probe — so the common case is ONE hash operation. TTL is set by the creating `safe_add`
+-- rather than passed to `incr`: shdict rejects `init_ttl` without `init`, and that raises a
+-- Lua error which aborts the whole log handler (it has silently zeroed every counter below
+-- it before).
+local function bump_indexed(key, ttl, counter_key, slot_prefix, member)
+    if analytics:incr(key, 1) then return end
+    if analytics:safe_add(key, 1, ttl) then
+        index_add(counter_key, slot_prefix, member, ttl)
+    else
+        -- Another worker created it between our probe and our add; our +1 still counts.
+        analytics:incr(key, 1)
+    end
+end
+
 local RING   = 1000
 local D24H   = 86400
 local D48H   = 172800
 local D1H    = 3600
 
+-- Distinct paths tracked per domain per day. Past this the tail folds into
+-- "other" rather than growing the zone — a URL-fuzzing bot is capped at one
+-- extra key, not thousands.
+local PATH_CARDINALITY_CAP = 2000
+
 -- ── Capture ──────────────────────────────────────────────────────────────────
 
 local host = normalize(ngx.var.host)
@@ -109,6 +192,22 @@ local rt      = tonumber(ngx.var.request_time)   or 0  -- seconds (float)
 local method  = ngx.var.request_method or "GET"
 local status  = ngx.var.status         or "0"
 
+-- Client country, resolved ONCE per request.
+--
+-- Hoisted out of the geo block because THREE consumers need it — the daily/per-minute
+-- rollups, the raw ring buffer behind /logs/recent, and the live SSE pipe. It used to be
+-- looked up inside the rollup and again inside pipe_log, so the ring buffer had no
+-- country at all: a request-log list showed flags on rows that arrived live and none on
+-- rows backfilled from /logs/recent, for the same traffic.
+--
+-- pcall'd because a corrupt or partially-written mmdb makes the lookup raise, and an
+-- error here would abort every counter below it (see the section header).
+local cc = nil
+if geo and geo.get_country_code then
+    local ok_cc, res = pcall(geo.get_country_code, ip)
+    if ok_cc and type(res) == "string" and res ~= "" then cc = res end
+end
+
 -- ── 1. Minute-bucket counters ────────────────────────────────────────────────
 
 local minute = math.floor(ts / 60)
@@ -136,26 +235,135 @@ analytics:incr("t:" .. host .. ":r", 1,       0)
 analytics:incr("t:" .. host .. ":i", req_len, 0)
 analytics:incr("t:" .. host .. ":o", bytes,   0)
 
--- Domain index (set once, ignore subsequent "exists" errors)
-analytics:safe_add("d:" .. host, 1)
+-- Domain index. `safe_add` returning true means this is the first request this box has
+-- ever seen for the host, which is exactly when the enumerable index needs appending —
+-- so domain discovery no longer costs a zone scan either.
+if analytics:safe_add("d:" .. host, 1) then
+    index_add("dn", "di:", host)
+end
+
+-- ── 3. Daily rollup: geo, visitors, paths, statuses ──────────────────────────
+--
+-- Each sub-block is pcall'd SEPARATELY, and that is load-bearing rather than
+-- defensive habit. A raised error in log_by_lua aborts the remainder of the
+-- handler, so one bad shdict call silently zeroes every counter written after it —
+-- which is exactly what an `incr(..., nil, ttl)` here did: paths AND statuses read
+-- as "no traffic" while requests and bandwidth looked perfect. Isolating them
+-- means a future edit can lose at most its own metric.
 
--- ── 3. GeoIP ─────────────────────────────────────────────────────────────────
+local day  = today()
+local gpfx = "g:" .. host .. ":" .. day .. ":"
 
-if geo then
-    local cc = geo.get_country_code(ip)
-    if cc and cc ~= "" then
+-- 3a. GeoIP. `cc` was resolved once above and is reused by the ring buffer and the
+-- live pipe, so a request costs exactly one mmdb lookup no matter how many consumers
+-- report its country.
+if cc then
+    pcall(function()
         -- Daily geo (for /analytics/geo endpoint)
-        analytics:incr("g:" .. host .. ":" .. today() .. ":" .. cc, 1, 0, D48H)
-        -- Per-minute geo (for time-series country breakdown)
-        analytics:incr("c:" .. host .. ":" .. minute .. ":" .. cc, 1, 0, D24H)
+        bump_indexed(gpfx .. cc, D48H, gpfx .. "cn", gpfx .. "ci:", cc)
+        -- Per-minute geo (for the time-series country breakdown). Indexed per MINUTE, so
+        -- the scraper reads only the minutes in its window.
+        local mpfx = "c:" .. host .. ":" .. minute .. ":"
+        bump_indexed(mpfx .. cc, D24H, mpfx .. "n", mpfx .. "i:", cc)
+    end)
+end
+
+-- 3b. Distinct visitors — a COUNT, never an identity.
+--
+-- The address is hashed with a salt that is generated on this box, rotates daily,
+-- and is never written anywhere but this shared dict, so the marker keys are not
+-- reversible to an IP and are worthless the next day. Only the resulting counter
+-- is ever flushed to Postgres; no per-visitor row exists at any layer. That is
+-- what lets us report real visitor numbers while still not collecting behavioural
+-- analytics on anyone's end users.
+--
+-- Per DAY, not per minute: a per-minute set is ~1440x the keys for a number
+-- nobody reads, and would evict the counters it exists to annotate.
+pcall(function()
+if visitors then
+    local salt_key = "vsalt:" .. day
+    local salt = visitors:get(salt_key)
+    if not salt then
+        -- First writer wins; a racing worker's safe_add fails and it re-reads. The
+        -- salt only has to be unpredictable and stable for the day.
+        salt = tostring(ngx.now()) .. tostring(math.random(1, 2147483647))
+        local ok = visitors:safe_add(salt_key, salt, D48H)
+        if not ok then salt = visitors:get(salt_key) or salt end
+    end
+    -- safe_add is the dedup: it succeeds ONLY on this visitor's first request
+    -- today, so the counter below increments exactly once per distinct visitor.
+    local marker = "vd:" .. host .. ":" .. day .. ":" .. ngx.crc32_long(salt .. ip)
+    if visitors:safe_add(marker, 1, D48H) then
+        analytics:incr(gpfx .. "v", 1, 0, D48H)
     end
 end
+end)
+
+-- 3c. Top paths — OPT-IN, non-static only, normalized and cardinality-capped.
+--
+-- Off unless the project turned it on, because this one block is the most expensive thing
+-- the log handler does. Measured on the shipped edge image, per request:
+--
+--     paths (whole block)        1.72 us      <- 57% of the counter path
+--       of which string work     1.38 us      normalize_path + is_static_asset
+--     minute buckets (5 incr)    0.61 us
+--     country (2 incr)           0.24 us
+--     status (1 incr)            0.14 us
+--     this flag read (1 get)     0.07 us
+--
+-- So the gate costs 0.07 us and saves 1.72 us — the counter path drops from ~3.0 us to
+-- ~1.35 us with paths off. It is also the highest-cardinality dimension (up to
+-- PATH_CARDINALITY_CAP keys per domain per day, against ~200 countries and a few dozen
+-- statuses) and the largest column in the daily rollup.
+--
+-- Absent flag = off. A box whose dict was just restarted therefore collects nothing until
+-- the API re-pushes, which is the safe direction: no data beats data nobody asked to pay
+-- for. The DB is the source of truth and re-pushes on every route apply.
+local collect_paths = rules_dict and rules_dict:get("cfg:" .. host .. ":paths")
+
+pcall(function()
+if collect_paths and not is_static_asset(uri) then
+    -- Hoisted: normalize_path is the single most expensive call in this handler (~1.4 us
+    -- of string work), so it runs exactly once even though both the counter key and the
+    -- index slot need its result.
+    local norm = normalize_path(uri)
+    local path_key = gpfx .. "p:" .. norm
+    -- incr-then-add: the hot path is ONE hash hit on an existing key. `incr` with
+    -- no init returns nil for an absent key, which doubles as the existence probe,
+    -- so only a path unseen today reaches the cap check below.
+    --
+    -- Deliberately no init_ttl here: shdict rejects init_ttl unless init is also
+    -- given ("'init_ttl' must be used with 'init'"), and that raises a Lua error
+    -- which aborts the whole log handler — taking the status counters below down
+    -- with it. The TTL is set by the safe_add on the creating request instead.
+    if not analytics:incr(path_key, 1) then
+        local distinct = analytics:incr(gpfx .. "pn", 1, 0, D48H) or 0
+        if distinct > PATH_CARDINALITY_CAP then
+            analytics:incr(gpfx .. "p:other", 1, 0, D48H)
+        else
+            -- `pn` was already the distinct-path counter, so the slots just give it an
+            -- enumerable side. Stored WITHOUT the "p:" prefix; the reader adds it back.
+            if analytics:safe_add(path_key, 1, D48H) then
+                analytics:set(gpfx .. "pi:" .. distinct, norm, D48H)
+            end
+        end
+    end
+end
+end)
+
+-- 3d. Status-code mix. Bounded by construction (a few dozen codes).
+pcall(function()
+    bump_indexed(gpfx .. "s:" .. status, D48H, gpfx .. "sn", gpfx .. "si:", status)
+end)
 
 -- ── 4. Raw request ring buffer ──────────────────────────────────────────────
 
 pcall(function()
     local ok_j, j = pcall(cjson.encode, {
         ip     = ip,
+        -- nil is simply omitted by cjson, so a box with no GeoIP writes the same shape
+        -- minus this key rather than a null the reader has to special-case.
+        country = cc,
         ts     = ts,
         method = method,
         status = status,
@@ -178,5 +386,5 @@ end)
 if pipe_log and pipe_log.pipe_request_log
    and request_data:get("log_pipe:sub:" .. host) then
     ngx.timer.at(0, pipe_log.pipe_request_log,
-                 host, ip, ts, ua, uri, req_len, bytes, rt, method, status)
+                 host, ip, ts, ua, uri, req_len, bytes, rt, method, status, cc)
 end

package/dist/server/migrations/0073_project_volumes_object_storage.sql

@@ -0,0 +1,3 @@
+ALTER TABLE "project" ADD COLUMN IF NOT EXISTS "volumes" jsonb;
+--> statement-breakpoint
+ALTER TABLE "project" ADD COLUMN IF NOT EXISTS "object_storage" jsonb;

package/dist/server/migrations/0074_project_compose_path.sql

@@ -0,0 +1 @@
+ALTER TABLE "project" ADD COLUMN IF NOT EXISTS "compose_path" text;

package/dist/server/migrations/0075_service_command_argv.sql

@@ -0,0 +1 @@
+ALTER TABLE "service" ADD COLUMN IF NOT EXISTS "command_argv" jsonb;

package/dist/server/migrations/0076_rollback_retention.sql

@@ -0,0 +1,17 @@
+-- Rollback retention: auto-sized window inputs on the project, and the removal
+-- of a per-service artifact flag that was never written or read.
+--
+-- `rollback_window` keeps its existing meaning with one addition: NULL now means
+-- "auto" — resolve to `rollback_window_computed` (measured from the deploy host's
+-- free disk and this project's average snapshot size), falling back to
+-- instance_settings.default_rollback_window when nothing was measured yet. A
+-- non-null `rollback_window` is still an explicit operator override.
+ALTER TABLE "project" ADD COLUMN IF NOT EXISTS "snapshot_size_bytes" bigint;
+--> statement-breakpoint
+ALTER TABLE "project" ADD COLUMN IF NOT EXISTS "rollback_window_computed" integer;
+--> statement-breakpoint
+ALTER TABLE "project" ADD COLUMN IF NOT EXISTS "capacity_measured_at" timestamp;
+--> statement-breakpoint
+-- Dead since it was introduced: the deployment-level `artifact_retained_at` is
+-- the only retention flag any code path writes or reads.
+ALTER TABLE "service_deployment" DROP COLUMN IF EXISTS "artifact_retained_at";

package/dist/server/migrations/0077_domain_redirect.sql

@@ -0,0 +1,13 @@
+-- Per-domain canonical redirect: this hostname answers with a redirect to another
+-- of the project's hostnames instead of serving the app.
+--
+-- Lives on `domain` rather than in a routing blob because the domain table IS the
+-- store for a project's public endpoints (deriveProjectRouteState builds
+-- publicEndpoints FROM these rows), and a redirect is a property of one hostname.
+--
+-- `redirect_to` NULL = serve the app (every existing row, unchanged).
+-- `redirect_status` NULL = 301. Kept nullable rather than DEFAULT 301 so "no
+-- redirect" is one state, not a status attached to a row that doesn't redirect.
+ALTER TABLE "domain" ADD COLUMN IF NOT EXISTS "redirect_to" text;
+--> statement-breakpoint
+ALTER TABLE "domain" ADD COLUMN IF NOT EXISTS "redirect_status" integer;

package/dist/server/migrations/0078_grant_source_scope.sql

@@ -0,0 +1,30 @@
+-- Per-repo source access scope on every grant-bearing table.
+--
+-- A grant already carries a VERB in `permissions_json` (read / write / admin).
+-- This column adds the SURFACE: may the holder read file CONTENT, and under which
+-- paths. The two do not overlap, and the absence of a scope is meaningful:
+--
+--   read + scope_json NULL                        -> metadata only (branches,
+--                                                    detect, deploy). THE DEFAULT.
+--   read + {"v":1,"read":{"paths":["src/**"]}}    -> file content under src only
+--   write + {"v":1,"write":{"paths":[...]}}       -> content writes under those paths
+--
+-- So a repo grant no longer implies the right to crawl the repository. NULL is the
+-- restrictive state, which is why this is nullable with no default and no backfill:
+-- every existing row correctly reads as metadata-only.
+--
+-- Encoded as JSON text (not jsonb, not text[]) to match `permissions_json` — the
+-- column types have to behave identically across PGlite and Postgres without
+-- driver-specific casting, and the repos already own (de)serialisation.
+--
+-- Shape + matching semantics: packages/core/src/source-access.ts. Malformed JSON,
+-- an unknown "v", or an unparseable pattern all resolve to NO access rather than
+-- unrestricted access.
+ALTER TABLE "resource_grant" ADD COLUMN IF NOT EXISTS "scope_json" text;
+--> statement-breakpoint
+ALTER TABLE "personal_access_token_grant" ADD COLUMN IF NOT EXISTS "scope_json" text;
+--> statement-breakpoint
+-- Pending invite grants carry the scope through acceptance; without it, a
+-- restricted invitee would materialise into a metadata-only grant and silently
+-- lose the content access the inviter picked.
+ALTER TABLE "invitation_pending_grant" ADD COLUMN IF NOT EXISTS "scope_json" text;

package/dist/server/migrations/0079_project_readiness.sql

@@ -0,0 +1,18 @@
+-- Deploy-time readiness gate, per project.
+--
+-- Named `readiness`, NOT `health_check`, on purpose: a compose service already
+-- carries a `healthcheck` (the Docker HEALTHCHECK directive, stored in
+-- `service.advanced`) which the daemon runs and which has nothing to do with
+-- whether a DEPLOY is allowed to finish. Two fields one capital letter apart in
+-- the same openship.json would be a footgun.
+--
+-- Nullable with NO default, and no backfill: NULL means "off", so every existing
+-- project moves to the new off-by-default behaviour. Previously the 15s
+-- stabilization watch and the 45s TCP probe were hardcoded on every deploy and
+-- could fail (and then force-remove) a container that was running fine. Opting
+-- in is now explicit.
+--
+-- Shape (all fields optional, all defaults off) — see OpenshipReadiness:
+--   { enabled, path, port, timeoutSeconds,
+--     stabilization, stabilizationSeconds, onFailure: "warn" | "fail" }
+ALTER TABLE "project" ADD COLUMN IF NOT EXISTS "readiness" jsonb;

package/dist/server/migrations/0080_deployment_error_classification.sql

@@ -0,0 +1,26 @@
+-- Durable failure classification for a deployment.
+--
+-- The pipeline already classifies failures precisely: a port conflict throws
+-- DeployError(msg, "PORT_IN_USE", {port, pid, command, systemdUnit,
+-- isManagedDeployment, …}) and that reaches onFailure intact. It was then split,
+-- and only half of it was kept: `error_message` went to this table while
+-- `errorCode`/`errorDetails` went ONLY to the in-memory SSE session. Once that
+-- session was evicted (or the API restarted) all that survived a port conflict
+-- was an English sentence, so nothing could offer the operator — or an MCP agent
+-- — a way forward.
+--
+-- The one attempt to recover the code was inverted: build-status re-derived it
+-- with error_message.includes("PORT_IN_USE"), and none of the coded messages
+-- contain that token (they read "Port 3000 is already in use by …"). So the
+-- failures we DID classify lost their code and the ones we didn't got it guessed.
+--
+-- `error_code` is free text with no check constraint, matching `status` on this
+-- table: codes come from DeployError call sites across packages/adapters and must
+-- be extendable without a migration. `error_details` is the DeployError details
+-- bag verbatim (never contains secrets — it is process/port metadata).
+--
+-- Both nullable with no backfill: NULL means "not classified", which is exactly
+-- true of every pre-existing row.
+ALTER TABLE "deployment" ADD COLUMN IF NOT EXISTS "error_code" text;
+--> statement-breakpoint
+ALTER TABLE "deployment" ADD COLUMN IF NOT EXISTS "error_details" jsonb;

package/dist/server/migrations/0081_project_readiness_repair.sql

@@ -0,0 +1,34 @@
+-- Repair: create `project.readiness` on a database that skipped 0079.
+--
+-- WHY THIS EXISTS (and why it is not just "0079 again"):
+--
+-- 0079 was originally `0079_project_health_check` (adding `project.health_check`)
+-- and was replaced in place by `0079_project_readiness` (adding
+-- `project.readiness`) before either shipped. Both journal entries carry the SAME
+-- `when` — 1785882107324 — because the second reused the first's slot.
+--
+-- drizzle's migrator reads only the single most recent applied row and applies a
+-- migration when `lastDbMigration.created_at < migration.folderMillis`
+-- (pg-core/dialect.js). That comparison is STRICT, so on any database that had
+-- already run the old 0079, the new one is equal, not greater — it is skipped
+-- without a word. 0080 carried a later stamp and applied normally, which is why
+-- such a database ends up with 0080's columns, an orphan `health_check`, and no
+-- `readiness` at all. Every query selecting the project row then dies with
+-- `column "readiness" does not exist` (42703).
+--
+-- Bumping 0079's `when` would NOT fix it: the max applied stamp is now 0080's, so
+-- anything ordered before 0080 still compares as already-applied. The repair has
+-- to sort AFTER 0080, which is what this file is.
+--
+-- Idempotent on purpose. A fresh database creates the column at 0079 and this is
+-- a no-op; a database that skipped 0079 creates it here. Either way the end state
+-- is identical, so this is safe to leave in history permanently rather than being
+-- a one-off someone has to remember to delete.
+ALTER TABLE "project" ADD COLUMN IF NOT EXISTS "readiness" jsonb;--> statement-breakpoint
+
+-- Drop the column the replaced migration left behind. `health_check` was only
+-- ever created by 0079_project_health_check, which never shipped in a release and
+-- is referenced by no schema, query, or type in the codebase — so on the
+-- databases that have it, it is dead weight with a confusingly similar name to
+-- the compose service `healthcheck`. Guarded, so this is a no-op everywhere else.
+ALTER TABLE "project" DROP COLUMN IF EXISTS "health_check";

package/dist/server/migrations/0082_edge_target_verification.sql

@@ -0,0 +1,39 @@
+-- Proof that a routing TARGET is ours, as Openship Cloud's shared edge now requires
+-- before forwarding `<slug>.opsh.io` to it.
+--
+-- Keyed on (organization_id, target) because that is what the upstream keys it on:
+-- one target serves every free domain on that box, so this is target-level state,
+-- not domain-level. One install holds several rows — its own address plus one per
+-- remote server it deploys to.
+--
+-- The token is persisted HERE, not only on the edge, because the upstream's list
+-- endpoint returns id/target/status/expiry but NOT the token. Verification lasts 90
+-- days and the upstream re-probes THE SAME token within 7 days of expiry; a 404 then
+-- expires it and the free domain stops resolving ~83 days after a green deploy with
+-- nothing in our logs. So a token that lived only on a box's disk would be
+-- unrecoverable after a rebuild, and rows here are never deleted when a free domain
+-- is dropped.
+--
+-- `server_id` is ON DELETE SET NULL, not cascade: losing the server row must not
+-- take the token with it.
+CREATE TABLE IF NOT EXISTS "edge_target_verification" (
+  "id" text PRIMARY KEY NOT NULL,
+  "organization_id" text NOT NULL REFERENCES "organization"("id") ON DELETE CASCADE,
+  "target" text NOT NULL,
+  "host" text NOT NULL,
+  "server_id" text REFERENCES "servers"("id") ON DELETE SET NULL,
+  "verification_id" integer,
+  "token" text,
+  "retired_tokens" jsonb,
+  "challenge_path" text,
+  "status" text NOT NULL DEFAULT 'pending',
+  "validated_ip" text,
+  "expires_at" timestamp,
+  "last_checked_at" timestamp,
+  "last_error" text,
+  "created_at" timestamp NOT NULL DEFAULT now(),
+  "updated_at" timestamp NOT NULL DEFAULT now()
+);
+--> statement-breakpoint
+CREATE UNIQUE INDEX IF NOT EXISTS "uq_edge_target_verification"
+  ON "edge_target_verification" ("organization_id", "target");

package/dist/server/migrations/0083_service_incident.sql

@@ -0,0 +1,84 @@
+-- Continuous container health monitoring: the durable memory behind the alerts.
+--
+-- Openship knew a container had crashed at exactly ONE moment — the ~15s post-deploy
+-- stabilization watch. After that nobody looked again, so an OOM-kill at 3am or a
+-- Postgres that starts bouncing after a host reboot reached the operator only when a
+-- user complained. The health watch closes that, but a poller that alerts on "the
+-- container isn't running" gets muted within a day: a redeploy recreating containers,
+-- an operator's `docker stop`, an unreachable host, and a crash loop all look exactly
+-- like a failure at a point in time.
+--
+-- So the unit of alerting is an INCIDENT, not a state reading: opened once, escalated
+-- only when it gets worse, resolved once with a downtime duration. This table is that
+-- memory — it survives the control-plane restart that in-process state would not (a box
+-- down for three days must not re-page on every API restart), and it doubles as the
+-- project Health tab's history.
+CREATE TABLE IF NOT EXISTS "service_incident" (
+  "id" text PRIMARY KEY NOT NULL,
+  "organization_id" text NOT NULL REFERENCES "organization"("id") ON DELETE CASCADE,
+  -- Null for SERVER-scoped incidents (an unreachable box gets ONE incident, not one
+  -- per service on it — that fan-out is the single loudest source of alert fatigue).
+  "project_id" text REFERENCES "project"("id") ON DELETE CASCADE,
+  -- The resolved service row, when there is one. SET NULL, not cascade: deleting a
+  -- service must not erase the record of the outage it had (`service_key` still
+  -- identifies it, and `service_name` is snapshotted below).
+  "service_id" text REFERENCES "service"("id") ON DELETE SET NULL,
+  -- Stable identity across a container RECREATE, which is the whole point: every
+  -- deploy mints a new container id, so keying incidents on container id would open a
+  -- fresh incident for the same workload after each redeploy. Service id when we
+  -- resolved one, else the container name (single-app deploys have no service row);
+  -- `server:<id>` for server-scoped rows.
+  "service_key" text NOT NULL,
+  -- Snapshot of the display name at incident time — the alert text must still read
+  -- correctly after a rename, and history rows must not silently change meaning.
+  "service_name" text NOT NULL,
+  "server_id" text REFERENCES "servers"("id") ON DELETE SET NULL,
+  "container_id" text,
+  -- down | crash_loop | unhealthy | server_unreachable. Ordered by severity in the
+  -- watcher: an incident escalates upward (and re-notifies), never downward.
+  "kind" text NOT NULL,
+  "status" text NOT NULL DEFAULT 'open', -- open | resolved
+  "reason" text,
+  "exit_code" integer,
+  "restart_count" integer NOT NULL DEFAULT 0,
+  "oom_killed" boolean NOT NULL DEFAULT false,
+  -- Consecutive ticks that agreed on this verdict. An incident is only opened (and
+  -- only notified) at >= 2, so a snapshot taken mid-restart or during a slow
+  -- healthcheck start never pages anyone.
+  "confirmations" integer NOT NULL DEFAULT 0,
+  -- Notification bookkeeping. Deliberately a count, not a flag: open, escalation and
+  -- resolve each notify exactly once, and nothing else ever does.
+  "notify_count" integer NOT NULL DEFAULT 0,
+  "notified_at" timestamp,
+  "log_excerpt" text,
+  "opened_at" timestamp NOT NULL DEFAULT now(),
+  "resolved_at" timestamp,
+  -- Last tick that actually OBSERVED this workload. Not advanced when the host is
+  -- unreachable — during a connectivity loss we know nothing, so nothing is claimed.
+  "last_seen_at" timestamp NOT NULL DEFAULT now(),
+  "created_at" timestamp NOT NULL DEFAULT now(),
+  "updated_at" timestamp NOT NULL DEFAULT now()
+);
+--> statement-breakpoint
+-- One open incident per workload — the dedup primitive. As a partial unique index,
+-- "we are already alerting about this" is a DB invariant rather than sweep
+-- bookkeeping: two overlapping ticks (a manual Run now landing on a cron tick) cannot
+-- both open one and double-notify.
+CREATE UNIQUE INDEX IF NOT EXISTS "uq_service_incident_open"
+  ON "service_incident" ("project_id", "service_key")
+  WHERE "status" = 'open' AND "project_id" IS NOT NULL;
+--> statement-breakpoint
+-- Same invariant for server-scoped rows. Needs its own index because Postgres treats
+-- NULLs as distinct in a unique index, so the one above would happily allow a second
+-- open row for the same box.
+CREATE UNIQUE INDEX IF NOT EXISTS "uq_service_incident_open_server"
+  ON "service_incident" ("server_id")
+  WHERE "status" = 'open' AND "project_id" IS NULL;
+--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "idx_service_incident_project"
+  ON "service_incident" ("project_id", "opened_at" DESC);
+--> statement-breakpoint
+-- Intent marker. `disableProject` stopped the container and recorded NOTHING, so
+-- "the operator turned this off" was unknowable — the health watch would page about
+-- every deliberately-disabled project, forever. Set on disable, cleared on enable.
+ALTER TABLE "project" ADD COLUMN IF NOT EXISTS "disabled_at" timestamp;

package/dist/server/migrations/0084_analytics_daily_rollup.sql

@@ -0,0 +1,72 @@
+-- Widen the daily analytics rollup: distinct visitors, top paths, status mix.
+--
+-- The visitor pipeline already persisted per-country counts, and the dashboard
+-- already rendered cards for "unique IPs" and a top-paths table. Neither was real:
+-- `top_paths` was hardcoded `[]` in the service layer, and "unique IPs" was
+-- `unique_requests` — the count of non-static REQUESTS — so five page views from
+-- one browser read as five visitors. These columns are what makes those two
+-- numbers mean what the UI has been claiming.
+--
+-- All three land HERE, on the daily table, rather than on the per-minute one:
+-- visitors and paths are the high-cardinality metrics, and holding them per-minute
+-- would multiply the edge's shared-dict key count by ~1440 to feed a chart nobody
+-- asked for — evicting, in the process, the request counters they annotate.
+-- Countries already live here for the same reason, and the edge keeps all four
+-- under one `g:{domain}:{day}:` key prefix so a reader gets the whole day in a
+-- single dict scan.
+--
+-- Additive with defaults on purpose: `migrations-additive.test.ts` rejects a
+-- NOT NULL column without a DEFAULT, because an older cross-version dump omits the
+-- column entirely and Drizzle then emits DEFAULT for it — with no default that's a
+-- NULL into NOT NULL on the newer receiver, i.e. a broken cloud/project transfer.
+
+-- Distinct visitors for the day, deduplicated AT THE EDGE and stored as a count.
+--
+-- Privacy is a property of where the dedup happens: the edge hashes each address
+-- with a salt that is generated on that box, rotates daily, and never leaves its
+-- shared memory. Only the cardinality of that set is written here. No address and
+-- no per-visitor row exists at any layer, which is what keeps the published
+-- "we don't collect behavioural analytics on your end users" true while still
+-- reporting a real visitor number.
+--
+-- Understates on a very busy domain: the edge's `visitors` zone LRU-evicts past
+-- roughly 1M distinct/day. mgmt_api `GET /status` reports that zone's free space so
+-- a reader can label the number approximate instead of presenting an eviction
+-- artifact as a measurement.
+ALTER TABLE "server_analytics_geo" ADD COLUMN IF NOT EXISTS "visitors" integer DEFAULT 0 NOT NULL;
+--> statement-breakpoint
+
+-- Top paths: { "/": 900, "/orders/:id": 120, "other": 40 }.
+--
+-- Normalized at the edge before it ever becomes a key: query string stripped (so a
+-- ?token= or ?session= can never be persisted here), numeric and UUID segments
+-- collapsed to `:id`, key length capped, and the tail past a cardinality cap folded
+-- into "other" — otherwise a scanner walking /wp-admin variants mints thousands of
+-- permanent keys and evicts the real counters.
+ALTER TABLE "server_analytics_geo" ADD COLUMN IF NOT EXISTS "paths" jsonb;
+--> statement-breakpoint
+
+-- Status-code mix: { "200": 4210, "404": 17, "502": 3 }. Status was captured into
+-- the edge's raw-request ring buffer (RAM, 1h) but never aggregated, so error rate
+-- was unanswerable from any persisted data. Bounded by construction.
+ALTER TABLE "server_analytics_geo" ADD COLUMN IF NOT EXISTS "statuses" jsonb;
+--> statement-breakpoint
+
+-- Widen the per-minute bandwidth counters from int4 to int8.
+--
+-- These are bytes for ONE MINUTE, and int4 caps at 2,147,483,647 — about 2.1 GB,
+-- which a single minute reaches at roughly 286 Mbps. That is unremarkable for a
+-- site serving video or large downloads. Past it Postgres raises "integer out of
+-- range", the upsert for that minute fails, and the scrape for the whole domain
+-- dies with it — so the busiest domains on a box would be precisely the ones with
+-- no analytics, and it would look like the feature simply didn't work for them.
+--
+-- Now that collection runs unattended on a schedule rather than only when someone
+-- opens the tab, that failure would happen in a job with nobody reading the error.
+--
+-- Widening is lossless and there is no application change: the edge already
+-- accumulates these as Lua numbers (doubles, exact to 2^53) and Drizzle reads them
+-- back with mode:"number", which is exact over the same range.
+ALTER TABLE "server_analytics" ALTER COLUMN "bandwidth_in" TYPE bigint;
+--> statement-breakpoint
+ALTER TABLE "server_analytics" ALTER COLUMN "bandwidth_out" TYPE bigint;