openship 0.6.0 → 0.6.1

package/dist/server/lua/geo_country.lua

@@ -14,9 +14,10 @@
 --                 image from apps/api/assets/geoip, so no runtime download)
 --
 -- Falls back gracefully - if the library or database is missing,
--- get_country_code() returns nil.  Both site_logger.lua and pipe_log.lua
--- wrap this module in pcall(require, ...) so a missing DB never crashes
--- the request pipeline. mgmt_api `GET /status` reports whether geo actually
+-- get_country_code() returns nil.  site_logger.lua wraps this module in
+-- pcall(require, ...) so a missing DB never crashes the request pipeline
+-- (the country it resolves flows into the ring buffer, and from there to
+-- both /logs/recent and the live SSE stream). mgmt_api `GET /status` reports whether geo actually
 -- resolved, so a silently-dark edge is visible instead of just yielding an
 -- empty country map.
 

package/dist/server/lua/pipe_stream.lua

@@ -2,6 +2,14 @@
 -- content_by_lua: SSE endpoint for real-time request log streaming.
 -- GET /logs/stream?domain=example.com
 -- Internal only - 127.0.0.1:9145
+--
+-- Reads the SAME ring buffer site_logger.lua writes (rlog:{domain}:seq +
+-- rlog:{domain}:{slot}), by a PER-CONNECTION cursor. Nothing is consumed: N
+-- concurrent watchers each walk the ring independently and none steals another's
+-- frames. This is what replaced the old single-subscriber log_pipe:q queue —
+-- a containerized edge tails this over `docker exec curl`, and a curl that
+-- outlives its client (the daemon buffers its stdout, so no EPIPE) can no longer
+-- drain the queue out from under the next reader.
 
 local sh = ngx.shared.request_data
 if not sh then
@@ -20,12 +28,10 @@ end
 domain = domain:lower()
 if domain:sub(1, 4) == "www." then domain = domain:sub(5) end
 
-local SUB_KEY   = "log_pipe:sub:" .. domain
-local QUEUE_KEY = "log_pipe:q:"   .. domain
-
--- Clear stale queue entries, mark subscriber active
-sh:delete(QUEUE_KEY)
-sh:set(SUB_KEY, true, 30)
+-- MUST match site_logger.lua's RING — same slot arithmetic on both ends.
+local RING    = 1000
+local SEQ_KEY = "rlog:" .. domain .. ":seq"
+local slot_key = function(seq) return "rlog:" .. domain .. ":" .. (seq % RING) end
 
 ngx.header["Content-Type"]      = "text/event-stream"
 ngx.header["Cache-Control"]     = "no-cache, no-store"
@@ -36,55 +42,62 @@ ngx.header["X-Accel-Buffering"] = "no"
 -- Without this, ngx.flush on an empty buffer sends chunked-EOF (0\r\n\r\n)
 -- and terminates the response immediately.
 ngx.print(": connected\n\n")
-if not ngx.flush(true) then
-    sh:delete(SUB_KEY)
-    return
-end
+if not ngx.flush(true) then return end
 
-local started  = ngx.now()
-local last_hb  = started
-local last_ref = started
+-- Start at the current head: stream only requests that arrive AFTER connect.
+-- History up to this point is served separately by /logs/recent, and the client
+-- dedups the two by the deterministic `{host}:{seq}` id, so a row seen by both
+-- collapses to one.
+local cursor  = tonumber(sh:get(SEQ_KEY)) or 0
+local started = ngx.now()
+local last_hb = started
 
 while true do
-    -- Max 1 hour per connection
-    if ngx.now() - started > 3600 then
-        sh:delete(SUB_KEY)
-        return
-    end
+    -- Max 1 hour per connection.
+    if ngx.now() - started > 3600 then return end
+
+    local head = tonumber(sh:get(SEQ_KEY)) or 0
+
+    -- If we've fallen more than a full ring behind, the oldest unread slots have
+    -- already been overwritten. Jump to the oldest slot still holding its own
+    -- entry (head-RING maps to head's slot, so head-RING+1 is the oldest live one)
+    -- and skip the lost range — same bound the old 2000-entry queue had.
+    if cursor < head - RING then cursor = head - RING end
 
-    -- Drain up to 100 queued entries per cycle
     local sent = 0
-    while sent < 100 do
-        local entry = sh:rpop(QUEUE_KEY)
-        if not entry then break end
-        ngx.print("event: request\ndata: ", entry, "\n\n")
-        sent = sent + 1
+    while cursor < head and sent < 100 do
+        local next_seq = cursor + 1
+        local entry = sh:get(slot_key(next_seq))
+        if entry then
+            ngx.print("event: request\ndata: ", entry, "\n\n")
+            sent = sent + 1
+            cursor = next_seq
+        elseif next_seq < head then
+            -- A later seq already exists, so this can't be the incr(seq)→set(slot)
+            -- write window: the slot is genuinely lost (expired or wrapped). Skip it.
+            cursor = next_seq
+        else
+            -- next_seq == head: the newest entry's slot may not be written yet
+            -- (the microsecond window between incr(seq) and set(slot)). Retry it
+            -- next cycle without advancing.
+            break
+        end
     end
 
     if sent > 0 then
-        if not ngx.flush(true) then
-            sh:delete(SUB_KEY)
-            return
-        end
+        if not ngx.flush(true) then return end
     end
 
     local now = ngx.now()
 
-    -- Heartbeat every 15s
+    -- Heartbeat every 15s. Its flush is also how a dead client is detected: once
+    -- the reader (a `docker exec curl`) goes away and its socket closes, this
+    -- flush fails and the connection tears down here.
     if now - last_hb > 15 then
         ngx.print(": ping\n\n")
-        if not ngx.flush(true) then
-            sh:delete(SUB_KEY)
-            return
-        end
+        if not ngx.flush(true) then return end
         last_hb = now
     end
 
-    -- Refresh subscriber TTL every 10s
-    if now - last_ref > 10 then
-        sh:set(SUB_KEY, true, 30)
-        last_ref = now
-    end
-
     ngx.sleep(0.05)
 end

package/dist/server/lua/site_logger.lua

@@ -27,8 +27,10 @@
 -- Shared dict key schema (request_data zone):
 --   rlog:{domain}:seq               monotonic write pointer
 --   rlog:{domain}:{slot}            JSON entry (ring buf)       (TTL 1h)
---   log_pipe:sub:{domain}           live subscriber flag        (TTL 30s)
---   log_pipe:q:{domain}             live log queue entries
+--
+-- The ring is ALSO the live source: pipe_stream.lua's SSE endpoint reads these
+-- slots by a per-connection cursor, so there is no separate live queue or
+-- subscriber flag — every watcher sees the full stream and steals from none.
 --
 -- NOTE on `:u` — it counts non-static REQUESTS, not people. It was surfaced as
 -- "unique IPs" all the way to the dashboard, which it never was. Real distinct
@@ -43,10 +45,6 @@ if not geo_ok then
     ngx.log(ngx.WARN, "[site_logger] geo_country module not available. GeoIP disabled.")
 end
 
--- SAFE LOAD: Pipe module (per-worker, cached by require)
-local pipe_ok, pipe_log = pcall(require, "openship.pipe_log")
-if not pipe_ok then pipe_log = nil end
-
 local analytics    = ngx.shared.analytics
 local request_data = ngx.shared.request_data
 -- Per-host edge config, pushed by the API (see mgmt_api's /analytics/config).
@@ -194,11 +192,12 @@ local status  = ngx.var.status         or "0"
 
 -- Client country, resolved ONCE per request.
 --
--- Hoisted out of the geo block because THREE consumers need it — the daily/per-minute
--- rollups, the raw ring buffer behind /logs/recent, and the live SSE pipe. It used to be
--- looked up inside the rollup and again inside pipe_log, so the ring buffer had no
--- country at all: a request-log list showed flags on rows that arrived live and none on
--- rows backfilled from /logs/recent, for the same traffic.
+-- Hoisted out of the geo block because both the daily/per-minute rollups and the raw
+-- ring buffer need it. The ring is the single source for the request log — /logs/recent
+-- reads it directly and the live SSE stream (pipe_stream.lua) reads the same slots by
+-- cursor — so resolving the country here means every row carries its flag no matter
+-- which path delivered it. An earlier split (looked up in the rollup, again in a
+-- separate live pipe) left the ring with no country at all.
 --
 -- pcall'd because a corrupt or partially-written mmdb makes the lookup raise, and an
 -- error here would abort every counter below it (see the section header).
@@ -356,10 +355,20 @@ pcall(function()
     bump_indexed(gpfx .. "s:" .. status, D48H, gpfx .. "sn", gpfx .. "si:", status)
 end)
 
--- ── 4. Raw request ring buffer ──────────────────────────────────────────────
+-- ── 4. Raw request ring buffer (also the live SSE source) ────────────────────
+--
+-- `seq` is claimed BEFORE encoding so the stored JSON can carry a deterministic
+-- id (`{host}:{seq}`). That id is the dedup key everywhere downstream: /logs/recent
+-- returns these entries verbatim, and pipe_stream.lua streams the same slots live,
+-- so the same request reaches the client with one stable id instead of a fresh
+-- random one per source. `incr` with init 0 never returns nil here; guard anyway
+-- so a shdict hiccup can't index a nil into the slot key.
 
 pcall(function()
+    local seq = request_data:incr("rlog:" .. host .. ":seq", 1, 0)
+    if not seq then return end
     local ok_j, j = pcall(cjson.encode, {
+        id     = host .. ":" .. seq,
         ip     = ip,
         -- nil is simply omitted by cjson, so a box with no GeoIP writes the same shape
         -- minus this key rather than a null the reader has to special-case.
@@ -374,17 +383,6 @@ pcall(function()
         rt     = rt,
     })
     if ok_j and j then
-        local seq  = request_data:incr("rlog:" .. host .. ":seq", 1, 0)
-        local slot = seq % RING
-        request_data:set("rlog:" .. host .. ":" .. slot, j, D1H)
+        request_data:set("rlog:" .. host .. ":" .. (seq % RING), j, D1H)
     end
 end)
-
--- ── 5. Live-log pipe (only when a subscriber is watching) ───────────────────
--- Fire via timer to avoid blocking the log phase
-
-if pipe_log and pipe_log.pipe_request_log
-   and request_data:get("log_pipe:sub:" .. host) then
-    ngx.timer.at(0, pipe_log.pipe_request_log,
-                 host, ip, ts, ua, uri, req_len, bytes, rt, method, status, cc)
-end

package/dist/{setup-FA4PW4NU.js → setup-M2ZDGH3O.js}

@@ -4,15 +4,15 @@ import { createRequire as __ospCreateRequire } from "node:module";
 const require = __ospCreateRequire(import.meta.url);
 import {
   SystemManager
-} from "./chunk-EP2TNALB.js";
-import "./chunk-GUQ3QCJ7.js";
+} from "./chunk-3NZICQHF.js";
+import "./chunk-IOT5ZX23.js";
 import "./chunk-IW2UKBVF.js";
 import "./chunk-ZCFVHA25.js";
-import "./chunk-QEH4QUBX.js";
+import "./chunk-EW6KB2J7.js";
 import "./chunk-I3DPMTSF.js";
 import "./chunk-VZ2ZHPS2.js";
 import "./chunk-3LDWWY7U.js";
-import "./chunk-OGY63DGD.js";
+import "./chunk-YIPMDGG6.js";
 import "./chunk-ZFJXPCQP.js";
 import "./chunk-OTLSJYAC.js";
 import "./chunk-PXKCZB2I.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openship",
-  "version": "0.6.0",
+  "version": "0.6.1",
   "description": "Openship CLI - deploy, manage, and open your Openship platform from the terminal",
   "license": "Apache-2.0",
   "repository": {

package/dist/server/lua/pipe_log.lua

@@ -1,77 +0,0 @@
--- pipe_log.lua
--- Push request log into shared dict queue log_pipe:q:{host} for /stream SSE.
--- Payload shape matches requestLogger.js (id, ip, country when known,
--- timestamp, date, uri, method, status, userAgent, requestSize, responseSize,
--- responseTime).
---
--- Called via ngx.timer.at from site_logger (premature is first arg).
-
-local cjson = require "cjson.safe"
-
-local SUB_PREFIX   = "log_pipe:sub:"
-local QUEUE_PREFIX = "log_pipe:q:"
-local QUEUE_MAX_LEN = 2000
-
--- SAFE LOAD: GeoIP (per-worker, cached by require)
-local geo_ok, geo = pcall(require, "openship.geo_country")
-if not geo_ok then geo = nil end
-
--- Reused payload table per worker (cleared after encode to avoid GC pressure)
-local payload = {}
-
--- `cc` is the country site_logger already resolved for this request. Passed in rather
--- than looked up again: one mmdb hit per request, and the live frame can never disagree
--- with the rollup or the ring buffer about where a hit came from.
-local function pipe_request_log(premature, host, ip, ts, ua, uri,
-                                req_len, bytes, rt, method, status, cc)
-    if premature then return end
-
-    local sh = ngx.shared.request_data
-    if not sh then return end
-
-    -- Double-check subscriber is still active (guard against race)
-    if not sh:get(SUB_PREFIX .. host) then return end
-
-    pcall(function()
-        -- Falls back to its own lookup only if the caller passed nothing, which happens
-        -- when a box is mid-deploy and still has the previous site_logger.
-        local country = cc
-        if (not country or country == "") and geo and geo.get_country_code then
-            local ok_cc, res = pcall(geo.get_country_code, ip)
-            if ok_cc then country = res end
-        end
-
-        payload.id           = string.format("%.3f-%d", ts, math.random(10000, 99999))
-        payload.host         = host
-        payload.ip           = ip
-        if country and type(country) == "string" and country ~= "" then
-            payload.country = country
-        end
-        payload.timestamp    = ts
-        payload.date         = os.date("!%Y-%m-%d %H:%M:%S", ts)
-        payload.uri          = uri
-        payload.method       = method or "GET"
-        payload.status       = tonumber(status) or 0
-        payload.userAgent    = ua or ""
-        payload.requestSize  = tonumber(req_len) or 0
-        payload.responseSize = tonumber(bytes) or 0
-        payload.responseTime = tonumber(rt) or 0
-
-        local body = cjson.encode(payload)
-
-        -- Clear payload table for next reuse
-        for k in pairs(payload) do payload[k] = nil end
-
-        if not body then return end
-
-        local qkey = QUEUE_PREFIX .. host
-        sh:lpush(qkey, body)
-        if sh:llen(qkey) > QUEUE_MAX_LEN then
-            sh:rpop(qkey)
-        end
-    end)
-end
-
-return {
-    pipe_request_log = pipe_request_log,
-}