openship 0.1.11 → 0.2.0

package/dist/server/lua/geo_country.lua

@@ -0,0 +1,101 @@
+-- geo_country.lua
+-- MaxMind GeoLite2-Country lookup with per-worker LRU cache.
+--
+-- DB and cache are per-worker only: one load per worker, shared by all
+-- requests. No reload per stream or per request.
+--
+-- Dependencies (installed by deployLuaScripts):
+--   opm: anjia0532/lua-resty-maxminddb
+--   apt: libmaxminddb0 libmaxminddb-dev  (C library for FFI)
+--   Database: /usr/share/GeoIP/GeoLite2-Country.mmdb
+--
+-- Falls back gracefully - if the library or database is missing,
+-- get_country_code() returns nil.  Both site_logger.lua and pipe_log.lua
+-- wrap this module in pcall(require, ...) so a missing DB never crashes
+-- the request pipeline.
+
+local _M = {}
+
+local DB_PATH = "/usr/share/GeoIP/GeoLite2-Country.mmdb"
+
+local mmdb_ok, maxmind = pcall(require, "resty.maxminddb")
+local lrucache_ok, lrucache = pcall(require, "resty.lrucache")
+
+local cache            -- one LRU per worker
+local geo_initted = false  -- DB opened once per worker on first lookup
+
+-- Init MaxMind once per worker; subsequent lookups (any stream) reuse.
+local function ensure_geo_init()
+    if geo_initted then
+        return true
+    end
+    if not mmdb_ok or not maxmind then
+        return false
+    end
+    local ok, err = maxmind.init(DB_PATH)
+    if not ok then
+        ngx.log(ngx.WARN,
+            "[geo_country] GeoLite2 database not found at " .. DB_PATH ..
+            " - geo lookups disabled: " .. (err or "unknown"))
+        return false
+    end
+    geo_initted = true
+    ngx.log(ngx.INFO, "[geo_country] GeoLite2-Country loaded from " .. DB_PATH)
+    return true
+end
+
+-- Create LRU cache once per worker (max 50000 IPs).
+local function get_cache()
+    if cache then
+        return cache
+    end
+    if not lrucache_ok or not lrucache then
+        return nil
+    end
+    local c, err = lrucache.new(50000)
+    if not c then
+        return nil
+    end
+    cache = c
+    return cache
+end
+
+--- Get country ISO code for IP. Uses in-memory LRU cache then MaxMind DB.
+-- @param ip string (e.g. "8.8.8.8")
+-- @return string|nil country code (e.g. "US") or nil
+function _M.get_country_code(ip)
+    if not ip or ip == "" then
+        return nil
+    end
+
+    local c = get_cache()
+    if c then
+        local cached = c:get(ip)
+        if cached ~= nil then
+            -- false means "looked up, no result" - avoids repeated DB misses
+            return (cached == false) and nil or cached
+        end
+    end
+
+    if not ensure_geo_init() then
+        if c then c:set(ip, false) end
+        return nil
+    end
+
+    local ok, res, err = pcall(maxmind.lookup, ip)
+    local code = nil
+    if ok and res and res.country and res.country.iso_code then
+        code = res.country.iso_code
+    end
+
+    if c then
+        c:set(ip, code or false)
+    end
+    return code
+end
+
+function _M.ensure_geo_init()
+    return ensure_geo_init()
+end
+
+return _M

package/dist/server/lua/mgmt_api.lua

@@ -0,0 +1,291 @@
+-- mgmt_api.lua
+-- REST analytics endpoints backed by ngx.shared.dict.
+-- Internal management port only (127.0.0.1:9145).
+
+local cjson = require "cjson.safe"
+
+local analytics    = ngx.shared.analytics
+local request_data = ngx.shared.request_data
+local uri          = ngx.var.uri
+
+local function json(data, code)
+    ngx.status = code or 200
+    ngx.header["Content-Type"] = "application/json"
+    ngx.say(cjson.encode(data))
+    return ngx.exit(ngx.status)
+end
+
+local function bad(msg)
+    return json({ error = msg }, 400)
+end
+
+-- ── GET /health ──────────────────────────────────────────────────────────────
+
+if uri == "/health" then
+    ngx.say("ok")
+    return ngx.exit(200)
+end
+
+-- ── /rules - per-route rules cache (written by the API, read by rules_guard) ──
+-- POST /rules  body: { "host": "...", "rules": [ { "pathPrefix": "/", "spec": {...} } ] }
+--   → replaces the ruleset for that host in the `rules` shared dict (reload-free).
+--   An empty/absent `rules` array clears the host.
+-- GET  /rules?host=...   → the stored ruleset (debug).
+-- DELETE /rules?host=... → clear the host's ruleset.
+-- Handled BEFORE the analytics guard: rules use their own dict.
+if uri == "/rules" then
+    local rules = ngx.shared.rules
+    if not rules then return json({ error = "rules dict unavailable" }, 503) end
+    local method = ngx.req.get_method()
+
+    if method == "GET" then
+        local host = ngx.var.arg_host
+        if not host or host == "" then return bad("missing ?host=") end
+        host = host:lower()
+        local raw = rules:get(host)
+        return json({ host = host, rules = raw and cjson.decode(raw) or {} })
+    end
+
+    if method == "DELETE" then
+        local host = ngx.var.arg_host
+        if not host or host == "" then return bad("missing ?host=") end
+        rules:delete(host:lower())
+        return json({ ok = true })
+    end
+
+    if method == "POST" then
+        ngx.req.read_body()
+        local body = ngx.req.get_body_data()
+        if not body then return bad("empty body") end
+        local payload = cjson.decode(body)
+        if type(payload) ~= "table" or not payload.host then
+            return bad("expected { host, rules: [...] }")
+        end
+        local host = tostring(payload.host):lower()
+        local list = payload.rules
+        if type(list) ~= "table" or #list == 0 then
+            rules:delete(host)                 -- empty set = clear the host
+            return json({ ok = true, host = host, count = 0 })
+        end
+        local ok, err = rules:set(host, cjson.encode(list))
+        if not ok then return json({ error = "set failed: " .. (err or "?") }, 500) end
+        return json({ ok = true, host = host, count = #list })
+    end
+
+    return json({ error = "method not allowed" }, 405)
+end
+
+if not analytics then
+    return json({ error = "analytics dict unavailable" }, 503)
+end
+
+-- ── GET /analytics - minute-bucket time series ───────────────────────────────
+-- ?domain=example.com&from=EPOCH_MIN&to=EPOCH_MIN
+
+if uri == "/analytics" then
+    local domain = ngx.var.arg_domain
+    if not domain or domain == "" then return bad("missing ?domain=") end
+    domain = domain:lower()
+
+    local from_m = tonumber(ngx.var.arg_from)
+    local to_m   = tonumber(ngx.var.arg_to)
+    if not from_m or not to_m then
+        return bad("missing ?from= and ?to= (epoch minutes)")
+    end
+    -- Cap at 24h
+    if to_m - from_m > 1440 then to_m = from_m + 1440 end
+
+    local buckets = {}
+    for m = from_m, to_m do
+        local p = "s:" .. domain .. ":" .. m
+        local r = analytics:get(p .. ":r")
+        if r then
+            -- Response time stored as microseconds, convert to seconds (float)
+            local rt_us = analytics:get(p .. ":t") or 0
+            local bucket = {
+                minute           = m,
+                requests         = r,
+                unique_requests  = analytics:get(p .. ":u") or 0,
+                bandwidth_in     = analytics:get(p .. ":i") or 0,
+                bandwidth_out    = analytics:get(p .. ":o") or 0,
+                response_time    = rt_us / 1000000,
+            }
+
+            -- Collect per-minute country data if present
+            local cpfx = "c:" .. domain .. ":" .. m .. ":"
+            local all_keys = analytics:get_keys(10000)
+            local countries = {}
+            local has_countries = false
+            for _, k in ipairs(all_keys) do
+                if k:sub(1, #cpfx) == cpfx then
+                    countries[k:sub(#cpfx + 1)] = analytics:get(k) or 0
+                    has_countries = true
+                end
+            end
+            if has_countries then
+                bucket.countries = countries
+            end
+
+            buckets[#buckets + 1] = bucket
+        end
+    end
+
+    return json({ domain = domain, buckets = buckets })
+end
+
+-- ── POST /analytics/flush - read + delete minute buckets ─────────────────────
+-- Same as GET /analytics but deletes the returned buckets from shared memory.
+-- Used by the scraper to atomically move data from OpenResty → DB.
+-- ?domain=example.com&from=EPOCH_MIN&to=EPOCH_MIN
+
+if uri == "/analytics/flush" and ngx.req.get_method() == "POST" then
+    local domain = ngx.var.arg_domain
+    if not domain or domain == "" then return bad("missing ?domain=") end
+    domain = domain:lower()
+
+    local from_m = tonumber(ngx.var.arg_from)
+    local to_m   = tonumber(ngx.var.arg_to)
+    if not from_m or not to_m then
+        return bad("missing ?from= and ?to= (epoch minutes)")
+    end
+    if to_m - from_m > 1440 then to_m = from_m + 1440 end
+
+    local buckets = {}
+    local flushed = 0
+    for m = from_m, to_m do
+        local p = "s:" .. domain .. ":" .. m
+        local r = analytics:get(p .. ":r")
+        if r then
+            local rt_us = analytics:get(p .. ":t") or 0
+            local bucket = {
+                minute           = m,
+                requests         = r,
+                unique_requests  = analytics:get(p .. ":u") or 0,
+                bandwidth_in     = analytics:get(p .. ":i") or 0,
+                bandwidth_out    = analytics:get(p .. ":o") or 0,
+                response_time    = rt_us / 1000000,
+            }
+
+            -- Collect per-minute country data
+            local cpfx = "c:" .. domain .. ":" .. m .. ":"
+            local all_keys = analytics:get_keys(10000)
+            local countries = {}
+            local has_countries = false
+            for _, k in ipairs(all_keys) do
+                if k:sub(1, #cpfx) == cpfx then
+                    countries[k:sub(#cpfx + 1)] = analytics:get(k) or 0
+                    analytics:delete(k)
+                    has_countries = true
+                end
+            end
+            if has_countries then bucket.countries = countries end
+
+            buckets[#buckets + 1] = bucket
+
+            -- Delete the minute-bucket counter keys
+            analytics:delete(p .. ":r")
+            analytics:delete(p .. ":i")
+            analytics:delete(p .. ":o")
+            analytics:delete(p .. ":t")
+            analytics:delete(p .. ":u")
+            flushed = flushed + 1
+        end
+    end
+
+    return json({ domain = domain, buckets = buckets, flushed = flushed })
+end
+
+-- ── GET /analytics/totals - lifetime counters ────────────────────────────────
+-- ?domain=example.com      → single domain
+-- (no domain)              → all known domains
+
+if uri == "/analytics/totals" then
+    local domain = ngx.var.arg_domain
+
+    if not domain or domain == "" then
+        local keys = analytics:get_keys(10000)
+        local domains = {}
+        for _, k in ipairs(keys) do
+            local d = k:match("^d:(.+)$")
+            if d then
+                domains[#domains + 1] = {
+                    domain        = d,
+                    requests      = analytics:get("t:" .. d .. ":r") or 0,
+                    bandwidth_in  = analytics:get("t:" .. d .. ":i") or 0,
+                    bandwidth_out = analytics:get("t:" .. d .. ":o") or 0,
+                }
+            end
+        end
+        return json({ domains = domains })
+    end
+
+    domain = domain:lower()
+    return json({
+        domain        = domain,
+        requests      = analytics:get("t:" .. domain .. ":r") or 0,
+        bandwidth_in  = analytics:get("t:" .. domain .. ":i") or 0,
+        bandwidth_out = analytics:get("t:" .. domain .. ":o") or 0,
+    })
+end
+
+-- ── GET /analytics/geo - country breakdown for a day ─────────────────────────
+-- ?domain=example.com&day=YYYYMMDD
+
+if uri == "/analytics/geo" then
+    local domain = ngx.var.arg_domain
+    if not domain or domain == "" then return bad("missing ?domain=") end
+    domain = domain:lower()
+
+    local day = ngx.var.arg_day
+    if not day or #day ~= 8 then
+        day = os.date("!%Y%m%d")
+    end
+
+    local prefix = "g:" .. domain .. ":" .. day .. ":"
+    local keys = analytics:get_keys(10000)
+    local countries = {}
+    for _, k in ipairs(keys) do
+        if k:sub(1, #prefix) == prefix then
+            countries[k:sub(#prefix + 1)] = analytics:get(k) or 0
+        end
+    end
+
+    return json({ domain = domain, day = day, countries = countries })
+end
+
+-- ── GET /logs/recent - raw request ring buffer ───────────────────────────────
+-- ?domain=example.com&limit=50
+
+if uri == "/logs/recent" then
+    if not request_data then
+        return json({ error = "request_data dict unavailable" }, 503)
+    end
+
+    local domain = ngx.var.arg_domain
+    if not domain or domain == "" then return bad("missing ?domain=") end
+    domain = domain:lower()
+
+    local limit = math.min(tonumber(ngx.var.arg_limit) or 50, 1000)
+    local seq   = request_data:get("rlog:" .. domain .. ":seq") or 0
+    local out   = {}
+
+    for i = 0, limit - 1 do
+        local slot = (seq - i) % 1000
+        if slot < 0 then slot = slot + 1000 end
+        local raw = request_data:get("rlog:" .. domain .. ":" .. slot)
+        if raw then out[#out + 1] = raw end
+    end
+
+    -- Return pre-encoded JSON entries as a JSON array
+    ngx.header["Content-Type"] = "application/json"
+    ngx.print("[")
+    for idx, entry in ipairs(out) do
+        if idx > 1 then ngx.print(",") end
+        ngx.print(entry)
+    end
+    ngx.say("]")
+    return ngx.exit(200)
+end
+
+-- ── 404 ──────────────────────────────────────────────────────────────────────
+return json({ error = "not found" }, 404)

package/dist/server/lua/pipe_log.lua

@@ -0,0 +1,71 @@
+-- pipe_log.lua
+-- Push request log into shared dict queue log_pipe:q:{host} for /stream SSE.
+-- Payload shape matches requestLogger.js (id, ip, country when known,
+-- timestamp, date, uri, method, status, userAgent, requestSize, responseSize,
+-- responseTime).
+--
+-- Called via ngx.timer.at from site_logger (premature is first arg).
+
+local cjson = require "cjson.safe"
+
+local SUB_PREFIX   = "log_pipe:sub:"
+local QUEUE_PREFIX = "log_pipe:q:"
+local QUEUE_MAX_LEN = 2000
+
+-- SAFE LOAD: GeoIP (per-worker, cached by require)
+local geo_ok, geo = pcall(require, "openship.geo_country")
+if not geo_ok then geo = nil end
+
+-- Reused payload table per worker (cleared after encode to avoid GC pressure)
+local payload = {}
+
+local function pipe_request_log(premature, host, ip, ts, ua, uri,
+                                req_len, bytes, rt, method, status)
+    if premature then return end
+
+    local sh = ngx.shared.request_data
+    if not sh then return end
+
+    -- Double-check subscriber is still active (guard against race)
+    if not sh:get(SUB_PREFIX .. host) then return end
+
+    pcall(function()
+        local country = nil
+        if geo and geo.get_country_code then
+            country = geo.get_country_code(ip)
+        end
+
+        payload.id           = string.format("%.3f-%d", ts, math.random(10000, 99999))
+        payload.host         = host
+        payload.ip           = ip
+        if country and type(country) == "string" and country ~= "" then
+            payload.country = country
+        end
+        payload.timestamp    = ts
+        payload.date         = os.date("!%Y-%m-%d %H:%M:%S", ts)
+        payload.uri          = uri
+        payload.method       = method or "GET"
+        payload.status       = tonumber(status) or 0
+        payload.userAgent    = ua or ""
+        payload.requestSize  = tonumber(req_len) or 0
+        payload.responseSize = tonumber(bytes) or 0
+        payload.responseTime = tonumber(rt) or 0
+
+        local body = cjson.encode(payload)
+
+        -- Clear payload table for next reuse
+        for k in pairs(payload) do payload[k] = nil end
+
+        if not body then return end
+
+        local qkey = QUEUE_PREFIX .. host
+        sh:lpush(qkey, body)
+        if sh:llen(qkey) > QUEUE_MAX_LEN then
+            sh:rpop(qkey)
+        end
+    end)
+end
+
+return {
+    pipe_request_log = pipe_request_log,
+}

package/dist/server/lua/pipe_stream.lua

@@ -0,0 +1,90 @@
+-- pipe_stream.lua
+-- content_by_lua: SSE endpoint for real-time request log streaming.
+-- GET /logs/stream?domain=example.com
+-- Internal only - 127.0.0.1:9145
+
+local sh = ngx.shared.request_data
+if not sh then
+    ngx.status = 503
+    ngx.say("shared dict unavailable")
+    return ngx.exit(503)
+end
+
+local domain = ngx.var.arg_domain
+if not domain or domain == "" then
+    ngx.status = 400
+    ngx.say("missing ?domain= parameter")
+    return ngx.exit(400)
+end
+
+domain = domain:lower()
+if domain:sub(1, 4) == "www." then domain = domain:sub(5) end
+
+local SUB_KEY   = "log_pipe:sub:" .. domain
+local QUEUE_KEY = "log_pipe:q:"   .. domain
+
+-- Clear stale queue entries, mark subscriber active
+sh:delete(QUEUE_KEY)
+sh:set(SUB_KEY, true, 30)
+
+ngx.header["Content-Type"]      = "text/event-stream"
+ngx.header["Cache-Control"]     = "no-cache, no-store"
+ngx.header["Connection"]        = "keep-alive"
+ngx.header["X-Accel-Buffering"] = "no"
+
+-- Send initial SSE comment so the first flush has data.
+-- Without this, ngx.flush on an empty buffer sends chunked-EOF (0\r\n\r\n)
+-- and terminates the response immediately.
+ngx.print(": connected\n\n")
+if not ngx.flush(true) then
+    sh:delete(SUB_KEY)
+    return
+end
+
+local started  = ngx.now()
+local last_hb  = started
+local last_ref = started
+
+while true do
+    -- Max 1 hour per connection
+    if ngx.now() - started > 3600 then
+        sh:delete(SUB_KEY)
+        return
+    end
+
+    -- Drain up to 100 queued entries per cycle
+    local sent = 0
+    while sent < 100 do
+        local entry = sh:rpop(QUEUE_KEY)
+        if not entry then break end
+        ngx.print("event: request\ndata: ", entry, "\n\n")
+        sent = sent + 1
+    end
+
+    if sent > 0 then
+        if not ngx.flush(true) then
+            sh:delete(SUB_KEY)
+            return
+        end
+    end
+
+    local now = ngx.now()
+
+    -- Heartbeat every 15s
+    if now - last_hb > 15 then
+        ngx.print(": ping\n\n")
+        if not ngx.flush(true) then
+            sh:delete(SUB_KEY)
+            return
+        end
+        last_hb = now
+    end
+
+    -- Refresh subscriber TTL every 10s
+    if now - last_ref > 10 then
+        sh:set(SUB_KEY, true, 30)
+        last_ref = now
+    end
+
+    ngx.sleep(0.05)
+end

package/dist/server/lua/rules_guard.lua

@@ -0,0 +1,118 @@
+-- rules_guard.lua
+-- access_by_lua driver: enforce per-route rules (method · IP/CIDR allow-deny ·
+-- country allow/ban · bad user-agent · hotlink · rate-limit) for the request's
+-- host. Rules are pushed reload-free by mgmt_api `POST /rules`; the DB
+-- `route_rule` table is the source of truth. All parsing/compiling/matching
+-- lives in openship.rules_lib (loaded ONCE per worker) — this file is re-run
+-- every request so it stays minimal. See rules_lib.lua for the compiled shape.
+--
+-- Enforcement uses the connecting peer (ngx.var.remote_addr); front a real_ip
+-- config if behind a trusted proxy. No user-supplied value is ever used as a
+-- Lua pattern or written into a response header.
+
+local rules = ngx.shared.rules
+if not rules then return end
+
+local host = ngx.var.host
+if not host then return end
+
+local raw = rules:get(host)
+if not raw then return end          -- fast path: nothing configured for this host
+
+local lib = require "openship.rules_lib"
+local entries = lib.get(host, raw)
+local n = #entries
+if n == 0 then return end
+
+-- ── Longest matching pathPrefix wins ──
+local uri = ngx.var.uri or "/"
+local chosen, chosen_len = nil, -1
+for i = 1, n do
+    local e = entries[i]
+    local p = e.pathPrefix
+    if p == nil or p == "" or p == "/" then
+        if chosen_len < 0 then chosen, chosen_len = e, 0 end
+    elseif uri == p or string.sub(uri, 1, #p) == p then
+        if #p > chosen_len then chosen, chosen_len = e, #p end
+    end
+end
+if not chosen then return end
+
+local spec = chosen.spec
+local deny_status = spec.blockStatus or 403
+
+local ips = ngx.var.remote_addr or "0.0.0.0"
+local ipi = lib.ipv4_to_int(ips)
+
+-- Country resolved at most once per request (only when a country rule exists).
+local country_cache
+local function country_of()
+    if country_cache == nil then
+        local ok, geo = pcall(require, "openship.geo_country")
+        country_cache = (ok and geo and geo.get_country_code(ips)) or false
+    end
+    return country_cache or nil
+end
+
+-- ── 1. Access: method allow-list, IP allow/deny, country allow-list ──
+local access = spec.access
+if access then
+    if access.methods and not access.methods[ngx.req.get_method()] then
+        return ngx.exit(deny_status)
+    end
+    if lib.match_compiled(access.deny, ipi, ips) then return ngx.exit(deny_status) end
+    if access.allow and not lib.match_compiled(access.allow, ipi, ips) then
+        return ngx.exit(deny_status)
+    end
+    -- Allow-list: an unresolved country is not on the list → blocked (default-deny).
+    if access.allowCountries and not access.allowCountries[country_of() or ""] then
+        return ngx.exit(deny_status)
+    end
+end
+
+-- ── 2. Ban: IP/CIDR, country, user-agent ──
+local ban = spec.ban
+if ban then
+    if lib.match_compiled(ban.ips, ipi, ips) or lib.match_compiled(ban.cidrs, ipi, ips) then
+        return ngx.exit(deny_status)
+    end
+    if ban.countries then
+        local cc = country_of()
+        if cc and ban.countries[cc] then return ngx.exit(deny_status) end
+    end
+    if ban.emptyUA or ban.uas then
+        local ua = ngx.var.http_user_agent
+        if ban.emptyUA and (ua == nil or ua == "") then return ngx.exit(deny_status) end
+        if ban.uas and ua and ua ~= "" then
+            local ual = string.lower(ua)
+            for i = 1, #ban.uas do
+                if string.find(ual, ban.uas[i], 1, true) then return ngx.exit(deny_status) end
+            end
+        end
+    end
+end
+
+-- ── 3. Hotlink: only listed referer hosts (empty referer per allowEmpty) ──
+local hot = spec.hotlink
+if hot then
+    local ref = ngx.var.http_referer
+    if ref == nil or ref == "" then
+        if not hot.allowEmpty then return ngx.exit(deny_status) end
+    else
+        local rhost = string.match(ref, "^%w+://([^/]+)")          -- authority
+        if rhost then rhost = string.lower(string.match(rhost, "^([^:]+)") or rhost) end -- strip :port
+        if not (rhost and hot.referers[rhost]) then return ngx.exit(deny_status) end
+    end
+end
+
+-- ── 4. Rate limit — fixed 1s window per (host, path, ip) ──
+local rl = spec.rl
+if rl then
+    local key = "rl:" .. host .. ":" .. (chosen.pathPrefix or "/") .. ":" .. ips
+        .. ":" .. math.floor(ngx.now())
+    local c = rules:incr(key, 1, 0, 2)  -- init 0, 2s TTL → self-expiring buckets
+    if c and c > rl.limit then
+        ngx.header["Retry-After"] = "1"
+        return ngx.exit(rl.status)
+    end
+end