analytics-platform 0.0.0-stage → 0.2.2

package/lib/inventory.js

@@ -0,0 +1,142 @@
+// Software inventory module (protocol in sdk/protocols/inventory.md): the
+// packages the app runs (its package-lock.json, production packages only)
+// and its Node version, sent to POST /events/inventory a minute after start
+// and then once a day (the collector's `every`), so the Security check can
+// match them against known security holes. Read in the background, never in
+// a request; names and versions only.
+"use strict";
+
+const fs = require("node:fs");
+const os = require("node:os");
+const path = require("node:path");
+
+const FIRST_MS = 60 * 1000;
+const RETRY_S = 60 * 60;
+const EVERY_S = 24 * 60 * 60;
+const MAX_PACKAGES = 5000;
+const LOCK_FILES = ["package-lock.json", "npm-shrinkwrap.json"];
+
+// The packages the app itself asks for, from its package.json: the ones the
+// owner chose, as opposed to the hundreds those depend on. Production only,
+// to match the lock file. An empty set means we couldn't tell.
+function directNames(manifest) {
+  const names = new Set();
+  for (const key of ["dependencies", "optionalDependencies"]) {
+    const deps = manifest?.[key];
+    if (deps && typeof deps === "object") for (const name of Object.keys(deps)) names.add(name);
+  }
+  return names;
+}
+
+// A lock file -> [{ ecosystem, name, version }]: lockfile v2/v3 "packages"
+// (the name from the last node_modules/ in the path, or the entry's own for
+// aliases), else v1 "dependencies"; dev packages and links left out.
+// `direct` marks the ones package.json asks for by name.
+function lockPackages(lock, direct = new Set()) {
+  const found = new Map();
+  const add = (name, version) => {
+    if (typeof name !== "string" || !name || typeof version !== "string") return;
+    const clean = version.replace(/^v/, "");
+    // file:, git and tarball versions aren't releases.
+    if (!/^\d/.test(clean)) return;
+    const row = { ecosystem: "npm", name, version: clean };
+    if (direct.has(name)) row.direct = true;
+    found.set(`${name}@${clean}`, row);
+  };
+  if (lock?.packages && typeof lock.packages === "object") {
+    for (const [key, entry] of Object.entries(lock.packages)) {
+      const at = key.lastIndexOf("node_modules/");
+      // "" is the app itself; paths without node_modules/ are workspaces.
+      if (at === -1 || !entry || entry.dev || entry.link) continue;
+      add(entry.name || key.slice(at + "node_modules/".length), entry.version);
+    }
+  } else if (lock?.dependencies && typeof lock.dependencies === "object") {
+    const walk = (deps) => {
+      for (const [name, entry] of Object.entries(deps)) {
+        if (!entry || entry.dev) continue;
+        add(name, entry.version);
+        if (entry.dependencies && typeof entry.dependencies === "object") walk(entry.dependencies);
+      }
+    };
+    walk(lock.dependencies);
+  }
+  return [...found.values()];
+}
+
+// Options: dir (the folder with package-lock.json; default the working
+// directory), host (the server's name; default the host name).
+function createInventory(core, options = {}) {
+  const log = (...args) => core.log("inventory", ...args);
+  const dir = String(options.dir || process.cwd());
+  const host = String(options.host ?? os.hostname()).replace(/[^A-Za-z0-9._-]/g, "-").slice(0, 100) || "server";
+  let timer = null;
+  let closed = false;
+
+  // package.json's own dependencies, for `direct`. Missing or unreadable is
+  // not an error: the packages then go up unmarked.
+  async function direct() {
+    try {
+      return directNames(JSON.parse(await fs.promises.readFile(path.join(dir, "package.json"), "utf8")));
+    } catch (error) {
+      if (error.code !== "ENOENT") log("could not read package.json:", error.message);
+      return new Set();
+    }
+  }
+
+  async function packages() {
+    const chosen = await direct();
+    for (const name of LOCK_FILES) {
+      try {
+        return lockPackages(JSON.parse(await fs.promises.readFile(path.join(dir, name), "utf8")), chosen);
+      } catch (error) {
+        if (error.code !== "ENOENT") log(`could not read ${name}:`, error.message);
+      }
+    }
+    // No lock file: the Node version alone.
+    return [];
+  }
+
+  async function report() {
+    let wait = RETRY_S;
+    try {
+      if (core.isOn("inventory")) {
+        const list = await packages();
+        const res = await core.send("inventory", "/events/inventory", {
+          auth: "bearer",
+          body: {
+            site_id: core.siteId,
+            host,
+            at: new Date().toISOString(),
+            runtime: { node: process.versions.node },
+            packages: list.slice(0, MAX_PACKAGES),
+            truncated: list.length > MAX_PACKAGES,
+          },
+        });
+        if (res?.ok) {
+          const data = await res.json().catch(() => null);
+          wait = Number(data?.every) > 0 ? Math.max(RETRY_S, Number(data.every)) : EVERY_S;
+        }
+      }
+    } catch (error) {
+      log("could not send the inventory:", error?.message);
+    }
+    schedule(wait * 1000);
+  }
+
+  function schedule(ms) {
+    if (closed) return;
+    timer = setTimeout(report, ms);
+    timer.unref?.();
+  }
+
+  if (core.valid) schedule(FIRST_MS);
+
+  return {
+    close() {
+      closed = true;
+      clearTimeout(timer);
+    },
+  };
+}
+
+module.exports = { createInventory, lockPackages, directNames };

package/lib/ip.js

@@ -0,0 +1,104 @@
+// Addresses and ranges, with no Node built-ins so the edge runtime (Next.js
+// middleware) can use it too. IPv4 is a number, IPv6 a BigInt; IPv4-mapped
+// IPv6 (::ffff:1.2.3.4) is IPv4.
+"use strict";
+
+const V4 = /^(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(?:\.(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}$/;
+const HEX = /^[0-9a-f]{1,4}$/i;
+
+function v4(text) {
+  let n = 0;
+  for (const part of text.split(".")) n = n * 256 + Number(part);
+  return n;
+}
+
+// The 8 groups of an IPv6 address, or null when it isn't one.
+function v6Groups(text) {
+  let s = text.split("%")[0];
+  const cut = s.lastIndexOf(":");
+  if (cut < 0) return null;
+  const tail = s.slice(cut + 1);
+  if (tail.includes(".")) {
+    if (!V4.test(tail)) return null;
+    const n = v4(tail);
+    s = `${s.slice(0, cut + 1)}${Math.floor(n / 65536).toString(16)}:${(n % 65536).toString(16)}`;
+  }
+  const halves = s.split("::");
+  if (halves.length > 2) return null;
+  const left = halves[0] ? halves[0].split(":") : [];
+  const right = halves.length === 2 && halves[1] ? halves[1].split(":") : [];
+  if (![...left, ...right].every((g) => HEX.test(g))) return null;
+  if (halves.length === 1) return left.length === 8 ? left : null;
+  const fill = 8 - left.length - right.length;
+  return fill >= 1 ? [...left, ...Array(fill).fill("0"), ...right] : null;
+}
+
+function parseIp(text) {
+  const s = String(text || "").trim();
+  if (V4.test(s)) return { family: 4, n: v4(s) };
+  const groups = v6Groups(s);
+  if (!groups) return null;
+  let n = 0n;
+  for (const group of groups) n = (n << 16n) | BigInt(parseInt(group, 16));
+  if (n >> 32n === 0xffffn) return { family: 4, n: Number(n & 0xffffffffn) };
+  return { family: 6, n };
+}
+
+// "1.2.3.4", "1.2.3.0/24", "2001:db8::/32" -> { family, start, end }, or null.
+function parseRange(entry) {
+  const [addr, bits, extra] = String(entry || "").trim().split("/");
+  if (extra !== undefined || (bits !== undefined && !/^\d{1,3}$/.test(bits))) return null;
+  const ip = parseIp(addr);
+  if (!ip) return null;
+  const max = ip.family === 4 ? 32 : 128;
+  let prefix = bits === undefined ? max : Number(bits);
+  if (ip.family === 4 && addr.includes(":") && bits !== undefined) prefix -= 96;
+  if (prefix < 0 || prefix > max) return null;
+  if (ip.family === 4) {
+    const size = 2 ** (32 - prefix);
+    const start = Math.floor(ip.n / size) * size;
+    return { family: 4, start, end: start + size - 1 };
+  }
+  const host = (1n << BigInt(128 - prefix)) - 1n;
+  return { family: 6, start: ip.n & ~host, end: (ip.n & ~host) | host };
+}
+
+// Addresses and ranges, merged and sorted, looked up by binary search.
+class IpSet {
+  constructor(entries = []) {
+    const ranges = { 4: [], 6: [] };
+    for (const entry of entries) {
+      const range = parseRange(entry);
+      if (range) ranges[range.family].push(range);
+    }
+    this[4] = IpSet.merge(ranges[4], 1);
+    this[6] = IpSet.merge(ranges[6], 1n);
+  }
+
+  static merge(ranges, one) {
+    ranges.sort((a, b) => (a.start < b.start ? -1 : a.start > b.start ? 1 : 0));
+    const merged = [];
+    for (const r of ranges) {
+      const last = merged[merged.length - 1];
+      if (last && r.start <= last.end + one) {
+        if (r.end > last.end) last.end = r.end;
+      } else merged.push({ start: r.start, end: r.end });
+    }
+    return merged;
+  }
+
+  has(ip) {
+    const list = this[ip.family];
+    let lo = 0;
+    let hi = list.length - 1;
+    while (lo <= hi) {
+      const mid = (lo + hi) >> 1;
+      if (list[mid].start > ip.n) hi = mid - 1;
+      else if (list[mid].end < ip.n) lo = mid + 1;
+      else return true;
+    }
+    return false;
+  }
+}
+
+module.exports = { parseIp, parseRange, IpSet };

package/lib/maintenance.js

@@ -0,0 +1,81 @@
+// Maintenance mode module: while it's on (Security → Server package), visitors
+// get a "back soon" page (503, so search engines know it's temporary),
+// except allowed addresses and whoever opened the preview link
+// (?site_preview=<code>, remembered for a day in a cookie).
+"use strict";
+
+const { parseIp, IpSet } = require("./ip.js");
+
+const COOKIE = "site_preview";
+const escape = (text) => String(text).replace(/[&<>"']/g, (c) => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;", "'": "&#39;" }[c]));
+
+function page(message) {
+  return `<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
+<title>Back soon</title><style>body{margin:0;min-height:100vh;display:grid;place-items:center;font:16px/1.5 system-ui,sans-serif;background:#f8fafc;color:#0f172a}
+main{max-width:32rem;padding:2rem;text-align:center}h1{font-size:1.6rem;margin:0 0 .5rem}p{color:#475569;margin:0}
+@media(prefers-color-scheme:dark){body{background:#0f172a;color:#e2e8f0}p{color:#94a3b8}}</style></head>
+<body><main><h1>Back soon</h1><p>${escape(message || "We're doing some maintenance. Please check back in a few minutes.")}</p></main></body></html>`;
+}
+
+function cookieOf(req, name) {
+  const header = String(req.headers?.cookie || "");
+  for (const part of header.split(";")) {
+    const [k, ...v] = part.trim().split("=");
+    if (k !== name) continue;
+    // A malformed value can't be the preview code, and mustn't throw: the
+    // middleware lets a request through when its check fails.
+    try {
+      return decodeURIComponent(v.join("="));
+    } catch {
+      return "";
+    }
+  }
+  return "";
+}
+
+function queryOf(req, name) {
+  try {
+    return new URL(req.originalUrl || req.url || "/", "http://x").searchParams.get(name);
+  } catch {
+    return null;
+  }
+}
+
+// clientIp(req): the firewall's way of finding the visitor's address.
+// Options: skip(req) (true lets a request through, e.g. health checks).
+function createMaintenance(core, { clientIp, skip } = {}) {
+  let allowFor = null;
+  let allowSet = new IpSet([]);
+
+  function middleware(req, res, next = () => {}) {
+    try {
+      const m = core.isOn("maintenance") ? core.maintenance() : null;
+      if (!m?.on || skip?.(req)) return next();
+      if (allowFor !== m.allow) {
+        allowFor = m.allow;
+        allowSet = new IpSet(m.allow || []);
+      }
+      const ip = parseIp(clientIp ? clientIp(req) : req.ip || req.socket?.remoteAddress || "");
+      if (ip && allowSet.has(ip)) return next();
+      if (m.bypass) {
+        if (queryOf(req, COOKIE) === m.bypass) {
+          res.setHeader("set-cookie", `${COOKIE}=${encodeURIComponent(m.bypass)}; Path=/; Max-Age=86400; HttpOnly; SameSite=Lax`);
+          return next();
+        }
+        if (cookieOf(req, COOKIE) === m.bypass) return next();
+      }
+      res.statusCode = 503;
+      res.setHeader("retry-after", "600");
+      res.setHeader("cache-control", "no-store");
+      res.setHeader("content-type", "text/html; charset=utf-8");
+      res.end(page(m.message));
+    } catch (error) {
+      core.log("maintenance", "check failed, request passed:", error?.message);
+      next();
+    }
+  }
+
+  return { middleware };
+}
+
+module.exports = { createMaintenance, page };

package/lib/next.js

@@ -0,0 +1,251 @@
+// Next.js (and anything else on Vercel or another edge runtime): the firewall
+// in middleware.ts, and server errors from instrumentation.ts onRequestError.
+// No Node built-ins, so it runs in the edge runtime. For events from route
+// handlers and server actions (Node runtime), use the main package.
+//
+// The edge keeps no process between requests the way a server does: each
+// instance keeps the rules in memory while it lives, fetches them when
+// they're older than the rules' sync interval (in the background, with
+// event.waitUntil), and waits up to FIRST_WAIT_MS for them on a cold start.
+// It fails open: no rules, or the collector unreachable, lets requests pass.
+"use strict";
+
+const { createCore } = require("./core.js");
+const { parseIp, IpSet } = require("./ip.js");
+const { createLocalBans, compileProbes, isProbe } = require("./bans.js");
+const { compileRequestRules, matchRequest } = require("./requestRules.js");
+
+const PROTOCOL = 1;
+const MIN_SYNC_S = 30;
+const RETRY_S = 60;
+const FIRST_WAIT_MS = 1500;
+const FLUSH_MS = 5000;
+const BATCH = 200;
+const QUEUE_MAX = 2000;
+const COUNTER_MAX = 50000;
+
+const env = (name) => (typeof process !== "undefined" && process.env ? process.env[name] : undefined);
+
+function headerOf(request, name) {
+  const headers = request.headers;
+  const value = typeof headers?.get === "function" ? headers.get(name) : headers?.[name];
+  return String(Array.isArray(value) ? value[0] : value || "").trim();
+}
+
+// trustProxy: "vercel" (default on Vercel: its x-real-ip), "cloudflare"
+// (CF-Connecting-IP), a number of proxies in X-Forwarded-For, or a function.
+// Self-hosted Next.js has no connection address in middleware: set one.
+function clientIp(request, trustProxy) {
+  if (typeof trustProxy === "function") return String(trustProxy(request) || "");
+  if (trustProxy === "cloudflare") return headerOf(request, "cf-connecting-ip");
+  if (Number.isInteger(trustProxy) && trustProxy > 0) {
+    const chain = headerOf(request, "x-forwarded-for").split(",").map((s) => s.trim()).filter(Boolean);
+    return chain[Math.max(0, chain.length - trustProxy)] || "";
+  }
+  if (trustProxy === "vercel" || (trustProxy === undefined && env("VERCEL"))) {
+    return request.ip || headerOf(request, "x-real-ip") || headerOf(request, "x-forwarded-for").split(",")[0].trim();
+  }
+  return request.ip || "";
+}
+
+function compile(data) {
+  if (!data || data.v !== PROTOCOL) throw new Error(`unknown rules version ${data?.v}`);
+  return {
+    enabled: data.enabled === true,
+    mode: data.mode === "block" ? "block" : "monitor",
+    sync: Math.max(MIN_SYNC_S, Number(data.sync) || RETRY_S),
+    allow: new IpSet(data.allow),
+    // Crawlers the rate limit must not catch, as in the Node firewall.
+    goodbots: new IpSet(data.goodbots),
+    deny: new IpSet(data.deny),
+    list: new IpSet(data.lists),
+    shared: new IpSet(data.shared),
+    rate: data.rate && Number(data.rate.limit) > 0
+      ? { limit: Number(data.rate.limit), window: Math.max(1, Number(data.rate.window) || 60) }
+      : null,
+    banned: new IpSet(data.banned),
+    ban: data.ban && Number(data.ban.seconds) > 0 ? { seconds: Number(data.ban.seconds) } : null,
+    probes: compileProbes(data.probes),
+    // Virtual patching: rules that match the request, not the address.
+    requests: compileRequestRules(data.requests),
+  };
+}
+
+// Options: siteId, secret, endpoint; trustProxy (see clientIp); mode
+// ("monitor" / "block": overrides the dashboard); skip(request); counter
+// (async (key, ttlSeconds) => hits, shared between instances, e.g. Upstash
+// Redis: each instance counts on its own otherwise); debug.
+// Returns middleware(request, event): a Response when blocked, else undefined.
+function createNextFirewall(options = {}) {
+  const core = createCore({ ...options, client: "next" });
+  if (!core.valid) console.warn("[analytics-platform] siteId, secret and endpoint are required; every request passes");
+  let rules = null;
+  let etag = "";
+  let fetchedAt = 0;
+  let syncing = null;
+  let queue = new Map();
+  // The first report goes straight away (an edge instance may not live 5 seconds), then at most every FLUSH_MS.
+  let lastFlush = 0;
+  const counts = new Map();
+  const local = createLocalBans();
+
+  async function sync() {
+    try {
+      const res = await core.send("firewall", `/events/firewall/rules?site_id=${encodeURIComponent(core.siteId)}`, {
+        method: "GET", auth: "bearer", headers: etag ? { "if-none-match": etag } : {},
+      });
+      if (res?.status === 200) {
+        rules = compile(await res.json());
+        etag = res.headers.get("etag") || "";
+      }
+    } catch (error) {
+      core.log("firewall", "rules not used:", error?.message);
+    }
+    fetchedAt = Date.now();
+  }
+
+  function refresh(event) {
+    const due = Date.now() - fetchedAt >= (rules?.sync || RETRY_S) * 1000;
+    if (!core.valid || !due || syncing) return syncing;
+    syncing = sync().finally(() => { syncing = null; });
+    event?.waitUntil?.(syncing);
+    return syncing;
+  }
+
+  async function flush() {
+    if (!queue.size) return;
+    const rows = [...queue.values()];
+    queue = new Map();
+    lastFlush = Date.now();
+    for (let i = 0; i < rows.length; i += BATCH) {
+      await core.send("firewall", "/events/firewall/events", { auth: "bearer", body: { site_id: core.siteId, events: rows.slice(i, i + BATCH) } });
+    }
+  }
+
+  async function overRate(ip, rate) {
+    const window = Math.floor(Date.now() / 1000 / rate.window);
+    if (typeof options.counter === "function") {
+      return Number(await options.counter(`afw:${core.siteId}:${ip}:${window}`, rate.window * 2)) > rate.limit;
+    }
+    let entry = counts.get(ip);
+    if (!entry || entry.window !== window) {
+      if (!entry && counts.size >= COUNTER_MAX) counts.clear();
+      entry = { window, hits: 0 };
+      counts.set(ip, entry);
+    }
+    entry.hits += 1;
+    return entry.hits > rate.limit;
+  }
+
+  return async function analyticsFirewall(request, event) {
+    try {
+      if (!core.valid || options.skip?.(request)) return undefined;
+      const pending = refresh(event);
+      // Cold start: wait a moment for the first rules.
+      if (!rules && pending) await Promise.race([pending, new Promise((resolve) => setTimeout(resolve, FIRST_WAIT_MS))]);
+      const current = rules;
+      if (!current?.enabled) return undefined;
+      const ip = clientIp(request, options.trustProxy);
+      const address = parseIp(ip);
+      if (!address || current.allow.has(address)) return undefined;
+      const mode = options.mode === "block" || options.mode === "monitor" ? options.mode : current.mode;
+      const action = mode === "block" ? "block" : "monitor";
+      // A request rule may only watch, even when the site blocks.
+      let effective = action;
+      const path = request.nextUrl?.pathname || new URL(request.url).pathname;
+      // The rules match the query string too, which the path alone leaves out.
+      const target = (request.nextUrl?.pathname || "") + (request.nextUrl?.search || "")
+        || new URL(request.url).pathname + new URL(request.url).search;
+      let rule = current.deny.has(address) ? "deny"
+        : current.banned.has(address) || local.banned(ip) ? "ban"
+          : current.list.has(address) ? "list" : current.shared.has(address) ? "shared" : "";
+      if (!rule && isProbe(path, current.probes)) {
+        rule = "probe";
+        if (action === "block") local.ban(ip, current.probes.ban);
+      }
+      if (!rule) {
+        const matched = matchRequest(current.requests, { method: request.method, path: target, userAgent: headerOf(request, "user-agent") });
+        if (matched) {
+          rule = `request:${matched.id}`;
+          effective = action === "block" && matched.action === "block" ? "block" : "monitor";
+          if (effective === "block" && matched.ban) local.ban(ip, matched.ban);
+        }
+      }
+      if (!rule && current.rate && !current.goodbots.has(address) && await overRate(ip, current.rate)) {
+        rule = "rate";
+        if (current.ban) local.ban(ip, current.ban.seconds);
+      }
+      if (!rule) return undefined;
+      const method = String(request.method || "").slice(0, 10);
+      const key = `${ip}|${rule}|${effective}|${method}|${path}`;
+      const row = queue.get(key);
+      if (row) row.hits += 1;
+      else if (queue.size < QUEUE_MAX) {
+        queue.set(key, { at: new Date().toISOString(), ip, rule, action: effective, method, path: path.slice(0, 300), user_agent: headerOf(request, "user-agent").slice(0, 300), hits: 1 });
+      }
+      if (queue.size >= BATCH || Date.now() - lastFlush >= FLUSH_MS) event?.waitUntil?.(flush());
+      if (effective !== "block") return undefined;
+      const status = rule === "rate" ? 429 : 403;
+      return new Response(status === 429 ? "Too Many Requests" : "Forbidden", {
+        status,
+        headers: {
+          "content-type": "text/plain; charset=utf-8",
+          "cache-control": "no-store",
+          ...(status === 429 ? { "retry-after": String(current.rate.window) } : {}),
+        },
+      });
+    } catch (error) {
+      core.log("firewall", "check failed, request passed:", error?.message);
+      return undefined;
+    }
+  };
+}
+
+// instrumentation.ts: export const onRequestError = createOnRequestError({...}).
+// Sends each server error (pages, route handlers, server actions, middleware)
+// to the Errors report, unless the dashboard switched error monitoring off.
+function createOnRequestError(options = {}) {
+  const core = createCore({ ...options, client: "next" });
+  let switches = { at: 0, errors: true };
+
+  async function errorsOn() {
+    if (Date.now() - switches.at < RETRY_S * 1000) return switches.errors;
+    const res = await core.send("errors", `/events/platform/config?site_id=${encodeURIComponent(core.siteId)}`, {
+      method: "GET", auth: "bearer", headers: { "x-platform-modules": "errors" },
+    });
+    const data = res?.ok ? await res.json().catch(() => null) : null;
+    switches = { at: Date.now(), errors: data?.modules?.errors !== false };
+    return switches.errors;
+  }
+
+  return async function onRequestError(error, request, context) {
+    try {
+      if (!core.valid || !(await errorsOn())) return;
+      const headers = request?.headers || {};
+      const session = headers["x-analytics-session"];
+      await core.send("errors", "/events/server-errors", {
+        body: {
+          errors: [{
+            message: String(error?.message || error || "Unknown error"),
+            error_type: error?.name || "Error",
+            stack: typeof error?.stack === "string" ? error.stack : "",
+            kind: "error",
+            status_code: 500,
+            runtime: "Next.js",
+            runtime_version: env("NEXT_RUNTIME") || "",
+            os: "",
+            server_name: env("VERCEL_REGION") || env("HOSTNAME") || "",
+            method: String(request?.method || ""),
+            path: String(request?.path || "").split("?")[0],
+            route: String(context?.routePath || ""),
+            session_id: typeof session === "string" ? session : "",
+          }],
+        },
+      });
+    } catch {
+      // never throw from the error hook
+    }
+  };
+}
+
+module.exports = { createNextFirewall, createOnRequestError, clientIp };