analytics-platform 0.2.1 → 0.2.2

package/README.md

@@ -57,7 +57,7 @@ createPlatform({
 });
 ```
 
-**Switch modules off in the dashboard** (Security → Settings → Server package) without a
+**Switch modules off in the dashboard** (Security → Server package) without a
 deploy: the package checks about once a minute. A module turned off in your
 code isn't loaded at all, so the dashboard can't turn it on.
 

package/index.js

@@ -14,6 +14,7 @@
 "use strict";
 
 const { createCore } = require("./lib/core.js");
+const { clientIp } = require("./lib/firewall.js");
 
 // Module name -> [loader, on by default].
 const MODULES = {
@@ -46,8 +47,11 @@ function createPlatform(options = {}) {
     const setting = options[name] === undefined ? onByDefault : options[name];
     if (setting === false) continue;
     const opts = { trustProxy: options.trustProxy, ...(setting && typeof setting === "object" ? setting : {}) };
-    // Form checks find the visitor's address the way the firewall does.
-    if (["forms", "audit", "maintenance", "crawlers", "proxy"].includes(name) && modules.firewall) opts.clientIp = modules.firewall.clientIp;
+    // These find the visitor's address the way the firewall does, with the
+    // same trustProxy, whether or not the firewall itself is on.
+    if (["forms", "audit", "maintenance", "crawlers", "proxy"].includes(name)) {
+      opts.clientIp = modules.firewall?.clientIp || ((req) => clientIp(req, options.trustProxy));
+    }
     try {
       modules[name] = load(core, opts);
     } catch (error) {

package/lib/bans.js

@@ -5,13 +5,23 @@
 // of files no visitor asks for (.env, .git/...). No Node built-ins (edge-safe).
 "use strict";
 
+const { parseIp } = require("./ip.js");
+
 const MAX = 50000;
 
+// One key per address however it's spelled: "1.2.3.4" and "::ffff:1.2.3.4"
+// (and IPv6 written two ways) are the same visitor.
+function keyOf(ip) {
+  const address = parseIp(ip);
+  return address ? `${address.family}:${address.n}` : String(ip);
+}
+
 function createLocalBans() {
   const bans = new Map(); // address -> when the ban ends (ms)
   const logins = new Map(); // address -> { start, count }
 
-  function banned(ip) {
+  function banned(address) {
+    const ip = keyOf(address);
     const until = bans.get(ip);
     if (!until) return false;
     if (until > Date.now()) return true;
@@ -19,16 +29,18 @@ function createLocalBans() {
     return false;
   }
 
-  function ban(ip, seconds) {
+  function ban(address, seconds) {
     if (!(seconds > 0)) return;
+    const ip = keyOf(address);
     if (bans.size >= MAX) bans.clear();
     bans.set(ip, Math.max(bans.get(ip) || 0, Date.now() + seconds * 1000));
   }
 
   // One failed login; login: the rules' { limit, window, ban } (seconds).
   // True when it got the address banned.
-  function loginFailed(ip, login) {
+  function loginFailed(address, login) {
     if (!login) return false;
+    const ip = keyOf(address);
     const now = Date.now();
     let entry = logins.get(ip);
     if (!entry || now - entry.start >= login.window * 1000) {
@@ -38,7 +50,7 @@ function createLocalBans() {
     }
     entry.count += 1;
     if (entry.count < login.limit) return false;
-    ban(ip, login.ban);
+    ban(address, login.ban);
     logins.delete(ip);
     return true;
   }

package/lib/core.js

@@ -6,7 +6,7 @@
 // Sending never throws into the app.
 "use strict";
 
-const VERSION = "0.1.0";
+const VERSION = "0.2.2";
 const TIMEOUT_MS = 5000;
 const MIN_SYNC_S = 30;
 const RETRY_S = 60;

package/lib/crawlers.js

@@ -117,6 +117,7 @@ function createCrawlers(core, options = {}) {
     ? options.clientIp
     : (req) => req.ip || req.socket?.remoteAddress || "";
   let rows = new Map();
+  const keysIn = new Map(); // minute -> rows it has
   let agents = new Map();
   let sending = Promise.resolve();
 
@@ -127,11 +128,14 @@ function createCrawlers(core, options = {}) {
     const clean = String(path || "/").split(/[?#]/)[0].slice(0, MAX_PATH) || "/";
     let key = `${minute}|${bot}|${clean}|${status}`;
     // Past MAX_KEYS a crawler asking for thousands of URLs becomes one row.
-    if (!rows.has(key) && rows.size >= MAX_KEYS) key = `${minute}|${bot}|${OTHER}|${status}`;
+    // The cap is per minute: rows of a minute not yet sent don't count.
+    const used = keysIn.get(minute) || 0;
+    if (!rows.has(key) && used >= MAX_KEYS) key = `${minute}|${bot}|${OTHER}|${status}`;
     let row = rows.get(key);
     if (!row) {
       row = { minute, bot, path: key.split("|")[2], status, count: 0, bytes: 0, sum_ms: 0 };
       rows.set(key, row);
+      keysIn.set(minute, used + 1);
     }
     row.count += 1;
     row.bytes += Math.max(0, Number(bytes) || 0);
@@ -177,6 +181,7 @@ function createCrawlers(core, options = {}) {
         rows.delete(key);
       }
     }
+    for (const minute of keysIn.keys()) if (all || minute < current) keysIn.delete(minute);
     if (!ready.length) return sending;
     const sent = new Set(ready.map((row) => row.bot));
     const batchAgents = [...agents].filter(([bot]) => sent.has(bot)).map(([bot, ips]) => ({ bot, ips: [...ips] }));

package/lib/errors.js

@@ -13,6 +13,10 @@ const BATCH = 10;
 const QUEUE_MAX = 100;
 const FLUSH_MS = 2000;
 const EXIT_FLUSH_MS = 2000;
+// Every instance's uncaught-exception handler, with its flush: two platforms in
+// one process are still "no other handler", and the exit waits for both.
+const handlers = new Map();
+let exiting = false;
 
 // Options: serverName (default the host name), root (paths in stacks are
 // made relative to it; default process.cwd()), release (which build this is,
@@ -140,16 +144,24 @@ function createErrors(core, options = {}) {
   // after up to EXIT_FLUSH_MS to send it.
   function onUncaught(error, origin) {
     captureException(error, { kind: origin === "unhandledRejection" ? "rejection" : "error" });
-    if (process.listenerCount("uncaughtException") > 1) return;
+    if (exiting || process.listeners("uncaughtException").some((listener) => !handlers.has(listener))) return;
+    exiting = true;
     console.error(error);
-    const exit = setTimeout(() => process.exit(1), EXIT_FLUSH_MS);
-    flush().finally(() => {
-      clearTimeout(exit);
-      process.exit(1);
+    // After the other instances' handlers (called right after this one) have
+    // queued the error too.
+    process.nextTick(() => {
+      const exit = setTimeout(() => process.exit(1), EXIT_FLUSH_MS);
+      Promise.allSettled([...handlers.values()].map((send) => send())).finally(() => {
+        clearTimeout(exit);
+        process.exit(1);
+      });
     });
   }
 
-  if (options.captureUncaught !== false) process.on("uncaughtException", onUncaught);
+  if (options.captureUncaught !== false) {
+    handlers.set(onUncaught, flush);
+    process.on("uncaughtException", onUncaught);
+  }
 
   return {
     captureException,
@@ -157,6 +169,7 @@ function createErrors(core, options = {}) {
     flush,
     close() {
       process.off("uncaughtException", onUncaught);
+      handlers.delete(onUncaught);
       return flush();
     },
   };

package/lib/firewall.js

@@ -196,8 +196,8 @@ function createFirewall(core, options = {}) {
     }
     if (!rule && current.rate && !current.goodbots.has(address) && overRate(String(ip), current.rate)) {
       rule = "rate";
-      // Banned here straight away; the report bans it on the site's other servers.
-      if (current.ban) local.ban(String(ip), current.ban.seconds);
+      // Banned here straight away in block mode, as a probe is; the report bans it on the other servers.
+      if (action === "block" && current.ban) local.ban(String(ip), current.ban.seconds);
     }
     if (!rule) return allow;
     report({
@@ -259,4 +259,4 @@ function createFirewall(core, options = {}) {
   return { middleware, check, loginFailed, flush, close, ready, clientIp: (req) => clientIp(req, options.trustProxy) };
 }
 
-module.exports = { createFirewall, parseIp, parseRange, IpSet };
+module.exports = { createFirewall, clientIp, parseIp, parseRange, IpSet };

package/lib/flags.js

@@ -1,5 +1,5 @@
 // Feature flags module: turn features of your app on or off from the
-// dashboard (Security → Settings → Server package), for everyone, for a share of users, or
+// dashboard (Security → Server package), for everyone, for a share of users, or
 // for listed user IDs. isEnabled() answers from memory; the flags come with
 // the module switches (core.js), about once a minute.
 "use strict";

package/lib/maintenance.js

@@ -1,4 +1,4 @@
-// Maintenance mode module: while it's on (Security → Settings → Server package), visitors
+// Maintenance mode module: while it's on (Security → Server package), visitors
 // get a "back soon" page (503, so search engines know it's temporary),
 // except allowed addresses and whoever opened the preview link
 // (?site_preview=<code>, remembered for a day in a cookie).
@@ -21,11 +21,26 @@ function cookieOf(req, name) {
   const header = String(req.headers?.cookie || "");
   for (const part of header.split(";")) {
     const [k, ...v] = part.trim().split("=");
-    if (k === name) return decodeURIComponent(v.join("="));
+    if (k !== name) continue;
+    // A malformed value can't be the preview code, and mustn't throw: the
+    // middleware lets a request through when its check fails.
+    try {
+      return decodeURIComponent(v.join("="));
+    } catch {
+      return "";
+    }
   }
   return "";
 }
 
+function queryOf(req, name) {
+  try {
+    return new URL(req.originalUrl || req.url || "/", "http://x").searchParams.get(name);
+  } catch {
+    return null;
+  }
+}
+
 // clientIp(req): the firewall's way of finding the visitor's address.
 // Options: skip(req) (true lets a request through, e.g. health checks).
 function createMaintenance(core, { clientIp, skip } = {}) {
@@ -43,8 +58,7 @@ function createMaintenance(core, { clientIp, skip } = {}) {
       const ip = parseIp(clientIp ? clientIp(req) : req.ip || req.socket?.remoteAddress || "");
       if (ip && allowSet.has(ip)) return next();
       if (m.bypass) {
-        const url = new URL(req.originalUrl || req.url || "/", "http://x");
-        if (url.searchParams.get(COOKIE) === m.bypass) {
+        if (queryOf(req, COOKIE) === m.bypass) {
           res.setHeader("set-cookie", `${COOKIE}=${encodeURIComponent(m.bypass)}; Path=/; Max-Age=86400; HttpOnly; SameSite=Lax`);
           return next();
         }

package/lib/next.js

@@ -55,6 +55,8 @@ function compile(data) {
     mode: data.mode === "block" ? "block" : "monitor",
     sync: Math.max(MIN_SYNC_S, Number(data.sync) || RETRY_S),
     allow: new IpSet(data.allow),
+    // Crawlers the rate limit must not catch, as in the Node firewall.
+    goodbots: new IpSet(data.goodbots),
     deny: new IpSet(data.deny),
     list: new IpSet(data.lists),
     shared: new IpSet(data.shared),
@@ -169,7 +171,7 @@ function createNextFirewall(options = {}) {
           if (effective === "block" && matched.ban) local.ban(ip, matched.ban);
         }
       }
-      if (!rule && current.rate && await overRate(ip, current.rate)) {
+      if (!rule && current.rate && !current.goodbots.has(address) && await overRate(ip, current.rate)) {
         rule = "rate";
         if (current.ban) local.ban(ip, current.ban.seconds);
       }

package/lib/performance.js

@@ -40,6 +40,7 @@ function createPerformance(core, options = {}) {
   const serverName = String(options.serverName ?? os.hostname()).slice(0, 100);
   const runtime = `Node ${process.versions.node}`;
   let rows = new Map();
+  const keysIn = new Map(); // minute -> rows it has
   let sending = Promise.resolve();
 
   // Records one finished request.
@@ -47,11 +48,14 @@ function createPerformance(core, options = {}) {
     if (!core.isOn("performance")) return;
     const minute = Math.floor(Date.now() / 60000) * 60000;
     let key = `${minute}|${method}|${route}|${status}`;
-    if (!rows.has(key) && rows.size >= MAX_KEYS) key = `${minute}|${method}|(other)|${status}`;
+    // The cap is per minute: rows of a minute not yet sent don't count.
+    const used = keysIn.get(minute) || 0;
+    if (!rows.has(key) && used >= MAX_KEYS) key = `${minute}|${method}|(other)|${status}`;
     let row = rows.get(key);
     if (!row) {
       row = { minute, method, route: key.split("|")[2], status, count: 0, sum_ms: 0, max_ms: 0, buckets: new Array(BUCKET_EDGES_MS.length + 1).fill(0) };
       rows.set(key, row);
+      keysIn.set(minute, used + 1);
     }
     row.count += 1;
     row.sum_ms += ms;
@@ -91,6 +95,7 @@ function createPerformance(core, options = {}) {
         rows.delete(key);
       }
     }
+    for (const minute of keysIn.keys()) if (all || minute < current) keysIn.delete(minute);
     for (let i = 0; i < ready.length; i += MAX_ROWS) {
       const batch = ready.slice(i, i + MAX_ROWS);
       sending = sending.then(() => core.send("performance", "/events/server-requests", { body: { server_name: serverName, runtime, rows: batch } }));

package/lib/posture.js

@@ -204,6 +204,8 @@ function createPosture(core, options = {}) {
 
   function schedule(ms) {
     if (closed) return;
+    // A report called by hand restarts the wait rather than starting a second chain.
+    clearTimeout(timer);
     timer = setTimeout(report, ms);
     timer.unref?.();
   }

package/lib/proxy.js

@@ -23,7 +23,7 @@ const crypto = require("node:crypto");
 
 // The tracker files, by name: anything else is refused rather than forwarded.
 const FILES = new Set(["track.js", "s.js", "snapshot.js", "replay.js", "survey.js", "banner.js",
-  "leadform.js", "notice.js", "editor.js", "picker.js", "feedback.js", "shield.js"]);
+  "leadform.js", "notice.js", "editor.js", "picker.js", "feedback.js", "shield.js", "widget.js", "share.js"]);
 // A pinned version never changes; /v<major>/ follows new releases, so it's
 // held for an hour, the same as the collector serves it with.
 const TTL_PINNED_MS = 7 * 24 * 60 * 60 * 1000;
@@ -34,6 +34,9 @@ const MAX_BODY = 6 * 1024 * 1024;
 const PASS_HEADERS = ["content-type", "user-agent", "accept-language", "referer", "origin",
   "sec-fetch-site", "sec-ch-ua", "sec-ch-ua-mobile", "sec-ch-ua-platform"];
 
+const isEmptyObject = (value) => value !== null && typeof value === "object" && !Buffer.isBuffer(value)
+  && Object.getPrototypeOf(value) === Object.prototype && Object.keys(value).length === 0;
+
 const prefixFor = (secret) => crypto.createHash("sha256").update(`proxy|${secret}`).digest("hex").slice(0, 8);
 
 // Options: prefix (default: from the secret), enabled (default true).
@@ -84,8 +87,12 @@ function createProxy(core, options = {}) {
 
   // The request's body, however the app is put together: straight from the
   // stream, or from whatever a body parser mounted before us already read.
+  // Express 4's parsers set req.body = {} even for a type they skip (the
+  // tracker's text/plain), leaving the stream unread: that empty object isn't
+  // the body. req._body is how they mark one they did read.
   function readBody(req) {
-    if (req.readableEnded || req.body !== undefined) {
+    const skipped = req._body !== true && isEmptyObject(req.body);
+    if (req.readableEnded || (req.body !== undefined && !skipped)) {
       const parsed = req.body;
       if (parsed === undefined || parsed === null) return Promise.resolve("");
       if (typeof parsed === "string" || Buffer.isBuffer(parsed)) return Promise.resolve(parsed);
@@ -134,7 +141,7 @@ function createProxy(core, options = {}) {
     const headers = {
       // The visitor's address: the collector needs it for the country, the
       // daily visitor ID and the per-visitor rate limit.
-      "x-forwarded-for": (clientIp ? clientIp(req) : "") || req.socket?.remoteAddress || "",
+      "x-forwarded-for": (clientIp ? clientIp(req) : req.ip) || req.socket?.remoteAddress || "",
       "x-forwarded-proto": req.socket?.encrypted ? "https" : "http",
     };
     for (const name of PASS_HEADERS) {

package/lib/requestRules.js

@@ -20,25 +20,36 @@ const MAX_REGEX = 200;
 const lower = (value) => String(value ?? "").slice(0, MAX_TEXT).toLowerCase();
 // One pass only, as the protocol says: enough for %2e%2e, and no risk of
 // decoding a visitor's own text into something that looks like an attack.
-function decodeOnce(text) {
-  if (!text.includes("%") && !text.includes("+")) return text;
+// Each run of escapes is decoded on its own, so a stray "%" elsewhere can't
+// leave the whole text undecoded; a run that isn't valid UTF-8 still has its
+// ASCII escapes (%2e, %2f) decoded.
+function decodeRun(run) {
   try {
-    return decodeURIComponent(text.replace(/\+/g, " "));
+    return decodeURIComponent(run);
   } catch {
-    return text;
+    return run.replace(/%([0-7][0-9a-f])/gi, (_, hex) => String.fromCharCode(parseInt(hex, 16)));
   }
 }
+function decodeOnce(text) {
+  if (!text.includes("%") && !text.includes("+")) return text;
+  return text.replace(/\+/g, " ").replace(/(?:%[0-9a-f]{2})+/gi, decodeRun);
+}
 const normalizePath = (path) => lower(path).replace(/\\/g, "/").replace(/\/{2,}/g, "/");
 
 // A pattern can only come from us, but a careless one would be matched against
 // text an attacker chooses, in this very request, and JavaScript cannot time a
 // regex out. So the client checks the shape itself instead of trusting what it
-// was handed: a quantifier on a group that itself repeats ("(a+)+") takes
-// exponential time on input that nearly matches. Backreferences and lookbehind
+// was handed: a repeated group that can match the same text more than one way
+// takes exponential time on input that nearly matches. A group can do that when
+// it holds a quantifier ("(a+)+", "(\w\d?)+") or an alternation ("(a|ab)*"),
+// so a group followed by * + or { holds neither. Backreferences and lookbehind
 // are refused for the same reason (and they aren't in every engine we ship to).
+// The same check as backend/services/firewall/requestRules.js, freeRepeats
+// included; sdk/protocols/request-rules-cases.json holds the shared lists.
 function safePattern(pattern) {
   if (typeof pattern !== "string" || !pattern || pattern.length > MAX_REGEX) return false;
   if (/\\[1-9]/.test(pattern) || /\(\?</.test(pattern)) return false;
+  // Per open group: whether anything in it, nested groups too, repeats or branches.
   const open = [];
   for (let i = 0; i < pattern.length; i++) {
     const char = pattern[i];
@@ -47,22 +58,82 @@ function safePattern(pattern) {
       continue;
     }
     if (char === "[") {
-      // A character class: quantifiers inside it are literals.
+      // A character class: skip it whole, quantifiers inside it are literals.
       while (i < pattern.length && pattern[i] !== "]") i += pattern[i] === "\\" ? 2 : 1;
       continue;
     }
-    if (char === "(") open.push(i);
-    else if (char === ")") {
-      const start = open.pop();
-      if (start === undefined) return false;
+    if (char === "(") {
+      open.push(false);
+      // "(?:", "(?=", "(?!": that ? names the kind of group, it repeats nothing.
+      if (pattern[i + 1] === "?") i += 2;
+    } else if (char === ")") {
+      if (!open.length) return false;
+      const ambiguous = open.pop();
       const next = pattern[i + 1];
-      if (next === "*" || next === "+" || next === "{") {
-        const inside = pattern.slice(start + 1, i);
-        if (/(^|[^\\])[*+]/.test(inside) || /(^|[^\\])\{\d/.test(inside)) return false;
+      if (ambiguous && (next === "*" || next === "+" || next === "{")) return false;
+      if (ambiguous && open.length) open[open.length - 1] = true;
+    } else if (open.length && "|*+?{".includes(char)) {
+      open[open.length - 1] = true;
+    }
+  }
+  return open.length === 0 && freeRepeats(pattern) <= 1;
+}
+
+// Two repeats that can both take the same text multiply each other's work:
+// ".*=.*=.*x" against a query of "=" that never ends in "x" is polynomial,
+// and the 2000 characters the clients match at most is plenty for that to
+// hang a request. A repeat is pinned when the character straight after it is
+// one it can't match ("[^/]+/" ends at the first "/", once); any other is
+// free, and a pattern may hold one free repeat.
+const literalAt = (pattern, i) => {
+  const char = pattern[i];
+  if (char === undefined) return null;
+  if (char === "\\") return pattern[i + 1] && !/[a-z0-9]/i.test(pattern[i + 1]) ? pattern[i + 1] : null;
+  return ".[](){}*+?|^$".includes(char) ? null : char;
+};
+function freeRepeats(pattern) {
+  let free = 0;
+  for (let i = 0; i < pattern.length;) {
+    const start = i;
+    const char = pattern[i];
+    if (char === "\\") i += 2;
+    else if (char === "[") {
+      i++;
+      while (i < pattern.length && pattern[i] !== "]") i += pattern[i] === "\\" ? 2 : 1;
+      i++;
+    } else if (char === "(") {
+      i += pattern[i + 1] === "?" ? 3 : 1;
+      continue;
+    } else if (char === "|" || char === "^" || char === "$") {
+      i++;
+      continue;
+    } else i++;
+    // A group's text can't be tried against one character.
+    const atom = char === ")" ? null : pattern.slice(start, i);
+    const quantifier = pattern[i];
+    let repeats = quantifier === "*" || quantifier === "+";
+    if (quantifier === "{") {
+      const bounds = /^\{(\d*)(,(\d*))?\}/.exec(pattern.slice(i));
+      if (!bounds) continue;
+      // "{3}" is one length, not a choice; "{2,}" and "{2,9}" are choices.
+      repeats = bounds[2] !== undefined && bounds[3] !== bounds[1];
+      i += bounds[0].length - 1;
+    } else if (quantifier !== "?" && !repeats) continue;
+    i++;
+    if (pattern[i] === "?") i++;
+    if (!repeats) continue;
+    const next = literalAt(pattern, i);
+    let pinned = false;
+    if (atom && next !== null) {
+      try {
+        pinned = !new RegExp(`^(?:${atom})$`, "i").test(next);
+      } catch {
+        // not an atom on its own: free
       }
     }
+    if (!pinned) free++;
   }
-  return open.length === 0;
+  return free;
 }
 
 function compileSection(section) {
@@ -130,7 +201,8 @@ function matchRequest(rules, { method = "", path = "", userAgent = "" } = {}) {
   const rawQuery = lower(mark === -1 ? "" : target.slice(mark + 1));
   const ua = lower(userAgent);
   const decodedPath = normalizePath(decodeOnce(mark === -1 ? target : target.slice(0, mark)));
-  const decodedQuery = decodeOnce(rawQuery);
+  // Lowered again after decoding: %55NION is UNION, and the patterns are lower case.
+  const decodedQuery = decodeOnce(rawQuery).toLowerCase();
   const upper = String(method ?? "").toUpperCase();
   for (const rule of rules) {
     if (rule.methods && !rule.methods.has(upper)) continue;

package/lib/resources.js

@@ -384,7 +384,7 @@ function createResources(core, options = {}) {
     if (res.ok) {
       const data = await res.json().catch(() => null);
       if (Number(data?.every) > 0) every = Math.max(RETRY_S, Number(data.every));
-      if (data?.enabled === false) log("server resources are off in the dashboard (Security → Settings → Server package)");
+      if (data?.enabled === false) log("server resources are off in the dashboard (Security → Server package)");
       for (const minute of batch) rows.delete(minute);
       return true;
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "analytics-platform",
-  "version": "0.2.1",
+  "version": "0.2.2",
   "description": "Firewall, error monitoring and server analytics for your Node server, run from your analytics dashboard.",
   "author": "steviec87",
   "license": "MIT",