analytics-platform 0.0.0-stage → 0.2.2

package/lib/errors.js

@@ -0,0 +1,178 @@
+// Error monitoring module: the server's errors go to the collector (POST
+// /events/server-errors), where they join the browser's errors in the Errors
+// report. Uncaught exceptions and unhandled rejections are captured on their
+// own and the process still exits as Node would. errorHandler() captures
+// Express request errors (5xx only); captureException() captures errors you
+// catch. Errors are sent in small batches; sending never throws into your app.
+"use strict";
+
+const os = require("node:os");
+const { sessionOf } = require("./core.js");
+
+const BATCH = 10;
+const QUEUE_MAX = 100;
+const FLUSH_MS = 2000;
+const EXIT_FLUSH_MS = 2000;
+// Every instance's uncaught-exception handler, with its flush: two platforms in
+// one process are still "no other handler", and the exit waits for both.
+const handlers = new Map();
+let exiting = false;
+
+// Options: serverName (default the host name), root (paths in stacks are
+// made relative to it; default process.cwd()), release (which build this is,
+// for "introduced in" in the Errors report; defaults to ANALYTICS_RELEASE in
+// the environment), captureUncaught (default true), beforeSend(event, error) to
+// change an event or drop it (return null).
+function createErrors(core, options = {}) {
+  const config = {
+    serverName: String(options.serverName ?? os.hostname()).slice(0, 100),
+    root: String(options.root ?? process.cwd()),
+    // Letters, numbers and .-_+ only, as the collector stores it.
+    release: String(options.release ?? process.env.ANALYTICS_RELEASE ?? "")
+      .trim().replace(/[^\w.+-]/g, "").slice(0, 64),
+    beforeSend: typeof options.beforeSend === "function" ? options.beforeSend : null,
+  };
+  let queue = [];
+  let timer = null;
+  let sending = Promise.resolve();
+  const log = (...args) => core.log("errors", ...args);
+
+  // Paths relative to the app's root, so an issue keeps its identity across
+  // servers and deploy directories.
+  function relative(stack) {
+    const root = config.root.replace(/\/+$/, "");
+    return root ? stack.split(`file://${root}/`).join("").split(`${root}/`).join("") : stack;
+  }
+
+  // Fields of the request the error happened in (Express, or Node's own
+  // http.IncomingMessage).
+  function requestFields(req) {
+    if (!req || typeof req !== "object") return {};
+    const url = String(req.originalUrl || req.url || "");
+    const route = req.route?.path && typeof req.route.path === "string" ? `${req.baseUrl || ""}${req.route.path}` : "";
+    return {
+      method: String(req.method || ""),
+      path: url.split("?")[0],
+      route,
+      session_id: sessionOf(req),
+    };
+  }
+
+  function toEvent(error, { req, status, kind = "handled" } = {}) {
+    const isError = error instanceof Error;
+    const message = isError ? error.message : typeof error === "string" ? error : safeString(error);
+    return {
+      message: message || (isError ? error.name : "Unknown error"),
+      error_type: isError ? error.name || error.constructor?.name || "Error" : "Error",
+      stack: isError && typeof error.stack === "string" ? relative(error.stack) : "",
+      kind,
+      status_code: Number(status) || 0,
+      runtime: "Node",
+      runtime_version: process.versions.node,
+      os: process.platform,
+      server_name: config.serverName,
+      release: config.release,
+      ...requestFields(req),
+    };
+  }
+
+  function safeString(value) {
+    try {
+      return JSON.stringify(value);
+    } catch {
+      return String(value);
+    }
+  }
+
+  function schedule() {
+    if (timer || !queue.length) return;
+    timer = setTimeout(() => {
+      timer = null;
+      flush();
+    }, FLUSH_MS);
+    timer.unref?.();
+  }
+
+  const send = (errors) => core.send("errors", "/events/server-errors", { body: { errors } });
+
+  // Sends everything queued; resolves when sent (or failed).
+  function flush() {
+    if (timer) {
+      clearTimeout(timer);
+      timer = null;
+    }
+    const batches = [];
+    while (queue.length) batches.push(queue.splice(0, BATCH));
+    sending = sending.then(() => Promise.all(batches.map(send)));
+    return sending;
+  }
+
+  // Queues one error; `context` is { req, status, kind }. Returns nothing and
+  // never throws.
+  function captureException(error, context = {}) {
+    if (!core.isOn("errors")) return;
+    try {
+      let event = toEvent(error, context);
+      if (config.beforeSend) event = config.beforeSend(event, error);
+      if (!event) return;
+      if (queue.length >= QUEUE_MAX) {
+        log("queue full, error dropped");
+        return;
+      }
+      queue.push(event);
+      if (queue.length >= BATCH) flush();
+      else schedule();
+    } catch (failure) {
+      log("could not capture error:", failure?.message);
+    }
+  }
+
+  // Express error middleware; use after your routes. Captures 5xx errors (4xx
+  // are the client's) and passes the error on unchanged.
+  function errorHandler({ shouldCapture } = {}) {
+    return function analyticsErrorHandler(error, req, res, next) {
+      const status = Number(error?.status || error?.statusCode) || 500;
+      const capture = shouldCapture ? shouldCapture(error, req) : status >= 500;
+      if (capture) captureException(error, { req, status, kind: "handled" });
+      next(error);
+    };
+  }
+
+  // Uncaught exceptions, and unhandled rejections (which Node turns into
+  // uncaught exceptions by default). When no other handler is listening, the
+  // error is printed and the process exits with code 1, as without the SDK,
+  // after up to EXIT_FLUSH_MS to send it.
+  function onUncaught(error, origin) {
+    captureException(error, { kind: origin === "unhandledRejection" ? "rejection" : "error" });
+    if (exiting || process.listeners("uncaughtException").some((listener) => !handlers.has(listener))) return;
+    exiting = true;
+    console.error(error);
+    // After the other instances' handlers (called right after this one) have
+    // queued the error too.
+    process.nextTick(() => {
+      const exit = setTimeout(() => process.exit(1), EXIT_FLUSH_MS);
+      Promise.allSettled([...handlers.values()].map((send) => send())).finally(() => {
+        clearTimeout(exit);
+        process.exit(1);
+      });
+    });
+  }
+
+  if (options.captureUncaught !== false) {
+    handlers.set(onUncaught, flush);
+    process.on("uncaughtException", onUncaught);
+  }
+
+  return {
+    captureException,
+    errorHandler,
+    flush,
+    close() {
+      process.off("uncaughtException", onUncaught);
+      handlers.delete(onUncaught);
+      return flush();
+    },
+  };
+}
+
+module.exports = { createErrors };

package/lib/firewall.js

@@ -0,0 +1,262 @@
+// Firewall module: rules from the collector (GET /events/firewall/rules,
+// protocol in sdk/protocols/firewall.md) every minute or so; every request is
+// checked against them in memory, so the app never waits on the collector.
+// What was blocked (or, in monitor mode, would have been) goes back in
+// batches. Fails open: until rules arrive, and whenever the collector can't be
+// reached, requests pass (with the last rules that arrived, if any).
+"use strict";
+
+const fs = require("node:fs");
+const { parseIp, parseRange, IpSet } = require("./ip.js");
+const { createLocalBans, compileProbes, isProbe } = require("./bans.js");
+const { compileRequestRules, matchRequest } = require("./requestRules.js");
+
+const PROTOCOL = 1;
+const MIN_SYNC_S = 30;
+const RETRY_S = 60;
+const FLUSH_MS = 5000;
+const BATCH = 200;
+// Distinct (address, rule, path) rows waiting to be sent; more are dropped.
+const QUEUE_MAX = 2000;
+// Addresses the rate limit counts at once; past it the counts start over.
+const COUNTER_MAX = 100000;
+
+// ---- The client's address -------------------------------------------------
+
+function header(req, name) {
+  const value = req.headers?.[name];
+  return String(Array.isArray(value) ? value[0] : value || "").trim();
+}
+
+// trustProxy: undefined uses Express's req.ip (its own "trust proxy"
+// setting), else the connection's address; a number trusts that many proxies
+// in X-Forwarded-For (true = 1); "cloudflare" reads CF-Connecting-IP (only
+// when every request comes through Cloudflare); a function returns the address.
+function clientIp(req, trustProxy) {
+  if (typeof trustProxy === "function") return String(trustProxy(req) || "");
+  const direct = req.socket?.remoteAddress || "";
+  if (trustProxy === "cloudflare") return header(req, "cf-connecting-ip") || direct;
+  if (trustProxy === true || (Number.isInteger(trustProxy) && trustProxy > 0)) {
+    const hops = trustProxy === true ? 1 : trustProxy;
+    const chain = header(req, "x-forwarded-for").split(",").map((s) => s.trim()).filter(Boolean);
+    return chain.length ? chain[Math.max(0, chain.length - hops)] : direct;
+  }
+  return req.ip || direct;
+}
+
+// ---- The firewall ---------------------------------------------------------
+
+// Options: trustProxy (see clientIp); mode ("monitor" or "block": overrides
+// the dashboard's); skip(req) (true leaves a request unchecked);
+// onBlock(req, res, decision) (send your own response); cacheFile (rules are
+// kept there between restarts).
+function createFirewall(core, options = {}) {
+  const valid = core.valid;
+  const log = (...args) => core.log("firewall", ...args);
+
+  let rules = null;
+  let etag = "";
+  let syncTimer = null;
+  let flushTimer = null;
+  let closed = false;
+  const counts = new Map();
+  const local = createLocalBans();
+  let queue = new Map();
+
+  function compile(data) {
+    if (!data || data.v !== PROTOCOL) throw new Error(`unknown rules version ${data?.v}`);
+    return {
+      enabled: data.enabled === true,
+      mode: data.mode === "block" ? "block" : "monitor",
+      sync: Math.max(MIN_SYNC_S, Number(data.sync) || RETRY_S),
+      allow: new IpSet(data.allow),
+      // Crawlers the rate limit must not catch; absent from older rules, which
+      // is simply an empty set.
+      goodbots: new IpSet(data.goodbots),
+      deny: new IpSet(data.deny),
+      list: new IpSet(data.lists),
+      shared: new IpSet(data.shared),
+      rate: data.rate && Number(data.rate.limit) > 0
+        ? { limit: Number(data.rate.limit), window: Math.max(1, Number(data.rate.window) || 60) * 1000 }
+        : null,
+      banned: new IpSet(data.banned),
+      ban: data.ban && Number(data.ban.seconds) > 0 ? { seconds: Number(data.ban.seconds) } : null,
+      login: data.login && Number(data.login.limit) > 0
+        ? { limit: Number(data.login.limit), window: Number(data.login.window) || 3600, ban: Number(data.login.ban) || 3600 }
+        : null,
+      probes: compileProbes(data.probes),
+      // Virtual patching: rules that match the request, not the address.
+      requests: compileRequestRules(data.requests),
+    };
+  }
+
+  if (options.cacheFile) {
+    try {
+      const cached = JSON.parse(fs.readFileSync(options.cacheFile, "utf8"));
+      rules = compile(cached.rules);
+      etag = cached.etag || "";
+    } catch (error) {
+      if (error.code !== "ENOENT") log("could not read cacheFile:", error.message);
+    }
+  }
+
+  // Fixed window per address.
+  function overRate(key, rate) {
+    const now = Date.now();
+    let entry = counts.get(key);
+    if (!entry || now - entry.start >= rate.window) {
+      if (!entry && counts.size >= COUNTER_MAX) counts.clear();
+      entry = { start: now, hits: 0 };
+      counts.set(key, entry);
+    }
+    entry.hits += 1;
+    return entry.hits > rate.limit;
+  }
+
+  function report(event) {
+    const key = `${event.ip}|${event.rule}|${event.action}|${event.method}|${event.path}`;
+    const row = queue.get(key);
+    if (row) row.hits += 1;
+    else if (queue.size < QUEUE_MAX) queue.set(key, { ...event, at: new Date().toISOString(), hits: 1 });
+    if (queue.size >= BATCH) flush();
+    else if (!flushTimer) {
+      flushTimer = setTimeout(flush, FLUSH_MS);
+      flushTimer.unref?.();
+    }
+  }
+
+  async function flush() {
+    clearTimeout(flushTimer);
+    flushTimer = null;
+    if (!queue.size || !valid) return;
+    const rows = [...queue.values()];
+    queue = new Map();
+    for (let i = 0; i < rows.length; i += BATCH) {
+      await core.send("firewall", "/events/firewall/events", {
+        auth: "bearer",
+        body: { site_id: core.siteId, events: rows.slice(i, i + BATCH) },
+      });
+    }
+  }
+
+  async function sync() {
+    let wait = rules?.sync || RETRY_S;
+    try {
+      const res = await core.send("firewall", `/events/firewall/rules?site_id=${encodeURIComponent(core.siteId)}`, {
+        method: "GET", auth: "bearer", headers: etag ? { "if-none-match": etag } : {},
+      });
+      if (res?.status === 200) {
+        const data = await res.json();
+        rules = compile(data);
+        etag = res.headers.get("etag") || "";
+        wait = rules.sync;
+        if (options.cacheFile) {
+          fs.promises.writeFile(options.cacheFile, JSON.stringify({ etag, rules: data }))
+            .catch((error) => log("could not write cacheFile:", error.message));
+        }
+      }
+    } catch (error) {
+      log("could not fetch rules:", error?.message);
+    }
+    if (!closed) {
+      syncTimer = setTimeout(sync, wait * 1000);
+      syncTimer.unref?.();
+    }
+  }
+
+  // { ip, method, path, userAgent } -> { action: "allow" | "block" |
+  // "monitor", rule, status }. Allowed addresses skip every other rule.
+  function check({ ip, method = "", path = "", userAgent = "" } = {}) {
+    const allow = { action: "allow", rule: "", status: 200 };
+    const current = rules;
+    if (!current?.enabled) return allow;
+    const address = parseIp(ip);
+    if (!address || current.allow.has(address)) return allow;
+    const mode = options.mode === "block" || options.mode === "monitor" ? options.mode : current.mode;
+    const action = mode === "block" ? "block" : "monitor";
+    // A request rule may only watch, even when the site blocks.
+    let effective = action;
+    let rule = current.deny.has(address) ? "deny"
+      : current.banned.has(address) || local.banned(String(ip)) ? "ban"
+        : current.list.has(address) ? "list"
+          : current.shared.has(address) ? "shared" : "";
+    if (!rule && isProbe(path, current.probes)) {
+      rule = "probe";
+      // Asking for a secret file: banned here in block mode; the report bans it on the other servers.
+      if (action === "block") local.ban(String(ip), current.probes.ban);
+    }
+    if (!rule) {
+      const matched = matchRequest(current.requests, { method, path, userAgent });
+      if (matched) {
+        rule = `request:${matched.id}`;
+        effective = action === "block" && matched.action === "block" ? "block" : "monitor";
+        // Banned here straight away; the report bans it on the site's other servers.
+        if (effective === "block" && matched.ban) local.ban(String(ip), matched.ban);
+      }
+    }
+    if (!rule && current.rate && !current.goodbots.has(address) && overRate(String(ip), current.rate)) {
+      rule = "rate";
+      // Banned here straight away in block mode, as a probe is; the report bans it on the other servers.
+      if (action === "block" && current.ban) local.ban(String(ip), current.ban.seconds);
+    }
+    if (!rule) return allow;
+    report({
+      ip: String(ip),
+      rule,
+      action: effective,
+      method: String(method).slice(0, 10),
+      path: String(path).split("?")[0].slice(0, 300),
+      user_agent: String(userAgent).slice(0, 300),
+    });
+    return { action: effective, rule, status: rule === "rate" ? 429 : 403 };
+  }
+
+  // A failed login on your site (req, or the address). Counted here: past the
+  // site's limit the address is banned straight away, and reported, so it
+  // counts on the Security page and gets banned on the other servers too.
+  function loginFailed(reqOrIp) {
+    try {
+      const ip = typeof reqOrIp === "string" ? reqOrIp : clientIp(reqOrIp, options.trustProxy);
+      if (!parseIp(ip) || rules?.allow.has(parseIp(ip))) return;
+      local.loginFailed(String(ip), rules?.login);
+      report({ ip: String(ip), rule: "login", action: "monitor", method: "", path: "", user_agent: "" });
+    } catch (error) {
+      log("could not count a failed login:", error?.message);
+    }
+  }
+
+  // Express / Connect / node:http middleware.
+  function middleware(req, res, next = () => {}) {
+    let decision;
+    try {
+      if (options.skip?.(req)) return next();
+      decision = check({
+        ip: clientIp(req, options.trustProxy),
+        method: req.method,
+        path: req.originalUrl || req.url,
+        userAgent: header(req, "user-agent"),
+      });
+    } catch (error) {
+      log("check failed, request passed:", error?.message);
+      return next();
+    }
+    if (decision.action !== "block") return next();
+    if (options.onBlock) return options.onBlock(req, res, decision);
+    res.statusCode = decision.status;
+    if (decision.status === 429 && rules?.rate) res.setHeader("retry-after", String(Math.ceil(rules.rate.window / 1000)));
+    res.setHeader("content-type", "text/plain; charset=utf-8");
+    res.end(decision.status === 429 ? "Too Many Requests" : "Forbidden");
+  }
+
+  // Stops syncing and sends what's waiting.
+  function close() {
+    closed = true;
+    clearTimeout(syncTimer);
+    return flush();
+  }
+
+  const ready = valid ? sync() : Promise.resolve();
+  return { middleware, check, loginFailed, flush, close, ready, clientIp: (req) => clientIp(req, options.trustProxy) };
+}
+
+module.exports = { createFirewall, clientIp, parseIp, parseRange, IpSet };

package/lib/flags.js

@@ -0,0 +1,41 @@
+// Feature flags module: turn features of your app on or off from the
+// dashboard (Security → Server package), for everyone, for a share of users, or
+// for listed user IDs. isEnabled() answers from memory; the flags come with
+// the module switches (core.js), about once a minute.
+"use strict";
+
+// FNV-1a over "key:user", 0-99: the same user always gets the same answer,
+// in every package (Python and PHP work it out the same way).
+function bucket(key, user) {
+  let hash = 0x811c9dc5;
+  for (const byte of new TextEncoder().encode(`${key}:${user}`)) {
+    hash ^= byte;
+    hash = Math.imul(hash, 0x01000193) >>> 0;
+  }
+  return hash % 100;
+}
+
+function evaluate(flag, key, user) {
+  if (!flag.on) return false;
+  const id = user === undefined || user === null ? "" : String(user);
+  if (id && flag.users?.includes(id)) return true;
+  if (flag.rollout >= 100) return true;
+  if (flag.rollout <= 0 || !id) return false;
+  return bucket(key, id) < flag.rollout;
+}
+
+function createFlags(core) {
+  // key: the flag's key; options: user (for rollouts and user lists),
+  // default (when the flag isn't known yet, or the module is off; false).
+  function isEnabled(key, { user, default: fallback = false } = {}) {
+    try {
+      const flag = core.isOn("flags") ? core.flags()[key] : undefined;
+      return flag ? evaluate(flag, key, user) : fallback;
+    } catch {
+      return fallback;
+    }
+  }
+  return { isEnabled };
+}
+
+module.exports = { createFlags, bucket, evaluate };

package/lib/forms.js

@@ -0,0 +1,81 @@
+// Form protection module: checks a form your server received before you
+// accept it (POST /collectanalytics/shield/verify, the same check as spam
+// protection's). It reads the hidden "analytics-shield" answer the tracker
+// puts in protected forms, and checks the email, name and text for fake
+// sign-ups. Spam lands in the site's spam folder (Leads → Spam). Passwords,
+// card numbers and tokens are never sent. Fails open: when the check can't be
+// made, the form is accepted (checked: false).
+"use strict";
+
+const PRIVATE_FIELD = /pass|card|cvc|cvv|secret|token|ssn|iban|csrf|analytics-shield/i;
+const MAX_FIELDS = 50;
+const ANSWER = "analytics-shield";
+
+function plainFields(body) {
+  const out = {};
+  if (!body || typeof body !== "object") return out;
+  for (const [label, value] of Object.entries(body).slice(0, MAX_FIELDS)) {
+    if (PRIVATE_FIELD.test(label)) continue;
+    if (typeof value === "string") out[label] = value.slice(0, 2000);
+    else if (Array.isArray(value)) out[label] = value.filter((v) => typeof v === "string").join(", ").slice(0, 2000);
+  }
+  return out;
+}
+
+// clientIp(req): the firewall's way of finding the visitor's address.
+function createForms(core, { clientIp } = {}) {
+  const log = (...args) => core.log("forms", ...args);
+  const open = (reason) => ({ ok: true, spam: false, reasons: reason ? [reason] : [], checked: false });
+
+  // req: the request (its body is the form, parsed by your body parser), or
+  // null with options.fields. Options: fields, email, name, form (its name in
+  // the spam folder). Resolves to { ok, spam, reasons, checked }.
+  async function checkForm(req, options = {}) {
+    if (!core.isOn("forms")) return open();
+    try {
+      const body = options.fields || req?.body || {};
+      const token = typeof body[ANSWER] === "string" ? body[ANSWER] : "";
+      const ip = req ? (clientIp ? clientIp(req) : req.ip || req.socket?.remoteAddress || "") : "";
+      const res = await core.send("forms", "/collectanalytics/shield/verify", {
+        body: {
+          token,
+          fields: plainFields(body),
+          email: typeof options.email === "string" ? options.email : undefined,
+          name: typeof options.name === "string" ? options.name : undefined,
+          form: typeof options.form === "string" ? options.form : undefined,
+          ip,
+        },
+      });
+      if (!res || res.status >= 500 || res.status === 429) return open("unchecked");
+      const data = await res.json().catch(() => null);
+      if (res.status === 403) {
+        log("the site secret was refused; forms aren't checked");
+        return open("unchecked");
+      }
+      const spam = data?.success === false;
+      return { ok: !spam, spam, reasons: Array.isArray(data?.reasons) ? data.reasons : [], checked: true };
+    } catch (error) {
+      log("could not check a form:", error?.message);
+      return open("unchecked");
+    }
+  }
+
+  // Express middleware for a form's route (after your body parser). Spam gets
+  // a 422 (or your onSpam(req, res, result)); otherwise req.formCheck holds the
+  // result and the route runs.
+  function protectForm(options = {}) {
+    return async function analyticsProtectForm(req, res, next = () => {}) {
+      const result = await checkForm(req, options);
+      req.formCheck = result;
+      if (!result.spam) return next();
+      if (options.onSpam) return options.onSpam(req, res, result);
+      res.statusCode = 422;
+      res.setHeader("content-type", "text/plain; charset=utf-8");
+      res.end("Your message looks automated, so it wasn't sent. Please try again.");
+    };
+  }
+
+  return { checkForm, protectForm };
+}
+
+module.exports = { createForms, plainFields };

package/lib/headers.js

@@ -0,0 +1,22 @@
+// Security headers module: adds the headers picked on the dashboard's
+// Security page (they come with the module switches, core.js) to every
+// response, unless the app sets that header itself: your own code wins.
+"use strict";
+
+function createHeaders(core) {
+  function middleware(req, res, next = () => {}) {
+    try {
+      if (core.isOn("headers")) {
+        for (const [name, value] of core.headers()) {
+          if (!res.hasHeader(name)) res.setHeader(name, value);
+        }
+      }
+    } catch (error) {
+      core.log("headers", "could not add headers:", error?.message);
+    }
+    next();
+  }
+  return { middleware };
+}
+
+module.exports = { createHeaders };