analytics-platform 0.0.0-stage → 0.2.1

package/lib/posture.js

@@ -0,0 +1,224 @@
+// How the server is set up (protocol in sdk/protocols/posture.md): the app
+// settings that leave it open, whether it runs as root, and the files in the
+// folders it serves to the web that should never be public. Sent to
+// POST /events/posture a few minutes after start and then once a day (the
+// collector's `every`), so Security → Security check can say what to change.
+//
+// It only reads: nothing here changes a setting, a permission or a file, and
+// no file's contents are sent — paths, sizes, times and short values only.
+// The walk is bounded and runs in the background, never in a request; with no
+// folder served to the web there is nothing to walk and only the settings are
+// reported.
+"use strict";
+
+const fs = require("node:fs");
+const os = require("node:os");
+const path = require("node:path");
+
+const FIRST_MS = 3 * 60 * 1000;
+const RETRY_S = 60 * 60;
+const EVERY_S = 24 * 60 * 60;
+const MAX_CHECKS = 200;
+const MAX_EXPOSURE = 50;
+const MAX_ENTRIES = 20000;
+const MAX_DEPTH = 6;
+const MAX_PERM_CHECKS = 3000;
+// Folders that are conventionally served to the web, when the app has them.
+const STATIC_DIRS = ["public", "static"];
+const SKIP_DIRS = new Set([".git", ".svn", ".hg", "node_modules", "vendor", "cache", ".cache", ".next", ".history"]);
+const CODE_FILE = /\.(js|cjs|mjs|jsx|tsx?|php[0-9]?|phtml|py|rb|pl|sh)$/i;
+// Files that shouldn't be downloadable, by name; first match wins.
+const EXPOSURE = [
+  ["database_dump", /\.sql(\.(gz|bz2|xz|zip))?$/i],
+  ["private_key", /^(id_rsa|id_dsa|id_ecdsa|id_ed25519)$|\.(pem|key|p12|pfx|jks)$/i],
+  ["env", /^\.env($|\.)|^env\.(ini|txt)$/i],
+  ["archive", /\.(zip|tar|tgz|tar\.gz|tar\.bz2|rar|7z)$/i],
+  ["backup", /\.(bak|old|orig|save|swp|copy|backup)$|~$|\.(js|json|conf|config|ini|ya?ml)\.(bak|old|orig|save|txt)$/i],
+  ["info", /^(phpinfo|info|adminer)\.php$/i],
+  ["log", /^(debug|error|access)[-_.]?log$|\.log$/i],
+];
+
+const kindOf = (name) => (EXPOSURE.find(([, pattern]) => pattern.test(name)) || [null])[0];
+
+// Each root as { dir, prefix }: `prefix` is where the web serves it, so the
+// path we report is the one the dashboard can ask the site for.
+// A folder inside another folder on the list is already covered: without this,
+// every file under it would be reported twice.
+function withoutNested(roots) {
+  const seen = new Map(roots.map((root) => [root.dir, root]));
+  return [...seen.values()].filter((root, _, all) =>
+    !all.some((other) => other.dir !== root.dir && `${root.dir}/`.startsWith(`${other.dir}/`)));
+}
+
+function rootsOf(given, cwd) {
+  const list = [];
+  for (const entry of Array.isArray(given) ? given : []) {
+    const dir = typeof entry === "string" ? entry : entry?.dir;
+    if (!dir) continue;
+    const prefix = String((typeof entry === "object" && entry.prefix) || "").replace(/^\/+|\/+$/g, "");
+    list.push({ dir: path.resolve(cwd, String(dir)), prefix });
+  }
+  if (list.length) return withoutNested(list);
+  // Not told: the conventional folders, when they're there (Express static,
+  // Next.js public).
+  return withoutNested(STATIC_DIRS
+    .map((name) => ({ dir: path.join(cwd, name), prefix: "" }))
+    .filter(({ dir }) => { try { return fs.statSync(dir).isDirectory(); } catch { return false; } }));
+}
+
+// One bounded walk per root: files that shouldn't be public, and code any
+// account on the machine can write.
+async function walk(roots) {
+  const exposure = [];
+  const examples = [];
+  let writable = 0;
+  let checked = 0;
+  let entries = 0;
+  for (const root of roots) {
+    const queue = [[root.dir, 0]];
+    while (queue.length) {
+      const [dir, depth] = queue.shift();
+      let names;
+      try {
+        names = await fs.promises.readdir(dir, { withFileTypes: true });
+      } catch {
+        continue;
+      }
+      for (const item of names) {
+        if (++entries > MAX_ENTRIES) return { exposure, writable: checked ? writable : null, examples, truncated: true };
+        if (item.isSymbolicLink()) continue;
+        const full = path.join(dir, item.name);
+        const relative = [root.prefix, path.relative(root.dir, full).split(path.sep).join("/")].filter(Boolean).join("/");
+        if (item.isDirectory()) {
+          if (/^\.(git|svn|hg|bzr)$/.test(item.name)) {
+            if (exposure.length < MAX_EXPOSURE) exposure.push({ path: `${relative}/config`, kind: "vcs" });
+            continue;
+          }
+          if (depth + 1 <= MAX_DEPTH && !SKIP_DIRS.has(item.name)) queue.push([full, depth + 1]);
+          continue;
+        }
+        if (!item.isFile()) continue;
+        const kind = kindOf(item.name);
+        if (kind && exposure.length < MAX_EXPOSURE) {
+          const file = { path: relative, kind };
+          try {
+            const info = await fs.promises.stat(full);
+            file.size = info.size;
+            file.mtime = new Date(info.mtimeMs).toISOString();
+          } catch {
+            // reported without its size
+          }
+          exposure.push(file);
+        }
+        if (checked < MAX_PERM_CHECKS && CODE_FILE.test(item.name)) {
+          checked++;
+          try {
+            // The "others may write" bit: any account on the machine can rewrite it.
+            if ((await fs.promises.stat(full)).mode & 0o002) {
+              writable++;
+              if (examples.length < 3) examples.push(relative);
+            }
+          } catch {
+            // left out
+          }
+        }
+      }
+    }
+  }
+  return { exposure, writable: checked ? writable : null, examples, truncated: false };
+}
+
+// What the app itself can say about how it's set up.
+function appChecks() {
+  const checks = [];
+  const add = (id, failed, value) => {
+    if (failed === null) return checks.push({ id, state: "unknown" });
+    checks.push({ id, state: failed ? "fail" : "pass", ...(value ? { value } : {}) });
+  };
+  add("app.node_env", process.env.NODE_ENV !== "production", process.env.NODE_ENV ? `NODE_ENV=${process.env.NODE_ENV}` : "NODE_ENV is not set");
+  // Running the web app as root turns any hole in it into the whole machine.
+  if (typeof process.getuid === "function") {
+    add("server.runs_as_root", process.getuid() === 0);
+  }
+  return checks;
+}
+
+// Options: roots (the folders the web serves, as "public" or
+// { dir, prefix: "static" }; default: ./public and ./static when they exist),
+// host (the server's name; default the host name), checks (a function
+// returning more checks of your own).
+function createPosture(core, options = {}) {
+  const log = (...args) => core.log("posture", ...args);
+  const cwd = String(options.cwd || process.cwd());
+  const roots = rootsOf(options.roots, cwd);
+  const host = String(options.host ?? os.hostname()).replace(/[^A-Za-z0-9._-]/g, "-").slice(0, 100) || "server";
+  let timer = null;
+  let closed = false;
+
+  async function report() {
+    let wait = RETRY_S;
+    try {
+      // The set-up checks ride the software inventory switch.
+      if (core.isOn("inventory")) {
+        const { exposure, writable, examples, truncated } = await walk(roots);
+        const checks = appChecks();
+        if (writable !== null) {
+          checks.push({
+            id: "server.code_writable",
+            state: writable ? "fail" : "pass",
+            ...(writable ? { value: `${writable} files, e.g. ${examples.join(", ")}` } : {}),
+          });
+        }
+        for (const check of typeof options.checks === "function" ? await options.checks() : []) {
+          if (check?.id && ["pass", "fail", "unknown"].includes(check.state) && checks.length < MAX_CHECKS) {
+            checks.push({ id: String(check.id).toLowerCase().slice(0, 60), state: check.state, ...(check.value ? { value: String(check.value).slice(0, 200) } : {}) });
+          }
+        }
+        const res = await core.send("posture", "/events/posture", {
+          auth: "bearer",
+          body: {
+            site_id: core.siteId,
+            host,
+            at: new Date().toISOString(),
+            checks: checks.slice(0, MAX_CHECKS),
+            facts: {
+              agent: `node/${process.versions.node}`,
+              ...(roots.length ? { webroot: roots[0].dir } : {}),
+            },
+            // Left out when there's no folder to walk: the dashboard then says
+            // this wasn't checked, instead of taking an empty walk for a clean one.
+            ...(roots.length ? { exposure: exposure.slice(0, MAX_EXPOSURE) } : {}),
+            ...(truncated ? { truncated: true } : {}),
+          },
+        });
+        if (res?.ok) {
+          const data = await res.json().catch(() => null);
+          wait = Number(data?.every) > 0 ? Math.max(RETRY_S, Number(data.every)) : EVERY_S;
+        }
+      }
+    } catch (error) {
+      log("could not send the set-up checks:", error?.message);
+    }
+    schedule(wait * 1000);
+  }
+
+  function schedule(ms) {
+    if (closed) return;
+    timer = setTimeout(report, ms);
+    timer.unref?.();
+  }
+
+  if (core.valid) schedule(FIRST_MS);
+
+  return {
+    // What would be reported now (for your own checks and tests).
+    report,
+    roots,
+    close() {
+      closed = true;
+      clearTimeout(timer);
+    },
+  };
+}
+
+module.exports = { createPosture, rootsOf, withoutNested, walk, appChecks, kindOf };

package/lib/proxy.js

@@ -0,0 +1,191 @@
+// First-party proxy: serving the tracker from the site's own domain, so ad
+// blockers don't hide visitors.
+//
+// Blockers stop analytics by recognising the address a script comes from and
+// what it is called ("track.js", "collect"). The answer is to serve both from
+// the site's own domain, which until now meant writing Nginx, Cloudflare or
+// Vercel rules by hand. This app is already a server on that domain, so the
+// package does it itself:
+//
+//   <prefix>/js/<rest>   ->  <endpoint>/<rest>                  (tracker files)
+//   <prefix>/api/<rest>  ->  <endpoint>/collectanalytics/<rest> (the collector)
+//
+// The same two rules as the hand-written ones, so one install check covers
+// both. The prefix comes from the site secret: not guessable from the page,
+// and the same every time, so a browser's cache and a CDN in front stay valid.
+//
+// It runs before every other module: a visitor's beacon is our own traffic,
+// not theirs, so it shouldn't be counted as a page of the site, timed as one,
+// or turned away by the firewall's rate limit.
+"use strict";
+
+const crypto = require("node:crypto");
+
+// The tracker files, by name: anything else is refused rather than forwarded.
+const FILES = new Set(["track.js", "s.js", "snapshot.js", "replay.js", "survey.js", "banner.js",
+  "leadform.js", "notice.js", "editor.js", "picker.js", "feedback.js", "shield.js"]);
+// A pinned version never changes; /v<major>/ follows new releases, so it's
+// held for an hour, the same as the collector serves it with.
+const TTL_PINNED_MS = 7 * 24 * 60 * 60 * 1000;
+const TTL_MAJOR_MS = 60 * 60 * 1000;
+const TIMEOUT_MS = 5000;
+const MAX_BODY = 6 * 1024 * 1024;
+// The collector reads these; everything else, cookies above all, stays here.
+const PASS_HEADERS = ["content-type", "user-agent", "accept-language", "referer", "origin",
+  "sec-fetch-site", "sec-ch-ua", "sec-ch-ua-mobile", "sec-ch-ua-platform"];
+
+const prefixFor = (secret) => crypto.createHash("sha256").update(`proxy|${secret}`).digest("hex").slice(0, 8);
+
+// Options: prefix (default: from the secret), enabled (default true).
+function createProxy(core, options = {}) {
+  const enabled = options.enabled !== false;
+  const prefix = String(options.prefix || prefixFor(core.secret)).replace(/[^a-zA-Z0-9_-]/g, "");
+  const clientIp = typeof options.clientIp === "function" ? options.clientIp : null;
+  // One copy of each tracker file, so a repeat visitor costs no upstream
+  // request. A handful of small files: it never grows.
+  const files = new Map();
+
+  const on = () => enabled && core.valid && prefix !== "" && core.isOn("proxy");
+  const scriptUrl = (version = "1") => `/${prefix}/js/v${version}/s.js`;
+  const collectUrl = () => `/${prefix}/api/e`;
+
+  async function serveFile(rest, res) {
+    const match = /^v(\d+(?:\.\d+\.\d+)?)\/([a-z]+\.js)$/.exec(rest);
+    if (!match || !FILES.has(match[2])) return false;
+    const pinned = match[1].includes(".");
+    const held = files.get(rest);
+    let body = held && held.at > Date.now() - (pinned ? TTL_PINNED_MS : TTL_MAJOR_MS) ? held.body : null;
+    if (body === null) {
+      try {
+        const answer = await fetch(`${core.endpoint}/${rest}`, { signal: AbortSignal.timeout(TIMEOUT_MS) });
+        const text = answer.ok ? await answer.text() : "";
+        if (text.length > 500) {
+          body = text;
+          files.set(rest, { body, at: Date.now() });
+        }
+      } catch (error) {
+        core.log("proxy", `could not fetch ${rest}: ${error?.message}`);
+      }
+      // Upstream is unreachable: an old copy beats no tracker at all.
+      if (body === null && held) body = held.body;
+    }
+    if (body === null) {
+      res.writeHead(502).end();
+      return true;
+    }
+    res.writeHead(200, {
+      "content-type": "application/javascript; charset=utf-8",
+      "cache-control": pinned ? "public, max-age=31536000, immutable" : "public, max-age=3600",
+      "x-content-type-options": "nosniff",
+      "content-length": Buffer.byteLength(body),
+    }).end(body);
+    return true;
+  }
+
+  // The request's body, however the app is put together: straight from the
+  // stream, or from whatever a body parser mounted before us already read.
+  function readBody(req) {
+    if (req.readableEnded || req.body !== undefined) {
+      const parsed = req.body;
+      if (parsed === undefined || parsed === null) return Promise.resolve("");
+      if (typeof parsed === "string" || Buffer.isBuffer(parsed)) return Promise.resolve(parsed);
+      return Promise.resolve(JSON.stringify(parsed));
+    }
+    return new Promise((resolve, reject) => {
+      const chunks = [];
+      let size = 0;
+      req.on("data", (chunk) => {
+        size += chunk.length;
+        if (size > MAX_BODY) {
+          req.destroy();
+          reject(new Error("too large"));
+          return;
+        }
+        chunks.push(chunk);
+      });
+      req.on("end", () => resolve(Buffer.concat(chunks)));
+      req.on("error", reject);
+    });
+  }
+
+  async function serveCollector(rest, req, res, search) {
+    const method = String(req.method || "GET").toUpperCase();
+    if (method === "OPTIONS") {
+      // Same origin as the page, so a preflight shouldn't happen; answer it
+      // cheaply rather than passing it on.
+      res.writeHead(204, {
+        "access-control-allow-methods": "GET, POST, OPTIONS",
+        "access-control-allow-headers": "Content-Type",
+        "access-control-max-age": "86400",
+      }).end();
+      return;
+    }
+    if (method !== "GET" && method !== "POST") {
+      res.writeHead(404, { "content-type": "text/plain" }).end("Not found");
+      return;
+    }
+    let body;
+    try {
+      body = method === "POST" ? await readBody(req) : undefined;
+    } catch {
+      res.writeHead(413).end();
+      return;
+    }
+    const headers = {
+      // The visitor's address: the collector needs it for the country, the
+      // daily visitor ID and the per-visitor rate limit.
+      "x-forwarded-for": (clientIp ? clientIp(req) : "") || req.socket?.remoteAddress || "",
+      "x-forwarded-proto": req.socket?.encrypted ? "https" : "http",
+    };
+    for (const name of PASS_HEADERS) {
+      if (req.headers[name]) headers[name] = req.headers[name];
+    }
+    try {
+      const answer = await fetch(`${core.endpoint}/collectanalytics/${rest}${search}`, {
+        method,
+        headers,
+        body,
+        signal: AbortSignal.timeout(TIMEOUT_MS),
+      });
+      const out = {};
+      for (const name of ["content-type", "cache-control", "retry-after"]) {
+        const value = answer.headers.get(name);
+        if (value) out[name] = value;
+      }
+      res.writeHead(answer.status, out).end(Buffer.from(await answer.arrayBuffer()));
+    } catch (error) {
+      // The tracker already treats a failed beacon as lost and carries on.
+      core.log("proxy", `could not reach the collector: ${error?.message}`);
+      res.writeHead(502).end();
+    }
+  }
+
+  // Express / Connect / node:http middleware. Anything that isn't ours is
+  // passed straight on, so the cost to every other request is one compare.
+  function middleware(req, res, next = () => {}) {
+    if (!on()) return next();
+    const url = String(req.originalUrl || req.url || "");
+    if (!url.startsWith(`/${prefix}/`)) return next();
+    const [path, search = ""] = url.split("?");
+    const rest = path.slice(prefix.length + 2);
+    const kind = rest.startsWith("js/") ? "js" : rest.startsWith("api/") ? "api" : "";
+    const tail = rest.slice(kind.length + 1);
+    if (kind === "" || tail === "" || tail.includes("..") || !/^[A-Za-z0-9/_.-]{1,120}$/.test(tail)) {
+      res.writeHead(404, { "content-type": "text/plain" }).end("Not found");
+      return undefined;
+    }
+    const done = kind === "js"
+      ? serveFile(tail, res).then((served) => {
+        if (!served) res.writeHead(404, { "content-type": "text/plain" }).end("Not found");
+      })
+      : serveCollector(tail, req, res, search ? `?${search}` : "");
+    return done.catch((error) => {
+      core.log("proxy", `failed: ${error?.message}`);
+      if (!res.headersSent) res.writeHead(502).end();
+    });
+  }
+
+  return { prefix, scriptUrl, collectUrl, middleware, enabled: on };
+}
+
+module.exports = { createProxy, prefixFor };

package/lib/requestRules.js

@@ -0,0 +1,145 @@
+// Request rules (virtual patching; protocol in sdk/protocols/firewall.md):
+// the firewall has always matched addresses, these match the request — its
+// method, path, query string and user agent — so an attack is turned away
+// before the vulnerable code runs. The rules come with the others, from the
+// collector, and are compiled once per fetch; matching is plain string work on
+// text we already have, so it costs nothing per request.
+//
+// Shared by the Express/node:http middleware and the Next.js edge one, so
+// both decide the same way: nothing here touches the filesystem or the
+// network, which is what the edge runtime allows.
+"use strict";
+
+const MAX_RULES = 200;
+// How much of each part a rule may look at. Patterns come from our rules, but
+// the text comes from whoever is asking, and no regex engine we ship to can be
+// timed out: a bounded slice keeps a bad pattern from being a hung request.
+const MAX_TEXT = 2000;
+const MAX_REGEX = 200;
+
+const lower = (value) => String(value ?? "").slice(0, MAX_TEXT).toLowerCase();
+// One pass only, as the protocol says: enough for %2e%2e, and no risk of
+// decoding a visitor's own text into something that looks like an attack.
+function decodeOnce(text) {
+  if (!text.includes("%") && !text.includes("+")) return text;
+  try {
+    return decodeURIComponent(text.replace(/\+/g, " "));
+  } catch {
+    return text;
+  }
+}
+const normalizePath = (path) => lower(path).replace(/\\/g, "/").replace(/\/{2,}/g, "/");
+
+// A pattern can only come from us, but a careless one would be matched against
+// text an attacker chooses, in this very request, and JavaScript cannot time a
+// regex out. So the client checks the shape itself instead of trusting what it
+// was handed: a quantifier on a group that itself repeats ("(a+)+") takes
+// exponential time on input that nearly matches. Backreferences and lookbehind
+// are refused for the same reason (and they aren't in every engine we ship to).
+function safePattern(pattern) {
+  if (typeof pattern !== "string" || !pattern || pattern.length > MAX_REGEX) return false;
+  if (/\\[1-9]/.test(pattern) || /\(\?</.test(pattern)) return false;
+  const open = [];
+  for (let i = 0; i < pattern.length; i++) {
+    const char = pattern[i];
+    if (char === "\\") {
+      i++;
+      continue;
+    }
+    if (char === "[") {
+      // A character class: quantifiers inside it are literals.
+      while (i < pattern.length && pattern[i] !== "]") i += pattern[i] === "\\" ? 2 : 1;
+      continue;
+    }
+    if (char === "(") open.push(i);
+    else if (char === ")") {
+      const start = open.pop();
+      if (start === undefined) return false;
+      const next = pattern[i + 1];
+      if (next === "*" || next === "+" || next === "{") {
+        const inside = pattern.slice(start + 1, i);
+        if (/(^|[^\\])[*+]/.test(inside) || /(^|[^\\])\{\d/.test(inside)) return false;
+      }
+    }
+  }
+  return open.length === 0;
+}
+
+function compileSection(section) {
+  if (!section || typeof section !== "object") return null;
+  const out = {};
+  for (const matcher of ["exact", "prefix", "suffix", "contains"]) {
+    const list = Array.isArray(section[matcher]) ? section[matcher].filter((s) => typeof s === "string").map(lower) : [];
+    if (list.length) out[matcher] = list;
+  }
+  if (safePattern(section.regex)) {
+    try {
+      out.regex = new RegExp(section.regex, "i");
+    } catch {
+      // A pattern we can't compile is dropped; the rule's other parts stay.
+    }
+  }
+  return Object.keys(out).length ? out : null;
+}
+
+function compileRequestRules(data) {
+  const given = Array.isArray(data?.rules) ? data.rules.slice(0, MAX_RULES) : [];
+  const rules = [];
+  for (const rule of given) {
+    if (!rule || typeof rule.id !== "string" || !rule.id) continue;
+    // The middleware runs before the body is parsed, so we never see it: a
+    // rule that needs the body is skipped whole, not matched on the rest.
+    if (rule.body) continue;
+    const compiled = {
+      id: rule.id,
+      action: rule.action === "block" ? "block" : "monitor",
+      ban: Number(rule.ban) > 0 ? Number(rule.ban) : 0,
+      methods: Array.isArray(rule.methods) && rule.methods.length
+        ? new Set(rule.methods.map((method) => String(method).toUpperCase())) : null,
+      path: compileSection(rule.path),
+      query: compileSection(rule.query),
+      ua: compileSection(rule.ua),
+    };
+    // A rule with nothing to match would match everything.
+    if (!compiled.path && !compiled.query && !compiled.ua) continue;
+    rules.push(compiled);
+  }
+  return rules;
+}
+
+// Any one entry of the section, against the text as it came and decoded once.
+function sectionHits(section, raw, decoded) {
+  const texts = decoded === raw ? [raw] : [raw, decoded];
+  for (const text of texts) {
+    if (section.exact?.includes(text)) return true;
+    if (section.prefix?.some((entry) => text.startsWith(entry))) return true;
+    if (section.suffix?.some((entry) => text.endsWith(entry))) return true;
+    if (section.contains?.some((entry) => text.includes(entry))) return true;
+    if (section.regex?.test(text)) return true;
+  }
+  return false;
+}
+
+// The first rule that matches, or null. `path` is the full target as the
+// server received it, query string and all.
+function matchRequest(rules, { method = "", path = "", userAgent = "" } = {}) {
+  if (!rules || !rules.length) return null;
+  const target = String(path ?? "");
+  const mark = target.indexOf("?");
+  const rawPath = normalizePath(mark === -1 ? target : target.slice(0, mark));
+  const rawQuery = lower(mark === -1 ? "" : target.slice(mark + 1));
+  const ua = lower(userAgent);
+  const decodedPath = normalizePath(decodeOnce(mark === -1 ? target : target.slice(0, mark)));
+  const decodedQuery = decodeOnce(rawQuery);
+  const upper = String(method ?? "").toUpperCase();
+  for (const rule of rules) {
+    if (rule.methods && !rule.methods.has(upper)) continue;
+    if (rule.path && !sectionHits(rule.path, rawPath, decodedPath)) continue;
+    if (rule.query && !sectionHits(rule.query, rawQuery, decodedQuery)) continue;
+    if (rule.ua && !sectionHits(rule.ua, ua, ua)) continue;
+    return rule;
+  }
+  return null;
+}
+
+module.exports = { compileRequestRules, matchRequest, safePattern, MAX_RULES, MAX_TEXT };