analytics-platform 0.0.0-stage → 0.2.2

package/lib/performance.js

@@ -0,0 +1,120 @@
+// Performance & API monitoring module: times every request (from arriving to
+// the response being sent) and sends one row per minute, method, route and
+// status code (how many, total and slowest time, and a count per time bucket)
+// to POST /events/server-requests. Nothing about visitors is sent.
+//
+// The route is the framework's own pattern when there is one (Express
+// "/users/:id"); otherwise numbers and IDs in the path become ":id", and
+// unmatched 404s are one route, so random paths don't make thousands.
+"use strict";
+
+const os = require("node:os");
+
+// Keep in step with the collector (expresscollect/Analytics/serverRequests.js).
+const BUCKET_EDGES_MS = [5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000];
+const FLUSH_MS = 60 * 1000;
+const MAX_KEYS = 300;
+const MAX_ROWS = 500;
+const NOT_FOUND = "/(not found)";
+
+// "/orders/123/items/9f8e..." -> "/orders/:id/items/:id"
+function normalizeRoute(url) {
+  const path = String(url || "/").split(/[?#]/)[0] || "/";
+  const route = path.split("/").map((segment) => (
+    /^\d+$/.test(segment) || /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(segment)
+      || /^[0-9a-f]{16,}$/i.test(segment) || (segment.length >= 20 && /\d/.test(segment))
+      ? ":id" : segment
+  )).join("/");
+  return (route.startsWith("/") ? route : `/${route}`).slice(0, 200);
+}
+
+function bucketOf(ms) {
+  const i = BUCKET_EDGES_MS.findIndex((edge) => ms <= edge);
+  return i === -1 ? BUCKET_EDGES_MS.length : i;
+}
+
+// Options: serverName (default the host name), route(req) (your own route
+// name for a request, or "" to leave it out).
+function createPerformance(core, options = {}) {
+  const log = (...args) => core.log("performance", ...args);
+  const serverName = String(options.serverName ?? os.hostname()).slice(0, 100);
+  const runtime = `Node ${process.versions.node}`;
+  let rows = new Map();
+  const keysIn = new Map(); // minute -> rows it has
+  let sending = Promise.resolve();
+
+  // Records one finished request.
+  function record({ method, route, status, ms }) {
+    if (!core.isOn("performance")) return;
+    const minute = Math.floor(Date.now() / 60000) * 60000;
+    let key = `${minute}|${method}|${route}|${status}`;
+    // The cap is per minute: rows of a minute not yet sent don't count.
+    const used = keysIn.get(minute) || 0;
+    if (!rows.has(key) && used >= MAX_KEYS) key = `${minute}|${method}|(other)|${status}`;
+    let row = rows.get(key);
+    if (!row) {
+      row = { minute, method, route: key.split("|")[2], status, count: 0, sum_ms: 0, max_ms: 0, buckets: new Array(BUCKET_EDGES_MS.length + 1).fill(0) };
+      rows.set(key, row);
+      keysIn.set(minute, used + 1);
+    }
+    row.count += 1;
+    row.sum_ms += ms;
+    if (ms > row.max_ms) row.max_ms = ms;
+    row.buckets[bucketOf(ms)] += 1;
+  }
+
+  function routeOf(req, status) {
+    const own = options.route?.(req);
+    if (own) return String(own).slice(0, 200);
+    if (req.route?.path && typeof req.route.path === "string") return `${req.baseUrl || ""}${req.route.path}`.slice(0, 200);
+    return status === 404 ? NOT_FOUND : normalizeRoute(req.originalUrl || req.url);
+  }
+
+  // Express / Connect / node:http middleware.
+  function middleware(req, res, next = () => {}) {
+    const start = process.hrtime.bigint();
+    res.once("finish", () => {
+      try {
+        const ms = Number(process.hrtime.bigint() - start) / 1e6;
+        const status = res.statusCode || 0;
+        record({ method: String(req.method || "GET").toUpperCase().slice(0, 10), route: routeOf(req, status), status, ms });
+      } catch (error) {
+        log("could not time a request:", error?.message);
+      }
+    });
+    next();
+  }
+
+  // Sends the rows of finished minutes (all of them with all = true).
+  function flush(all = false) {
+    const current = Math.floor(Date.now() / 60000) * 60000;
+    const ready = [];
+    for (const [key, row] of rows) {
+      if (all || row.minute < current) {
+        ready.push({ ...row, minute: new Date(row.minute).toISOString(), sum_ms: Math.round(row.sum_ms * 10) / 10, max_ms: Math.round(row.max_ms * 10) / 10 });
+        rows.delete(key);
+      }
+    }
+    for (const minute of keysIn.keys()) if (all || minute < current) keysIn.delete(minute);
+    for (let i = 0; i < ready.length; i += MAX_ROWS) {
+      const batch = ready.slice(i, i + MAX_ROWS);
+      sending = sending.then(() => core.send("performance", "/events/server-requests", { body: { server_name: serverName, runtime, rows: batch } }));
+    }
+    return sending;
+  }
+
+  const timer = core.valid ? setInterval(() => flush(), FLUSH_MS) : null;
+  timer?.unref?.();
+
+  return {
+    middleware,
+    record,
+    flush: () => flush(true),
+    close() {
+      clearInterval(timer);
+      return flush(true);
+    },
+  };
+}
+
+module.exports = { createPerformance, normalizeRoute, BUCKET_EDGES_MS };

package/lib/posture.js

@@ -0,0 +1,226 @@
+// How the server is set up (protocol in sdk/protocols/posture.md): the app
+// settings that leave it open, whether it runs as root, and the files in the
+// folders it serves to the web that should never be public. Sent to
+// POST /events/posture a few minutes after start and then once a day (the
+// collector's `every`), so Security → Security check can say what to change.
+//
+// It only reads: nothing here changes a setting, a permission or a file, and
+// no file's contents are sent — paths, sizes, times and short values only.
+// The walk is bounded and runs in the background, never in a request; with no
+// folder served to the web there is nothing to walk and only the settings are
+// reported.
+"use strict";
+
+const fs = require("node:fs");
+const os = require("node:os");
+const path = require("node:path");
+
+const FIRST_MS = 3 * 60 * 1000;
+const RETRY_S = 60 * 60;
+const EVERY_S = 24 * 60 * 60;
+const MAX_CHECKS = 200;
+const MAX_EXPOSURE = 50;
+const MAX_ENTRIES = 20000;
+const MAX_DEPTH = 6;
+const MAX_PERM_CHECKS = 3000;
+// Folders that are conventionally served to the web, when the app has them.
+const STATIC_DIRS = ["public", "static"];
+const SKIP_DIRS = new Set([".git", ".svn", ".hg", "node_modules", "vendor", "cache", ".cache", ".next", ".history"]);
+const CODE_FILE = /\.(js|cjs|mjs|jsx|tsx?|php[0-9]?|phtml|py|rb|pl|sh)$/i;
+// Files that shouldn't be downloadable, by name; first match wins.
+const EXPOSURE = [
+  ["database_dump", /\.sql(\.(gz|bz2|xz|zip))?$/i],
+  ["private_key", /^(id_rsa|id_dsa|id_ecdsa|id_ed25519)$|\.(pem|key|p12|pfx|jks)$/i],
+  ["env", /^\.env($|\.)|^env\.(ini|txt)$/i],
+  ["archive", /\.(zip|tar|tgz|tar\.gz|tar\.bz2|rar|7z)$/i],
+  ["backup", /\.(bak|old|orig|save|swp|copy|backup)$|~$|\.(js|json|conf|config|ini|ya?ml)\.(bak|old|orig|save|txt)$/i],
+  ["info", /^(phpinfo|info|adminer)\.php$/i],
+  ["log", /^(debug|error|access)[-_.]?log$|\.log$/i],
+];
+
+const kindOf = (name) => (EXPOSURE.find(([, pattern]) => pattern.test(name)) || [null])[0];
+
+// Each root as { dir, prefix }: `prefix` is where the web serves it, so the
+// path we report is the one the dashboard can ask the site for.
+// A folder inside another folder on the list is already covered: without this,
+// every file under it would be reported twice.
+function withoutNested(roots) {
+  const seen = new Map(roots.map((root) => [root.dir, root]));
+  return [...seen.values()].filter((root, _, all) =>
+    !all.some((other) => other.dir !== root.dir && `${root.dir}/`.startsWith(`${other.dir}/`)));
+}
+
+function rootsOf(given, cwd) {
+  const list = [];
+  for (const entry of Array.isArray(given) ? given : []) {
+    const dir = typeof entry === "string" ? entry : entry?.dir;
+    if (!dir) continue;
+    const prefix = String((typeof entry === "object" && entry.prefix) || "").replace(/^\/+|\/+$/g, "");
+    list.push({ dir: path.resolve(cwd, String(dir)), prefix });
+  }
+  if (list.length) return withoutNested(list);
+  // Not told: the conventional folders, when they're there (Express static,
+  // Next.js public).
+  return withoutNested(STATIC_DIRS
+    .map((name) => ({ dir: path.join(cwd, name), prefix: "" }))
+    .filter(({ dir }) => { try { return fs.statSync(dir).isDirectory(); } catch { return false; } }));
+}
+
+// One bounded walk per root: files that shouldn't be public, and code any
+// account on the machine can write.
+async function walk(roots) {
+  const exposure = [];
+  const examples = [];
+  let writable = 0;
+  let checked = 0;
+  let entries = 0;
+  for (const root of roots) {
+    const queue = [[root.dir, 0]];
+    while (queue.length) {
+      const [dir, depth] = queue.shift();
+      let names;
+      try {
+        names = await fs.promises.readdir(dir, { withFileTypes: true });
+      } catch {
+        continue;
+      }
+      for (const item of names) {
+        if (++entries > MAX_ENTRIES) return { exposure, writable: checked ? writable : null, examples, truncated: true };
+        if (item.isSymbolicLink()) continue;
+        const full = path.join(dir, item.name);
+        const relative = [root.prefix, path.relative(root.dir, full).split(path.sep).join("/")].filter(Boolean).join("/");
+        if (item.isDirectory()) {
+          if (/^\.(git|svn|hg|bzr)$/.test(item.name)) {
+            if (exposure.length < MAX_EXPOSURE) exposure.push({ path: `${relative}/config`, kind: "vcs" });
+            continue;
+          }
+          if (depth + 1 <= MAX_DEPTH && !SKIP_DIRS.has(item.name)) queue.push([full, depth + 1]);
+          continue;
+        }
+        if (!item.isFile()) continue;
+        const kind = kindOf(item.name);
+        if (kind && exposure.length < MAX_EXPOSURE) {
+          const file = { path: relative, kind };
+          try {
+            const info = await fs.promises.stat(full);
+            file.size = info.size;
+            file.mtime = new Date(info.mtimeMs).toISOString();
+          } catch {
+            // reported without its size
+          }
+          exposure.push(file);
+        }
+        if (checked < MAX_PERM_CHECKS && CODE_FILE.test(item.name)) {
+          checked++;
+          try {
+            // The "others may write" bit: any account on the machine can rewrite it.
+            if ((await fs.promises.stat(full)).mode & 0o002) {
+              writable++;
+              if (examples.length < 3) examples.push(relative);
+            }
+          } catch {
+            // left out
+          }
+        }
+      }
+    }
+  }
+  return { exposure, writable: checked ? writable : null, examples, truncated: false };
+}
+
+// What the app itself can say about how it's set up.
+function appChecks() {
+  const checks = [];
+  const add = (id, failed, value) => {
+    if (failed === null) return checks.push({ id, state: "unknown" });
+    checks.push({ id, state: failed ? "fail" : "pass", ...(value ? { value } : {}) });
+  };
+  add("app.node_env", process.env.NODE_ENV !== "production", process.env.NODE_ENV ? `NODE_ENV=${process.env.NODE_ENV}` : "NODE_ENV is not set");
+  // Running the web app as root turns any hole in it into the whole machine.
+  if (typeof process.getuid === "function") {
+    add("server.runs_as_root", process.getuid() === 0);
+  }
+  return checks;
+}
+
+// Options: roots (the folders the web serves, as "public" or
+// { dir, prefix: "static" }; default: ./public and ./static when they exist),
+// host (the server's name; default the host name), checks (a function
+// returning more checks of your own).
+function createPosture(core, options = {}) {
+  const log = (...args) => core.log("posture", ...args);
+  const cwd = String(options.cwd || process.cwd());
+  const roots = rootsOf(options.roots, cwd);
+  const host = String(options.host ?? os.hostname()).replace(/[^A-Za-z0-9._-]/g, "-").slice(0, 100) || "server";
+  let timer = null;
+  let closed = false;
+
+  async function report() {
+    let wait = RETRY_S;
+    try {
+      // The set-up checks ride the software inventory switch.
+      if (core.isOn("inventory")) {
+        const { exposure, writable, examples, truncated } = await walk(roots);
+        const checks = appChecks();
+        if (writable !== null) {
+          checks.push({
+            id: "server.code_writable",
+            state: writable ? "fail" : "pass",
+            ...(writable ? { value: `${writable} files, e.g. ${examples.join(", ")}` } : {}),
+          });
+        }
+        for (const check of typeof options.checks === "function" ? await options.checks() : []) {
+          if (check?.id && ["pass", "fail", "unknown"].includes(check.state) && checks.length < MAX_CHECKS) {
+            checks.push({ id: String(check.id).toLowerCase().slice(0, 60), state: check.state, ...(check.value ? { value: String(check.value).slice(0, 200) } : {}) });
+          }
+        }
+        const res = await core.send("posture", "/events/posture", {
+          auth: "bearer",
+          body: {
+            site_id: core.siteId,
+            host,
+            at: new Date().toISOString(),
+            checks: checks.slice(0, MAX_CHECKS),
+            facts: {
+              agent: `node/${process.versions.node}`,
+              ...(roots.length ? { webroot: roots[0].dir } : {}),
+            },
+            // Left out when there's no folder to walk: the dashboard then says
+            // this wasn't checked, instead of taking an empty walk for a clean one.
+            ...(roots.length ? { exposure: exposure.slice(0, MAX_EXPOSURE) } : {}),
+            ...(truncated ? { truncated: true } : {}),
+          },
+        });
+        if (res?.ok) {
+          const data = await res.json().catch(() => null);
+          wait = Number(data?.every) > 0 ? Math.max(RETRY_S, Number(data.every)) : EVERY_S;
+        }
+      }
+    } catch (error) {
+      log("could not send the set-up checks:", error?.message);
+    }
+    schedule(wait * 1000);
+  }
+
+  function schedule(ms) {
+    if (closed) return;
+    // A report called by hand restarts the wait rather than starting a second chain.
+    clearTimeout(timer);
+    timer = setTimeout(report, ms);
+    timer.unref?.();
+  }
+
+  if (core.valid) schedule(FIRST_MS);
+
+  return {
+    // What would be reported now (for your own checks and tests).
+    report,
+    roots,
+    close() {
+      closed = true;
+      clearTimeout(timer);
+    },
+  };
+}
+
+module.exports = { createPosture, rootsOf, withoutNested, walk, appChecks, kindOf };

package/lib/proxy.js

@@ -0,0 +1,198 @@
+// First-party proxy: serving the tracker from the site's own domain, so ad
+// blockers don't hide visitors.
+//
+// Blockers stop analytics by recognising the address a script comes from and
+// what it is called ("track.js", "collect"). The answer is to serve both from
+// the site's own domain, which until now meant writing Nginx, Cloudflare or
+// Vercel rules by hand. This app is already a server on that domain, so the
+// package does it itself:
+//
+//   <prefix>/js/<rest>   ->  <endpoint>/<rest>                  (tracker files)
+//   <prefix>/api/<rest>  ->  <endpoint>/collectanalytics/<rest> (the collector)
+//
+// The same two rules as the hand-written ones, so one install check covers
+// both. The prefix comes from the site secret: not guessable from the page,
+// and the same every time, so a browser's cache and a CDN in front stay valid.
+//
+// It runs before every other module: a visitor's beacon is our own traffic,
+// not theirs, so it shouldn't be counted as a page of the site, timed as one,
+// or turned away by the firewall's rate limit.
+"use strict";
+
+const crypto = require("node:crypto");
+
+// The tracker files, by name: anything else is refused rather than forwarded.
+const FILES = new Set(["track.js", "s.js", "snapshot.js", "replay.js", "survey.js", "banner.js",
+  "leadform.js", "notice.js", "editor.js", "picker.js", "feedback.js", "shield.js", "widget.js", "share.js"]);
+// A pinned version never changes; /v<major>/ follows new releases, so it's
+// held for an hour, the same as the collector serves it with.
+const TTL_PINNED_MS = 7 * 24 * 60 * 60 * 1000;
+const TTL_MAJOR_MS = 60 * 60 * 1000;
+const TIMEOUT_MS = 5000;
+const MAX_BODY = 6 * 1024 * 1024;
+// The collector reads these; everything else, cookies above all, stays here.
+const PASS_HEADERS = ["content-type", "user-agent", "accept-language", "referer", "origin",
+  "sec-fetch-site", "sec-ch-ua", "sec-ch-ua-mobile", "sec-ch-ua-platform"];
+
+const isEmptyObject = (value) => value !== null && typeof value === "object" && !Buffer.isBuffer(value)
+  && Object.getPrototypeOf(value) === Object.prototype && Object.keys(value).length === 0;
+
+const prefixFor = (secret) => crypto.createHash("sha256").update(`proxy|${secret}`).digest("hex").slice(0, 8);
+
+// Options: prefix (default: from the secret), enabled (default true).
+function createProxy(core, options = {}) {
+  const enabled = options.enabled !== false;
+  const prefix = String(options.prefix || prefixFor(core.secret)).replace(/[^a-zA-Z0-9_-]/g, "");
+  const clientIp = typeof options.clientIp === "function" ? options.clientIp : null;
+  // One copy of each tracker file, so a repeat visitor costs no upstream
+  // request. A handful of small files: it never grows.
+  const files = new Map();
+
+  const on = () => enabled && core.valid && prefix !== "" && core.isOn("proxy");
+  const scriptUrl = (version = "1") => `/${prefix}/js/v${version}/s.js`;
+  const collectUrl = () => `/${prefix}/api/e`;
+
+  async function serveFile(rest, res) {
+    const match = /^v(\d+(?:\.\d+\.\d+)?)\/([a-z]+\.js)$/.exec(rest);
+    if (!match || !FILES.has(match[2])) return false;
+    const pinned = match[1].includes(".");
+    const held = files.get(rest);
+    let body = held && held.at > Date.now() - (pinned ? TTL_PINNED_MS : TTL_MAJOR_MS) ? held.body : null;
+    if (body === null) {
+      try {
+        const answer = await fetch(`${core.endpoint}/${rest}`, { signal: AbortSignal.timeout(TIMEOUT_MS) });
+        const text = answer.ok ? await answer.text() : "";
+        if (text.length > 500) {
+          body = text;
+          files.set(rest, { body, at: Date.now() });
+        }
+      } catch (error) {
+        core.log("proxy", `could not fetch ${rest}: ${error?.message}`);
+      }
+      // Upstream is unreachable: an old copy beats no tracker at all.
+      if (body === null && held) body = held.body;
+    }
+    if (body === null) {
+      res.writeHead(502).end();
+      return true;
+    }
+    res.writeHead(200, {
+      "content-type": "application/javascript; charset=utf-8",
+      "cache-control": pinned ? "public, max-age=31536000, immutable" : "public, max-age=3600",
+      "x-content-type-options": "nosniff",
+      "content-length": Buffer.byteLength(body),
+    }).end(body);
+    return true;
+  }
+
+  // The request's body, however the app is put together: straight from the
+  // stream, or from whatever a body parser mounted before us already read.
+  // Express 4's parsers set req.body = {} even for a type they skip (the
+  // tracker's text/plain), leaving the stream unread: that empty object isn't
+  // the body. req._body is how they mark one they did read.
+  function readBody(req) {
+    const skipped = req._body !== true && isEmptyObject(req.body);
+    if (req.readableEnded || (req.body !== undefined && !skipped)) {
+      const parsed = req.body;
+      if (parsed === undefined || parsed === null) return Promise.resolve("");
+      if (typeof parsed === "string" || Buffer.isBuffer(parsed)) return Promise.resolve(parsed);
+      return Promise.resolve(JSON.stringify(parsed));
+    }
+    return new Promise((resolve, reject) => {
+      const chunks = [];
+      let size = 0;
+      req.on("data", (chunk) => {
+        size += chunk.length;
+        if (size > MAX_BODY) {
+          req.destroy();
+          reject(new Error("too large"));
+          return;
+        }
+        chunks.push(chunk);
+      });
+      req.on("end", () => resolve(Buffer.concat(chunks)));
+      req.on("error", reject);
+    });
+  }
+
+  async function serveCollector(rest, req, res, search) {
+    const method = String(req.method || "GET").toUpperCase();
+    if (method === "OPTIONS") {
+      // Same origin as the page, so a preflight shouldn't happen; answer it
+      // cheaply rather than passing it on.
+      res.writeHead(204, {
+        "access-control-allow-methods": "GET, POST, OPTIONS",
+        "access-control-allow-headers": "Content-Type",
+        "access-control-max-age": "86400",
+      }).end();
+      return;
+    }
+    if (method !== "GET" && method !== "POST") {
+      res.writeHead(404, { "content-type": "text/plain" }).end("Not found");
+      return;
+    }
+    let body;
+    try {
+      body = method === "POST" ? await readBody(req) : undefined;
+    } catch {
+      res.writeHead(413).end();
+      return;
+    }
+    const headers = {
+      // The visitor's address: the collector needs it for the country, the
+      // daily visitor ID and the per-visitor rate limit.
+      "x-forwarded-for": (clientIp ? clientIp(req) : req.ip) || req.socket?.remoteAddress || "",
+      "x-forwarded-proto": req.socket?.encrypted ? "https" : "http",
+    };
+    for (const name of PASS_HEADERS) {
+      if (req.headers[name]) headers[name] = req.headers[name];
+    }
+    try {
+      const answer = await fetch(`${core.endpoint}/collectanalytics/${rest}${search}`, {
+        method,
+        headers,
+        body,
+        signal: AbortSignal.timeout(TIMEOUT_MS),
+      });
+      const out = {};
+      for (const name of ["content-type", "cache-control", "retry-after"]) {
+        const value = answer.headers.get(name);
+        if (value) out[name] = value;
+      }
+      res.writeHead(answer.status, out).end(Buffer.from(await answer.arrayBuffer()));
+    } catch (error) {
+      // The tracker already treats a failed beacon as lost and carries on.
+      core.log("proxy", `could not reach the collector: ${error?.message}`);
+      res.writeHead(502).end();
+    }
+  }
+
+  // Express / Connect / node:http middleware. Anything that isn't ours is
+  // passed straight on, so the cost to every other request is one compare.
+  function middleware(req, res, next = () => {}) {
+    if (!on()) return next();
+    const url = String(req.originalUrl || req.url || "");
+    if (!url.startsWith(`/${prefix}/`)) return next();
+    const [path, search = ""] = url.split("?");
+    const rest = path.slice(prefix.length + 2);
+    const kind = rest.startsWith("js/") ? "js" : rest.startsWith("api/") ? "api" : "";
+    const tail = rest.slice(kind.length + 1);
+    if (kind === "" || tail === "" || tail.includes("..") || !/^[A-Za-z0-9/_.-]{1,120}$/.test(tail)) {
+      res.writeHead(404, { "content-type": "text/plain" }).end("Not found");
+      return undefined;
+    }
+    const done = kind === "js"
+      ? serveFile(tail, res).then((served) => {
+        if (!served) res.writeHead(404, { "content-type": "text/plain" }).end("Not found");
+      })
+      : serveCollector(tail, req, res, search ? `?${search}` : "");
+    return done.catch((error) => {
+      core.log("proxy", `failed: ${error?.message}`);
+      if (!res.headersSent) res.writeHead(502).end();
+    });
+  }
+
+  return { prefix, scriptUrl, collectUrl, middleware, enabled: on };
+}
+
+module.exports = { createProxy, prefixFor };