hearald 0.0.4 → 0.1.0

package/src/server/analytics.js

@@ -1,17 +1,44 @@
 // Analytics router and helpers for hearald
 import chalk from "chalk";
 import path from "path";
+import { createHash } from "crypto";
 import { fileURLToPath } from "url";
-import { readFile, writeFile } from "fs/promises";
+import { readFile, writeFile, mkdir } from "fs/promises";
 import express from "express";
 import log from "./logger.js";
-import {
-  omitProperties,
-  iso,
-  defaultEndDate,
-  nowAsPstDate,
-} from "../shared/utils.js";
-import { loadEventsOverWindow } from "./serverEvents.js";
+import hearaldConfiguration from "./configuration.js";
+import { omitProperties, defaultEndDate, iso } from "../shared/utils.js";
+import { loadEventsOverWindow, windowBytes } from "./serverEvents.js";
+import { asyncHandler } from "./asyncHandler.js";
+
+const MAX_TIME_WINDOW_DAYS = 366;
+
+// Bump when the way numbers are computed or stored changes, so summaries cached by an older version
+// are not served.
+const CACHE_FORMAT_VERSION = 1;
+
+// Thrown when a window's event files are bigger than the memory budget.
+export class WindowTooLargeError extends Error {
+  constructor(bytes, limit) {
+    super(`window too large (${bytes} bytes of events, the limit is ${limit})`);
+    this.name = "WindowTooLargeError";
+  }
+}
+
+// Computing stats holds a whole window of events in memory, so run one computation at a time: with
+// several at once, memory use multiplies.
+let queue = Promise.resolve();
+export function runExclusive(task) {
+  const result = queue.then(task);
+  queue = result.catch(() => {});
+  return result;
+}
+
+// A histogram value is a string, number or boolean. 0, false and "" are real values.
+const histogramValue = (params, field) => {
+  const value = params ? params[field] : undefined;
+  return ["string", "number", "boolean"].includes(typeof value) ? value : null;
+};
 
 export function generateRawMetrics(events, rawMetricDefinitions) {
   const metrics = {};
@@ -22,12 +49,14 @@ export function generateRawMetrics(events, rawMetricDefinitions) {
       } else if (rawMetricDefinition.aggregator === "set_of_users") {
         metrics[name] = new Set();
       } else if (rawMetricDefinition.aggregator === "histogram") {
-        metrics[name] = {};
+        // no prototype, so a value named "constructor" or "__proto__" is just a key
+        metrics[name] = Object.create(null);
       }
     }
   );
 
-  events.forEach(({ e, p, u }) => {
+  events.forEach(({ e, p: eventParams, u }) => {
+    const p = eventParams || {};
     Object.entries(rawMetricDefinitions).forEach(
       ([name, rawMetricDefinition]) => {
         if (rawMetricDefinition.filter) {
@@ -36,19 +65,26 @@ export function generateRawMetrics(events, rawMetricDefinitions) {
           );
           if (!match) return;
         }
+        // A set_of_users metric that names an event counts only users who sent that event.
+        if (
+          rawMetricDefinition.aggregator === "set_of_users" &&
+          rawMetricDefinition.event &&
+          rawMetricDefinition.event !== e
+        ) {
+          return;
+        }
         if (
           rawMetricDefinition.aggregator === "count" &&
           rawMetricDefinition.event === e
         ) {
           metrics[name] += 1;
         } else if (rawMetricDefinition.aggregator === "set_of_users") {
-          metrics[name].add(u);
+          if (u !== null && u !== undefined) metrics[name].add(u);
         } else if (
           rawMetricDefinition.aggregator === "histogram" &&
           rawMetricDefinition.event === e
         ) {
-          const field = rawMetricDefinition.field;
-          const value = p && p[field] ? p[field] : null;
+          const value = histogramValue(p, rawMetricDefinition.field);
           if (value !== null) {
             metrics[name][value] = (metrics[name][value] || 0) + 1;
           }
@@ -92,38 +128,70 @@ export function generateComputedMetric(metrics, definition, timeWindow) {
       // Compute histogram for given event type and field
       const sourceEvent = definition.source;
       const field = definition.field;
-      const histogram = {};
+      const histogram = Object.create(null);
       // Find all events of the given type
       (metrics._allEvents || []).forEach((ev) => {
-        if (ev.e === sourceEvent && ev.p && ev.p[field]) {
-          const value = ev.p[field];
+        const value = ev.e === sourceEvent ? histogramValue(ev.p, field) : null;
+        if (value !== null) {
           histogram[value] = (histogram[value] || 0) + 1;
         }
       });
-      return histogram;
+      // back to a plain object (fromEntries keeps a "__proto__" key as an ordinary key)
+      return Object.fromEntries(Object.entries(histogram));
     }
+    default:
+      throw new Error(`hearald: unknown formula "${definition.formula}"`);
   }
 }
 
-export function cacheFilePath(endDate, timeWindow) {
-  const endDateIso = typeof endDate === "string" ? endDate : iso(endDate);
-  return `events/summary:${endDateIso}:${timeWindow}.json`;
+// A short fingerprint of what the numbers depend on: the metric definitions (not `ui`) and the
+// format version. Changing a definition must not serve summaries computed with the old one.
+export function cacheKey(statDefinitions) {
+  return createHash("sha1")
+    .update(
+      JSON.stringify([
+        CACHE_FORMAT_VERSION,
+        statDefinitions.raw_metrics,
+        statDefinitions.computed_metrics,
+      ])
+    )
+    .digest("hex")
+    .slice(0, 8);
+}
+
+export function cacheFilePath(endDate, timeWindow, statDefinitions) {
+  const endDateIso =
+    typeof endDate === "string" ? endDate : hearaldConfiguration.isoFn(endDate);
+  return path.join(
+    hearaldConfiguration.eventsDir,
+    `summary:${endDateIso}:${timeWindow}:${cacheKey(statDefinitions)}.json`
+  );
 }
 
-export async function readMetricsFromCache(endDateIso, timeWindow) {
-  const filePath = cacheFilePath(endDateIso, timeWindow);
+export async function readMetricsFromCache(
+  endDateIso,
+  timeWindow,
+  statDefinitions
+) {
+  const filePath = cacheFilePath(endDateIso, timeWindow, statDefinitions);
   try {
     return (await readFile(filePath)).toString("utf8");
   } catch (e) {
-    if (e.message.indexOf("ENOENT") !== -1) {
+    if (e.code === "ENOENT") {
       return null;
     }
     throw e;
   }
 }
 
-export async function writeMetricsToCache(endDateIso, timeWindow, statsJson) {
-  const filePath = cacheFilePath(endDateIso, timeWindow);
+export async function writeMetricsToCache(
+  endDateIso,
+  timeWindow,
+  statsJson,
+  statDefinitions
+) {
+  const filePath = cacheFilePath(endDateIso, timeWindow, statDefinitions);
+  await mkdir(hearaldConfiguration.eventsDir, { recursive: true });
   return writeFile(filePath, statsJson);
 }
 
@@ -137,8 +205,17 @@ export async function generateStats(statDefinitions, endDate, timeWindow) {
     if (def.source_file_suffix) fileSuffixes.add(def.source_file_suffix);
   });
 
+  // Refuse a window too big for memory before reading any of it
+  let bytes = 0;
+  for (const suffix of fileSuffixes) {
+    bytes += await windowBytes(endDate, timeWindow, suffix || undefined);
+  }
+  if (bytes > hearaldConfiguration.maxWindowBytes) {
+    throw new WindowTooLargeError(bytes, hearaldConfiguration.maxWindowBytes);
+  }
+
   // Load events from all relevant files for raw metrics
-  let allEvents = [];
+  let processedEventCount = 0;
   const eventsBySuffix = {};
   for (const suffix of fileSuffixes) {
     const events = await loadEventsOverWindow(
@@ -146,7 +223,7 @@ export async function generateStats(statDefinitions, endDate, timeWindow) {
       timeWindow,
       suffix || undefined
     );
-    allEvents = allEvents.concat(events);
+    processedEventCount += events.length;
     eventsBySuffix[suffix || ""] = events;
   }
 
@@ -159,8 +236,18 @@ export async function generateStats(statDefinitions, endDate, timeWindow) {
     ];
   });
 
-  // Store all events for computed metrics
-  metrics._allEvents = allEvents;
+  // Only a computed histogram reads the raw events, so only then keep them all together
+  if (
+    Object.values(computedMetricDefinitions).some(
+      (def) => def.formula === "histogram"
+    )
+  ) {
+    const allEvents = [];
+    for (const events of Object.values(eventsBySuffix)) {
+      for (const event of events) allEvents.push(event);
+    }
+    metrics._allEvents = allEvents;
+  }
 
   let anyComputed = true;
   while (anyComputed) {
@@ -182,7 +269,7 @@ export async function generateStats(statDefinitions, endDate, timeWindow) {
           computedMetricDefinition,
           timeWindow
         );
-        anyComputed = true;
+        anyComputed = typeof metrics[name] !== "undefined";
       }
     }
   }
@@ -213,9 +300,9 @@ export async function generateStats(statDefinitions, endDate, timeWindow) {
   });
   return {
     metrics: nestedMetrics,
-    endDate: iso(endDate),
+    endDate: hearaldConfiguration.isoFn(endDate),
     timeWindow,
-    processedEventCount: allEvents.length,
+    processedEventCount,
   };
 }
 
@@ -224,39 +311,81 @@ export async function handleAnalyticsDataRequest({
   response,
   statDefinitions,
 }) {
-  const defaultEndDateIso = iso(defaultEndDate(nowAsPstDate));
+  const defaultEndDateIso = hearaldConfiguration.isoFn(
+    defaultEndDate(hearaldConfiguration.nowFn())
+  );
   const endDateIso = request.params.date
     ? request.params.date
     : defaultEndDateIso;
-  const timeWindow = parseInt(request.params.last, 10);
-  if (!timeWindow) {
-    response.status(500).json({ Error: "invalid time window" });
+  // The date becomes part of a file name, so it must be a real calendar date and nothing else. The
+  // round trip rejects dates like 2026-02-31, which Date would quietly move to March.
+  if (
+    !/^\d{4}-\d{2}-\d{2}$/.test(endDateIso) ||
+    Number.isNaN(new Date(`${endDateIso}T00:00:00Z`).getTime()) ||
+    iso(new Date(`${endDateIso}T00:00:00Z`)) !== endDateIso
+  ) {
+    response.status(400).json({ error: "invalid date" });
     return;
   }
-  const dataFromFile = await readMetricsFromCache(endDateIso, timeWindow);
-  if (dataFromFile) {
+  const timeWindow = /^\d+$/.test(String(request.params.last))
+    ? parseInt(request.params.last, 10)
+    : 0;
+  if (!timeWindow || timeWindow > MAX_TIME_WINDOW_DAYS) {
+    response.status(400).json({ error: "invalid time window" });
+    return;
+  }
+  // A window that ends today (or later) is still filling up, so it is never cached.
+  const cacheable =
+    endDateIso < hearaldConfiguration.isoFn(hearaldConfiguration.nowFn());
+  const sendCached = (json) => {
     log.info(
       `Analytics for ${timeWindow} days ending on ${endDateIso} returned from cache`
     );
-    response.status(200).type("application/json").send(dataFromFile);
+    response.status(200).type("application/json").send(json);
+  };
+  const dataFromFile = cacheable
+    ? await readMetricsFromCache(endDateIso, timeWindow, statDefinitions)
+    : null;
+  if (dataFromFile) {
+    sendCached(dataFromFile);
     return;
   }
-  log.info(
-    `Analytics for ${timeWindow} days ending on ${endDateIso} computing from events`
-  );
   try {
-    const stats = await generateStats(
-      statDefinitions,
-      new Date(endDateIso),
-      timeWindow
-    );
-    await writeMetricsToCache(
-      endDateIso,
-      timeWindow,
-      JSON.stringify(stats, null, 2)
-    );
-    response.status(200).type("application/json").send(JSON.stringify(stats));
+    const stats = await runExclusive(async () => {
+      // someone ahead of us in the queue may just have computed this same window
+      const cached = cacheable
+        ? await readMetricsFromCache(endDateIso, timeWindow, statDefinitions)
+        : null;
+      if (cached) return { cached };
+      log.info(
+        `Analytics for ${timeWindow} days ending on ${endDateIso} computing from events`
+      );
+      const computed = await generateStats(
+        statDefinitions,
+        new Date(`${endDateIso}T00:00:00Z`),
+        timeWindow
+      );
+      if (cacheable) {
+        await writeMetricsToCache(
+          endDateIso,
+          timeWindow,
+          JSON.stringify(computed, null, 2),
+          statDefinitions
+        );
+      }
+      return { computed };
+    });
+    if (stats.cached) {
+      sendCached(stats.cached);
+      return;
+    }
+    response.status(200).type("application/json").send(JSON.stringify(stats.computed));
   } catch (e) {
+    if (e instanceof WindowTooLargeError) {
+      log.warn(e.message);
+      response.status(413).json({ error: "window too large" });
+      return;
+    }
     log.error("failed to compute analytics", e);
     response
       .status(500)
@@ -271,32 +400,56 @@ const __dirname = path.dirname(__filename);
 log.info("filename is", __filename);
 log.info("dirname is", __dirname);
 
-export function createAnalyticsRouter({ statDefinitions }) {
+/**
+ * @param {Object} options
+ * @param {Object} options.statDefinitions
+ * @param {function} options.auth - Express middleware that rejects anyone who may not see the stats
+ *   (for example basicAuth from hearald). Required: the dashboard shows your usage numbers.
+ * @param {boolean} [options.allowUnauthenticated=false] - Skip `auth`. Only for local development.
+ */
+export function createAnalyticsRouter({
+  statDefinitions,
+  auth,
+  allowUnauthenticated = false,
+}) {
+  if (typeof auth !== "function" && !allowUnauthenticated) {
+    throw new Error(
+      "hearald: the analytics dashboard needs an `auth` middleware (see basicAuth), " +
+        "or allowUnauthenticated: true for local development"
+    );
+  }
   log.info("statManifest", statDefinitions);
   const router = express.Router();
+  if (typeof auth === "function") router.use(auth);
 
-  router.get(`/data/:last/:date?`, async (request, response) => {
-    log.info(chalk.cyan("[API] GET /analytics/data/:last/:date?"));
-    await handleAnalyticsDataRequest({
-      request,
-      response,
-      statDefinitions,
-      loadEventsOverWindow,
-      nowAsPstDate,
-    });
-  });
+  // Two routes rather than an optional `:date?`, which Express 5 (path-to-regexp 8) rejects.
+  // asyncHandler sends a failed read to the error handler; Express 4 would crash on it.
+  router.get(
+    ["/data/:last", "/data/:last/:date"],
+    asyncHandler(async (request, response) => {
+      log.info(chalk.cyan("[API] GET /analytics/data/:last/:date?"));
+      await handleAnalyticsDataRequest({
+        request,
+        response,
+        statDefinitions,
+      });
+    })
+  );
 
-  router.get(`/layout`, async (req, res) => {
+  router.get(`/layout`, (req, res) => {
     res.json(statDefinitions.ui || {});
   });
 
-  router.get(`/`, async (request, response) => {
-    const fileContents = await readFile(
-      path.join(__dirname, "..", "..", "dist", "analytics.html")
-    );
-    const analyticsHtml = fileContents.toString("utf-8");
-    response.send(analyticsHtml);
-  });
+  router.get(
+    `/`,
+    asyncHandler(async (request, response) => {
+      const fileContents = await readFile(
+        path.join(__dirname, "..", "..", "dist", "analytics.html")
+      );
+      const analyticsHtml = fileContents.toString("utf-8");
+      response.send(analyticsHtml);
+    })
+  );
 
   return router;
 }

package/src/server/asyncHandler.js

@@ -0,0 +1,6 @@
+// Express 4 does not catch a rejected promise from an async handler, so one failed read would be an
+// unhandled rejection that kills the process. Express 5 does, but hearald supports both.
+export const asyncHandler = (handler) => (req, res, next) =>
+  Promise.resolve()
+    .then(() => handler(req, res, next))
+    .catch(next);

package/src/server/auth.js

@@ -0,0 +1,89 @@
+import { timingSafeEqual } from "crypto";
+import log from "./logger.js";
+import { asyncHandler } from "./asyncHandler.js";
+
+// Constant-time string comparison.
+export function safeEqual(a, b) {
+  const x = Buffer.from(String(a));
+  const y = Buffer.from(String(b));
+  return x.length === y.length && timingSafeEqual(x, y);
+}
+
+/**
+ * HTTP Basic authentication for the analytics dashboard.
+ *
+ * Wrong guesses are counted per `req.ip`. That is the caller's address only when your app sets
+ * Express's `trust proxy` to match your setup: behind a reverse proxy without it, every visitor
+ * shares the proxy's address (one attacker locks everybody out), and with a `trust proxy` that
+ * trusts too much a caller can send a forged X-Forwarded-For to dodge the lockout.
+ *
+ * @param {Object} options
+ * @param {(username: string, password: string) => boolean | Promise<boolean>} options.verify
+ *   Decides whether the credentials are right. Compare with a constant-time function, and store a
+ *   hash of the password, never the password. If it throws, the request gets a 500 (and is not
+ *   counted as a wrong guess).
+ * @param {string} [options.realm="hearald"]
+ * @param {number} [options.maxFailures=10] wrong guesses allowed per IP in the window, before 429
+ * @param {number} [options.windowMs=900000] the window, 15 minutes by default
+ * @param {number} [options.maxTrackedIps=10000] the most addresses remembered at once, so a flood of
+ *   distinct addresses cannot grow memory without limit (the oldest are forgotten first)
+ */
+export function basicAuth({ verify, realm = "hearald", maxFailures = 10, windowMs = 15 * 60 * 1000, maxTrackedIps = 10000, now = Date.now } = {}) {
+  if (typeof verify !== "function") throw new Error("basicAuth needs a verify(username, password) function");
+  const failures = new Map(); // ip -> { count, resetAt }, oldest first
+
+  function prune(t) {
+    for (const [ip, entry] of failures) {
+      if (entry.resetAt <= t) failures.delete(ip);
+    }
+    // Still too many (a burst inside one window): forget the oldest
+    for (const ip of failures.keys()) {
+      if (failures.size < maxTrackedIps) break;
+      failures.delete(ip);
+    }
+  }
+
+  return asyncHandler(async function basicAuthMiddleware(req, res, next) {
+    const ip = req.ip || "unknown";
+    const t = now();
+    let entry = failures.get(ip);
+    if (entry && entry.resetAt <= t) {
+      failures.delete(ip);
+      entry = undefined;
+    }
+    if (entry && entry.count >= maxFailures) {
+      res.set("Retry-After", String(Math.ceil((entry.resetAt - t) / 1000)));
+      return res.status(429).send("Too many attempts");
+    }
+
+    // The scheme name is case-insensitive (RFC 7235)
+    const match = /^basic\s+([A-Za-z0-9+/=]+)\s*$/i.exec(req.get("authorization") || "");
+    if (match) {
+      // Count the attempt now, before the (possibly slow) verify: if it were counted afterwards,
+      // any number of parallel guesses would all be checked against a count of zero.
+      if (!entry) {
+        prune(t);
+        entry = { count: 0, resetAt: t + windowMs };
+        failures.set(ip, entry);
+      }
+      entry.count += 1;
+
+      const decoded = Buffer.from(match[1], "base64").toString("utf8");
+      const colon = decoded.indexOf(":");
+      let ok = false;
+      try {
+        ok = colon > -1 && (await verify(decoded.slice(0, colon), decoded.slice(colon + 1)));
+      } catch (err) {
+        entry.count -= 1; // a broken verify is not the caller's wrong guess
+        log.error("basicAuth: verify threw", err);
+        return res.status(500).send("Authentication error");
+      }
+      if (ok) {
+        failures.delete(ip);
+        return next();
+      }
+    }
+    res.set("WWW-Authenticate", `Basic realm="${realm}", charset="UTF-8"`);
+    return res.status(401).send("Authentication required");
+  });
+}

package/src/server/configuration.js

@@ -0,0 +1,53 @@
+import { iso, nowAsPstDate } from "../shared/utils.js";
+
+class Configuration {
+  constructor({
+    nowFn = nowAsPstDate,
+    isoFn = iso,
+    logLevel = "warn",
+    getUserId = (req) => req.user?.id,
+    eventsDir = "events",
+    idleCloseMs = 60 * 1000,
+    maxWindowBytes = 64 * 1024 * 1024,
+  } = {}) {
+    this.nowFn = nowFn;
+    this.isoFn = isoFn;
+    this.logLevel = logLevel;
+    this.getUserId = getUserId;
+    this.eventsDir = eventsDir;
+    this.idleCloseMs = idleCloseMs;
+    this.maxWindowBytes = maxWindowBytes;
+  }
+
+  setLogLevel(level) {
+    this.logLevel = level;
+  }
+
+  setNowFn(nowFn) {
+    this.nowFn = nowFn;
+  }
+
+  setGetUserId(getUserId) {
+    this.getUserId = getUserId;
+  }
+
+  setEventsDir(eventsDir) {
+    this.eventsDir = eventsDir;
+  }
+
+  // How long an event file stays open after its last write
+  setIdleCloseMs(idleCloseMs) {
+    this.idleCloseMs = idleCloseMs;
+  }
+
+  // The most event-log bytes one dashboard request may read
+  setMaxWindowBytes(maxWindowBytes) {
+    this.maxWindowBytes = maxWindowBytes;
+  }
+
+  setIsoFn(isoFn) {
+    this.isoFn = isoFn;
+  }
+}
+
+export default new Configuration();

package/src/server/logger.js

@@ -1,28 +1,27 @@
+import hearaldConfiguration from "./configuration.js";
+
+const LEVELS = { debug: 0, info: 1, warn: 2, error: 3 };
+
 class Logger {
-  constructor(level) {
-    this.level = level || "info";
+  constructor(config = hearaldConfiguration) {
+    this.config = config;
   }
 
-  setLogLevel(level) {
-    this.level = level;
+  enabled(level) {
+    const configured = LEVELS[this.config.logLevel] ?? LEVELS.warn;
+    return LEVELS[level] >= configured;
   }
 
   debug(...args) {
-    if (this.level === "debug") {
-      console.log("[DEBUG]", ...args);
-    }
+    if (this.enabled("debug")) console.log("[DEBUG]", ...args);
   }
 
   info(...args) {
-    if (this.level === "debug" || this.level === "info") {
-      console.log("[INFO]", ...args);
-    }
+    if (this.enabled("info")) console.log("[INFO]", ...args);
   }
 
   warn(...args) {
-    if (this.level !== "error") {
-      console.warn("[WARN]", ...args);
-    }
+    if (this.enabled("warn")) console.warn("[WARN]", ...args);
   }
 
   error(...args) {
@@ -30,4 +29,5 @@ class Logger {
   }
 }
 
-export default new Logger("error");
+export { Logger };
+export default new Logger(hearaldConfiguration);