hearald 0.0.1

package/server/analytics.js

@@ -0,0 +1,321 @@
+// Analytics router and helpers for hearald
+import chalk from "chalk";
+import path from "path";
+import { fileURLToPath } from "url";
+import { readFile, writeFile } from "fs/promises";
+import express from "express";
+import {
+  omitProperties,
+  iso,
+  defaultEndDate,
+  nowAsPstDate,
+} from "../shared/utils.js";
+import { loadEventsOverWindow } from "./serverEvents.js";
+
+export function generateRawMetrics(events, rawMetricDefinitions) {
+  const metrics = {};
+  Object.entries(rawMetricDefinitions).forEach(
+    ([name, rawMetricDefinition]) => {
+      if (rawMetricDefinition.aggregator === "count") {
+        metrics[name] = 0;
+      } else if (rawMetricDefinition.aggregator === "set_of_users") {
+        metrics[name] = new Set();
+      } else if (rawMetricDefinition.aggregator === "histogram") {
+        metrics[name] = {};
+      }
+    }
+  );
+
+  events.forEach(({ e, p, u }) => {
+    Object.entries(rawMetricDefinitions).forEach(
+      ([name, rawMetricDefinition]) => {
+        if (rawMetricDefinition.filter) {
+          const match = Object.entries(rawMetricDefinition.filter).every(
+            ([filterField, filterValue]) => p[filterField] === filterValue
+          );
+          if (!match) return;
+        }
+        if (
+          rawMetricDefinition.aggregator === "count" &&
+          rawMetricDefinition.event === e
+        ) {
+          metrics[name] += 1;
+        } else if (rawMetricDefinition.aggregator === "set_of_users") {
+          metrics[name].add(u);
+        } else if (
+          rawMetricDefinition.aggregator === "histogram" &&
+          rawMetricDefinition.event === e
+        ) {
+          const field = rawMetricDefinition.field;
+          const value = p && p[field] ? p[field] : null;
+          if (value !== null) {
+            metrics[name][value] = (metrics[name][value] || 0) + 1;
+          }
+        }
+      }
+    );
+  });
+  // Convert histogram objects to sorted list of [label, count] tuples
+  Object.entries(rawMetricDefinitions).forEach(([name, def]) => {
+    if (def.aggregator === "histogram") {
+      const hist = metrics[name];
+      metrics[name] = Object.entries(hist).sort((a, b) => b[1] - a[1]);
+    }
+  });
+  return metrics;
+}
+
+export function generateComputedMetric(metrics, definition, timeWindow) {
+  const set = new Set();
+  let denominator;
+  switch (definition.formula) {
+    case "set_difference":
+      (metrics[definition.metrics.at(0)] || new Set()).forEach((v) =>
+        set.add(v)
+      );
+      definition.metrics.slice(1).forEach((metricName) => {
+        (metrics[metricName] || new Set()).forEach((v) => set.delete(v));
+      });
+      return set;
+    case "set_count":
+      return (metrics[definition.metrics.at(0)] || new Set()).size;
+    case "percent":
+      denominator = metrics[definition.metrics.at(1)];
+      if (!denominator) {
+        return 0;
+      }
+      return (100 * (metrics[definition.metrics.at(0)] || 0)) / denominator;
+    case "rate_time_window":
+      return (metrics[definition.metrics.at(0)] || 0) / timeWindow;
+    case "histogram": {
+      // Compute histogram for given event type and field
+      const sourceEvent = definition.source;
+      const field = definition.field;
+      const histogram = {};
+      // Find all events of the given type
+      (metrics._allEvents || []).forEach((ev) => {
+        if (ev.e === sourceEvent && ev.p && ev.p[field]) {
+          const value = ev.p[field];
+          histogram[value] = (histogram[value] || 0) + 1;
+        }
+      });
+      return histogram;
+    }
+  }
+}
+
+export function cacheFilePath(endDate, timeWindow) {
+  const endDateIso = typeof endDate === "string" ? endDate : iso(endDate);
+  return `events/summary:${endDateIso}:${timeWindow}.json`;
+}
+
+export async function readMetricsFromCache(endDateIso, timeWindow) {
+  const filePath = cacheFilePath(endDateIso, timeWindow);
+  try {
+    return (await readFile(filePath)).toString("utf8");
+  } catch (e) {
+    if (e.message.indexOf("ENOENT") !== -1) {
+      return null;
+    }
+    throw e;
+  }
+}
+
+export async function writeMetricsToCache(endDateIso, timeWindow, statsJson) {
+  const filePath = cacheFilePath(endDateIso, timeWindow);
+  return writeFile(filePath, statsJson);
+}
+
+export async function generateStats(statDefinitions, endDate, timeWindow) {
+  const rawMetricDefinitions = statDefinitions.raw_metrics;
+  const computedMetricDefinitions = statDefinitions.computed_metrics;
+
+  // Collect all file suffixes needed for metrics (from raw metrics now)
+  const fileSuffixes = new Set([""]); // default log file
+  Object.values(rawMetricDefinitions).forEach((def) => {
+    if (def.source_file_suffix) fileSuffixes.add(def.source_file_suffix);
+  });
+
+  // Load events from all relevant files for raw metrics
+  let allEvents = [];
+  const eventsBySuffix = {};
+  for (const suffix of fileSuffixes) {
+    const events = await loadEventsOverWindow(
+      endDate,
+      timeWindow,
+      suffix || undefined
+    );
+    allEvents = allEvents.concat(events);
+    eventsBySuffix[suffix || ""] = events;
+  }
+
+  // Generate raw metrics, using correct event set for each metric
+  const metrics = {};
+  Object.entries(rawMetricDefinitions).forEach(([name, def]) => {
+    const suffix = def.source_file_suffix || "";
+    metrics[name] = generateRawMetrics(eventsBySuffix[suffix], { [name]: def })[
+      name
+    ];
+  });
+
+  // Store all events for computed metrics
+  metrics._allEvents = allEvents;
+
+  let anyComputed = true;
+  while (anyComputed) {
+    anyComputed = false;
+    for (const [name, computedMetricDefinition] of Object.entries(
+      computedMetricDefinitions
+    )) {
+      if (typeof metrics[name] !== "undefined") {
+        continue;
+      }
+      if (
+        computedMetricDefinition.metrics &&
+        computedMetricDefinition.metrics.every(
+          (m) => typeof metrics[m] !== "undefined"
+        )
+      ) {
+        metrics[name] = generateComputedMetric(
+          metrics,
+          computedMetricDefinition,
+          timeWindow
+        );
+        anyComputed = true;
+      }
+    }
+  }
+  const hiddenMetrics = []
+    .concat(
+      Object.keys(rawMetricDefinitions).filter(
+        (k) => rawMetricDefinitions[k].hidden
+      )
+    )
+    .concat(
+      Object.keys(computedMetricDefinitions).filter(
+        (k) => computedMetricDefinitions[k].hidden
+      )
+    );
+  const flatMetrics = omitProperties(metrics, ...hiddenMetrics, "_allEvents");
+  const nestedMetrics = {};
+  Object.entries(flatMetrics).forEach(([metricName, metricValue]) => {
+    const metricNameParts = metricName.split(".");
+    let m = nestedMetrics;
+    while (metricNameParts.length > 1) {
+      const metricNamePart = metricNameParts.shift();
+      if (typeof m[metricNamePart] === "undefined") {
+        m[metricNamePart] = {};
+      }
+      m = m[metricNamePart];
+    }
+    m[metricNameParts[0]] = metricValue;
+  });
+  return {
+    metrics: nestedMetrics,
+    endDate: iso(endDate),
+    timeWindow,
+    processedEventCount: allEvents.length,
+  };
+}
+
+export async function handleAnalyticsDataRequest({
+  request,
+  response,
+  statDefinitions,
+}) {
+  const defaultEndDateIso = iso(defaultEndDate(nowAsPstDate));
+  const endDateIso = request.params.date
+    ? request.params.date
+    : defaultEndDateIso;
+  const timeWindow = parseInt(request.params.last, 10);
+  if (!timeWindow) {
+    response.status(500).json({ Error: "invalid time window" });
+    return;
+  }
+  const dataFromFile = await readMetricsFromCache(endDateIso, timeWindow);
+  if (dataFromFile) {
+    console.log(
+      `Analytics for ${timeWindow} days ending on ${endDateIso} returned from cache`
+    );
+    response.status(200).type("application/json").send(dataFromFile);
+    return;
+  }
+  console.log(
+    `Analytics for ${timeWindow} days ending on ${endDateIso} computing from events`
+  );
+  try {
+    const stats = await generateStats(
+      statDefinitions,
+      new Date(endDateIso),
+      timeWindow
+    );
+    await writeMetricsToCache(
+      endDateIso,
+      timeWindow,
+      JSON.stringify(stats, null, 2)
+    );
+    response.status(200).type("application/json").send(JSON.stringify(stats));
+  } catch (e) {
+    console.error("failed to compute analytics", e);
+    response
+      .status(500)
+      .type("application/json")
+      .send(JSON.stringify({ error: e.message }));
+  }
+}
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+console.log("filename is", __filename);
+console.log("dirname is", __dirname);
+
+export function createAnalyticsRouter({
+  statDefinitions,
+  loadEventsOverWindow,
+  templateReplaceMany,
+  analyticsHtmlPath,
+  enableLiveReload,
+  environment,
+}) {
+  console.log("statManifest", statDefinitions);
+  const router = express.Router();
+
+  router.get(`/data/:last/:date?`, async (request, response) => {
+    console.log(chalk.cyan("[API] GET /analytics/data/:last/:date?"));
+    await handleAnalyticsDataRequest({
+      request,
+      response,
+      statDefinitions,
+      loadEventsOverWindow,
+      nowAsPstDate,
+    });
+  });
+
+  router.get(`/layout`, async (req, res) => {
+    res.json(statDefinitions.ui || {});
+  });
+
+  router.get(`/`, async (request, response) => {
+    console.log(chalk.cyan("[Render] views/analytics.html loading"));
+    console.log(path.join(__dirname, "..", "dist", "analytics.html"));
+    const fileContents = await readFile(
+      path.join(__dirname, "..", "dist", "analytics.html")
+    );
+    const analyticsHtmlRaw = fileContents.toString("utf-8");
+    console.log("analyticsHtmlRaw", analyticsHtmlRaw);
+    const yesterday = nowAsPstDate();
+    yesterday.setDate(yesterday.getDate() - 1);
+    const analyticsHtml = templateReplaceMany(analyticsHtmlRaw, {
+      date: yesterday.toISOString().split("T")[0],
+      devHeadExtras:
+        environment === "development" && enableLiveReload
+          ? '<script src="/liveReloading.js"></script>'
+          : "",
+    });
+    console.log("analyticsHtml", analyticsHtml);
+    response.send(analyticsHtml);
+  });
+
+  return router;
+}

package/server/server.js

@@ -0,0 +1,48 @@
+// Express middleware for hearald event tracking
+import express from "express";
+import { nowAsPstDate } from "../shared/utils.js";
+import { writeEvent, closeEventFiles } from "./serverEvents.js";
+import { createAnalyticsRouter } from "./analytics.js";
+
+export async function logEvent(time, event) {
+  return writeEvent(time, event);
+}
+
+export async function shutDown() {
+  await closeEventFiles();
+}
+
+/**
+ * Returns an express middleware for hearald event tracking.
+ * @param {Object} options
+ * @param {string} [options.url='/e'] - The URL to listen for events.
+ * @param {function} [options.parseBody] - Custom request body parser (req => {e, u, p}).
+ * @returns {function} Express middleware
+ */
+export function createHearaldMiddleware({ url = "/e", parseBody } = {}) {
+  const router = express.Router();
+  router.post(url, async (req, res) => {
+    try {
+      let e, u, p;
+      if (parseBody) {
+        ({ e, u, p } = parseBody(req));
+      } else {
+        ({ e = null, u = null, p = {} } = req.body || {});
+        if (e === "visit") {
+          p.ip = req.ip;
+        }
+      }
+      await logEvent(nowAsPstDate(), { e, u, p });
+    } catch (err) {
+      // eslint-disable-next-line no-console
+      console.error("error writing event to event log", err);
+    }
+    res.status(204).send();
+  });
+  return router;
+}
+
+export function createHearaldAnalyticsRouter(options = {}) {
+  // Pass statManifest to the analytics router so /analytics/layout.json is available
+  return createAnalyticsRouter({ ...options });
+}

package/server/serverEvents.js

@@ -0,0 +1,105 @@
+import { readFile, open as openFile, mkdir } from "fs/promises";
+
+import { iso } from "../shared/utils.js";
+
+const eventFilePromises = new Map();
+const lastWriteTimeouts = new Map();
+
+export const openEventFile = async (fileName) => {
+  await mkdir("events", { recursive: true });
+  return openFile(`events/${fileName}.log`, "a");
+};
+
+export const writeEvent = async (time, event, isoFn = iso) => {
+  let fileName;
+
+  const dateString = isoFn(time);
+
+  if (event.e === "visit") {
+    fileName = `${dateString}-visits`;
+  } else if (event.e === "error") {
+    fileName = `${dateString}-errors`;
+  } else {
+    fileName = dateString;
+  }
+
+  if (!eventFilePromises.has(fileName)) {
+    eventFilePromises.set(fileName, openEventFile(fileName));
+  }
+
+  if (lastWriteTimeouts.has(fileName)) {
+    clearTimeout(lastWriteTimeouts.get(fileName));
+  }
+  lastWriteTimeouts.set(
+    fileName,
+    setTimeout(() => closeEventFile(fileName), 60 * 1000)
+  );
+
+  const eventFile = await eventFilePromises.get(fileName);
+  return eventFile.write(`${JSON.stringify({ t: time, ...event })}\n`);
+};
+
+export const closeEventFile = async (fileName) => {
+  const eventFilePromise = eventFilePromises.get(fileName);
+  eventFilePromises.delete(fileName);
+
+  const timeout = lastWriteTimeouts.get(fileName);
+  lastWriteTimeouts.delete(fileName);
+  if (timeout) {
+    clearTimeout(timeout);
+  }
+
+  if (eventFilePromise) {
+    return (await eventFilePromise).close();
+  }
+  return undefined;
+};
+
+export const closeEventFiles = async () => {
+  return Promise.all(
+    Array.from(eventFilePromises.keys()).map(async (fileName) =>
+      closeEventFile(fileName)
+    )
+  );
+};
+
+export const readEventsFromFile = async (fileName) => {
+  try {
+    return (await readFile(`events/${fileName}.log`))
+      .toString("utf8")
+      .split("\n")
+      .filter((r) => r.trim())
+      .map((row) => {
+        try {
+          return JSON.parse(row);
+        } catch (e) {
+          // skip malformed lines
+          return undefined;
+        }
+      })
+      .filter(Boolean);
+  } catch (e) {
+    if (e.message && e.message.indexOf("ENOENT") !== -1) {
+      // eslint-disable-next-line no-console
+      console.warn(`file not found: events/${fileName}.log`);
+    } else {
+      // eslint-disable-next-line no-console
+      console.error(e);
+    }
+    return [];
+  }
+};
+
+export const loadEventsOverWindow = async (d, n, fileSuffix, isoFn = iso) => {
+  let events = [];
+  for (let i = 0; i < n; i++) {
+    const newDate = new Date(d);
+    newDate.setDate(d.getDate() - i);
+    let fileName = isoFn(newDate);
+    if (fileSuffix) {
+      fileName += `-${fileSuffix}`;
+    }
+    events = [...events, ...(await readEventsFromFile(fileName))];
+  }
+  return events;
+};

package/server/serverTrackError.js

@@ -0,0 +1,47 @@
+// hearald/src/serverTrackError.js
+// Server-side utility to track errors
+
+export async function trackServerError({
+  error,
+  logEvent,
+  userId = null,
+  extraParams = {},
+  nowFn = () => new Date(),
+}) {
+  let message = "";
+  let stack = "";
+  if (error instanceof Error) {
+    message = error.message;
+    stack = error.stack || "";
+  } else if (typeof error === "string") {
+    message = error;
+  } else if (error && typeof error === "object") {
+    message = error.message || JSON.stringify(error);
+    stack = error.stack || "";
+  }
+  const params = {
+    message,
+    stack,
+    ...extraParams,
+  };
+  await logEvent(nowFn(), { e: "error", u: userId, p: params });
+}
+
+// Express error-handling middleware
+export function errorTrackingMiddleware({
+  logEvent,
+  nowFn = () => new Date(),
+  userIdExtractor = (req) => req.user?.id,
+}) {
+  // eslint-disable-next-line no-unused-vars
+  return function (err, req, res, next) {
+    trackServerError({
+      error: err,
+      logEvent,
+      userId: userIdExtractor(req),
+      extraParams: { url: req.originalUrl, method: req.method, ip: req.ip },
+      nowFn,
+    });
+    next(err);
+  };
+}

package/shared/events.js

@@ -0,0 +1,44 @@
+// Usage:
+// 1. Register custom events with a schema
+// 2. Emit custom events by name
+
+import { EventBus } from "bussin-js";
+
+// Shared event bus for all tracking events
+export const eventBus = new EventBus([]);
+
+const registeredEvents = {};
+
+/**
+ * Registers a custom event type and its schema.
+ * @param {string} eventName - The name of the event.
+ * @param {object} schema - The schema definition for the event payload.
+ */
+export function registerEvent(eventName, schema) {
+  // Prevent duplicate registration
+  if (registeredEvents[eventName]) return;
+  registeredEvents[eventName] = schema;
+  // Add the event type to the tracking event bus if not already present
+  if (!eventBus.hasType(eventName)) {
+    eventBus.addType(eventName);
+  }
+}
+
+/**
+ * Emits a custom event if registered and payload matches schema.
+ * @param {string} eventName - The name of the event to emit.
+ * @param {object} payload - The event payload.
+ * @returns {boolean} - True if emitted, false if not registered or invalid payload.
+ */
+export function emitEvent(eventName, payload) {
+  const schema = registeredEvents[eventName];
+  if (!schema) return false;
+  // Simple schema validation: check required keys and types
+  for (const key in schema) {
+    if (!(key in payload)) return false;
+    if (typeof payload[key] !== schema[key]) return false;
+  }
+  // Emit on the shared tracking event bus
+  eventBus.emit(eventName, payload);
+  return true;
+}

package/shared/utils.js

@@ -0,0 +1,51 @@
+export const pacificTimeFormatter = new Intl.DateTimeFormat("en-US", {
+  weekday: "short",
+  year: "numeric",
+  month: "numeric",
+  day: "numeric",
+  timeZone: "America/Los_Angeles",
+  timeZoneName: "short",
+});
+
+export const pacificTimeOffsetHours = (date) =>
+  pacificTimeFormatter.format(date || new Date()).endsWith("PDT") ? -7 : -8;
+
+export const dateStringToPstDate = (dateString) => {
+  const rawDate = dateString ? new Date(dateString) : new Date();
+  const offset = pacificTimeOffsetHours(rawDate);
+  const pstDate = new Date();
+  pstDate.setUTCFullYear(
+    rawDate.getUTCFullYear(),
+    rawDate.getUTCMonth(),
+    rawDate.getUTCDate()
+  );
+  pstDate.setUTCHours(-1 * offset, 0, 0, 0);
+  return pstDate;
+};
+
+export const nowAsPstDate = () => dateStringToPstDate();
+
+export function omitProperties(object, ...keys) {
+  let rest = object;
+  keys.forEach((key) => {
+    // eslint-disable-next-line no-unused-vars
+    const { [key]: _omit, ...other } = rest;
+    rest = other;
+  });
+  return rest;
+}
+
+export function iso(time) {
+  return [
+    time.getUTCFullYear(),
+    (time.getUTCMonth() + 1).toString().padStart(2, "0"),
+    time.getUTCDate().toString().padStart(2, "0"),
+  ].join("-");
+}
+
+export function defaultEndDate() {
+  const yesterday = nowAsPstDate();
+  yesterday.setDate(yesterday.getDate() - 1);
+  yesterday.setHours(0, 0, 0, 0);
+  return yesterday;
+}