hearald 0.0.4 → 0.1.0

package/src/server/server.js

@@ -1,13 +1,15 @@
 // Express middleware for hearald event tracking
 import express from "express";
-import log from "./logger.js";
-import { nowAsPstDate } from "../shared/utils.js";
-import { writeEvent, closeEventFiles } from "./serverEvents.js";
-import { createAnalyticsRouter } from "./analytics.js";
 
-export async function logEvent(time, event) {
-  return writeEvent(time, event);
-}
+import hearaldConfiguration from "./configuration.js";
+import log from "./logger.js";
+import {
+  writeEvent,
+  closeEventFiles,
+  VISIT_EVENT,
+  ERROR_EVENT,
+} from "./serverEvents.js";
+import { validateEvent } from "./validate.js";
 
 export async function shutDown() {
   await closeEventFiles();
@@ -18,9 +20,12 @@ export async function shutDown() {
  * @param {Object} options
  * @param {string} [options.url='/e'] - The URL to listen for events.
  * @param {function} [options.parseBody] - Custom request body parser (req => {e, u, p}).
+ * @param {Object} [options.schema] - Allowed events and their parameters (see validate.js). When
+ *   given, events not in it, and parameters not listed, are dropped. Strongly recommended for any
+ *   endpoint that is open to the internet.
  * @returns {function} Express middleware
  */
-export function eventEndpointMiddleware({ url = "/e", parseBody } = {}) {
+export function eventEndpointMiddleware({ url = "/e", parseBody, schema } = {}) {
   const router = express.Router();
   router.post(url, async (req, res) => {
     try {
@@ -29,11 +34,25 @@ export function eventEndpointMiddleware({ url = "/e", parseBody } = {}) {
         ({ e, u, p } = parseBody(req));
       } else {
         ({ e = null, u = null, p = {} } = req.body || {});
-        if (e === "visit") {
-          p.ip = req.ip;
+        if (p === null || typeof p !== "object" || Array.isArray(p)) p = {};
+        if (e === VISIT_EVENT) {
+          p = { ...p, ip: req.ip };
+        }
+      }
+      if (schema) {
+        const clean = validateEvent(schema, { e, u, p });
+        if (!clean) {
+          // Answer the same as for a good event, so the endpoint reveals nothing about the schema.
+          res.status(204).send();
+          return;
         }
+        ({ e, u, p } = clean);
       }
-      await logEvent(nowAsPstDate(), { e, u, p });
+      if (typeof e !== "string" || !e) {
+        res.status(204).send();
+        return;
+      }
+      await writeEvent(hearaldConfiguration.nowFn(), { e, u, p });
     } catch (err) {
       // eslint-disable-next-line no-console
       log.error("error writing event to event log", err);
@@ -42,3 +61,72 @@ export function eventEndpointMiddleware({ url = "/e", parseBody } = {}) {
   });
   return router;
 }
+
+export async function trackServerEvent({
+  event,
+  time = null,
+  userId,
+  params = {},
+}) {
+  await writeEvent(time || hearaldConfiguration.nowFn(), {
+    e: event,
+    u: userId,
+    p: params,
+  });
+}
+
+// JSON.stringify throws on circular structures (and on BigInt); an error handler must not.
+function safeStringify(value) {
+  try {
+    return JSON.stringify(value);
+  } catch (e) {
+    return String(value);
+  }
+}
+
+export async function trackServerError({
+  error,
+  userId = null,
+  extraParams = {},
+}) {
+  let message = "";
+  let stack = "";
+  if (error instanceof Error) {
+    message = error.message;
+    stack = error.stack || "";
+  } else if (typeof error === "string") {
+    message = error;
+  } else if (error && typeof error === "object") {
+    message = error.message || safeStringify(error);
+    stack = error.stack || "";
+  }
+  const params = {
+    message,
+    stack,
+    ...extraParams,
+  };
+  await writeEvent(hearaldConfiguration.nowFn(), {
+    e: ERROR_EVENT,
+    u: userId,
+    p: params,
+  });
+}
+
+// Express error-handling middleware
+export function errorTrackingMiddleware({}) {
+  // eslint-disable-next-line no-unused-vars
+  return function (err, req, res, next) {
+    // Recording the error must never stop it reaching the app's own handler, nor crash the process
+    // (a failed write is a rejected promise nobody awaits).
+    try {
+      trackServerError({
+        error: err,
+        userId: hearaldConfiguration.getUserId(req),
+        extraParams: { url: req.originalUrl, method: req.method, ip: req.ip },
+      }).catch((e) => log.error("could not record the error", e));
+    } catch (e) {
+      log.error("could not record the error", e);
+    }
+    next(err);
+  };
+}

package/src/server/serverEvents.js

@@ -1,72 +1,122 @@
-import { readFile, open as openFile, mkdir } from "fs/promises";
+import {
+  readFile,
+  readdir,
+  unlink,
+  stat,
+  open as openFile,
+  mkdir,
+} from "fs/promises";
+import { join } from "path";
 
+import hearaldConfiguration from "./configuration.js";
 import log from "./logger.js";
-import { iso } from "../shared/utils.js";
+
+export const VISIT_EVENT = "visit";
+export const ERROR_EVENT = "error";
 
 const eventFilePromises = new Map();
 const lastWriteTimeouts = new Map();
 
+// Events live in hearaldConfiguration.eventsDir (default "events", relative to the working directory).
+export const eventFilePath = (fileName) =>
+  join(hearaldConfiguration.eventsDir, `${fileName}.log`);
+
+// A crash can leave the last line unfinished. Appending straight after it would glue the next event
+// onto the broken line and lose both, so start a fresh line first.
+const endWithNewline = async (file) => {
+  const { size } = await file.stat();
+  if (size === 0) return;
+  const last = Buffer.alloc(1);
+  await file.read(last, 0, 1, size - 1);
+  if (last[0] !== 0x0a) await file.write("\n");
+};
+
 export const openEventFile = async (fileName) => {
-  await mkdir("events", { recursive: true });
-  return openFile(`events/${fileName}.log`, "a");
+  await mkdir(hearaldConfiguration.eventsDir, { recursive: true });
+  // "a+" appends like "a" but lets us look at the end of the file first
+  const file = await openFile(eventFilePath(fileName), "a+");
+  try {
+    await endWithNewline(file);
+  } catch (e) {
+    await file.close();
+    throw e;
+  }
+  return file;
 };
 
-export const writeEvent = async (time, event, isoFn = iso) => {
+export const writeEvent = async (time, event) => {
   let fileName;
 
-  const dateString = isoFn(time);
+  const dateString = hearaldConfiguration.isoFn(time);
 
-  if (event.e === "visit") {
+  if (event.e === VISIT_EVENT) {
     fileName = `${dateString}-visits`;
-  } else if (event.e === "error") {
+  } else if (event.e === ERROR_EVENT) {
     fileName = `${dateString}-errors`;
   } else {
     fileName = dateString;
   }
 
-  if (!eventFilePromises.has(fileName)) {
-    eventFilePromises.set(fileName, openEventFile(fileName));
+  const key = eventFilePath(fileName);
+  if (!eventFilePromises.has(key)) {
+    const opening = openEventFile(fileName);
+    eventFilePromises.set(key, opening);
+    // A failed open must not stay cached: every later write would fail with it, and closing it
+    // from the idle timer would be an unhandled rejection that kills the process.
+    opening.catch(() => {
+      if (eventFilePromises.get(key) === opening) {
+        eventFilePromises.delete(key);
+        clearIdleTimer(key);
+      }
+    });
   }
 
-  if (lastWriteTimeouts.has(fileName)) {
-    clearTimeout(lastWriteTimeouts.get(fileName));
-  }
-  lastWriteTimeouts.set(
-    fileName,
-    setTimeout(() => closeEventFile(fileName), 60 * 1000)
+  clearIdleTimer(key);
+  // Close by the key captured now: the events directory may have changed by the time this fires.
+  const timer = setTimeout(
+    () => closeByKey(key),
+    hearaldConfiguration.idleCloseMs
   );
+  timer.unref(); // an idle file handle must not keep a script alive
+  lastWriteTimeouts.set(key, timer);
 
-  const eventFile = await eventFilePromises.get(fileName);
+  const eventFile = await eventFilePromises.get(key);
   return eventFile.write(`${JSON.stringify({ t: time, ...event })}\n`);
 };
 
-export const closeEventFile = async (fileName) => {
-  const eventFilePromise = eventFilePromises.get(fileName);
-  eventFilePromises.delete(fileName);
+function clearIdleTimer(key) {
+  const timeout = lastWriteTimeouts.get(key);
+  lastWriteTimeouts.delete(key);
+  if (timeout) clearTimeout(timeout);
+}
 
-  const timeout = lastWriteTimeouts.get(fileName);
-  lastWriteTimeouts.delete(fileName);
-  if (timeout) {
-    clearTimeout(timeout);
+// Never rejects: it runs from timers and shutdown, where nobody could catch it. A failed open was
+// already reported to the writer that asked for it.
+async function closeByKey(key) {
+  const eventFilePromise = eventFilePromises.get(key);
+  eventFilePromises.delete(key);
+  clearIdleTimer(key);
+  if (!eventFilePromise) return;
+  try {
+    await (await eventFilePromise).close();
+  } catch (e) {
+    log.debug(`could not close ${key}: ${e.message}`);
   }
+}
 
-  if (eventFilePromise) {
-    return (await eventFilePromise).close();
-  }
-  return undefined;
-};
+// How many event files are open or opening (for tests and diagnostics).
+export const openEventFileCount = () => eventFilePromises.size;
+
+export const closeEventFile = (fileName) =>
+  closeByKey(eventFilePath(fileName));
 
 export const closeEventFiles = async () => {
-  return Promise.all(
-    Array.from(eventFilePromises.keys()).map(async (fileName) =>
-      closeEventFile(fileName)
-    )
-  );
+  await Promise.all(Array.from(eventFilePromises.keys()).map(closeByKey));
 };
 
 export const readEventsFromFile = async (fileName) => {
   try {
-    return (await readFile(`events/${fileName}.log`))
+    return (await readFile(eventFilePath(fileName)))
       .toString("utf8")
       .split("\n")
       .filter((r) => r.trim())
@@ -80,9 +130,9 @@ export const readEventsFromFile = async (fileName) => {
       })
       .filter(Boolean);
   } catch (e) {
-    if (e.message && e.message.indexOf("ENOENT") !== -1) {
-      // eslint-disable-next-line no-console
-      log.warn(`file not found: events/${fileName}.log`);
+    if (e.code === "ENOENT") {
+      // a day with no events has no file; that is normal
+      log.debug(`no events file: ${eventFilePath(fileName)}`);
     } else {
       // eslint-disable-next-line no-console
       log.error(e);
@@ -91,16 +141,77 @@ export const readEventsFromFile = async (fileName) => {
   }
 };
 
-export const loadEventsOverWindow = async (d, n, fileSuffix, isoFn = iso) => {
-  let events = [];
+// The file names of the n days ending on d (UTC fields, like `iso`).
+export const windowFileNames = (d, n, fileSuffix) => {
+  const names = [];
   for (let i = 0; i < n; i++) {
+    // `iso` reads the UTC fields, so step back in UTC too (local-time maths skips or repeats a day
+    // around daylight saving changes on servers not set to UTC)
     const newDate = new Date(d);
-    newDate.setDate(d.getDate() - i);
-    let fileName = isoFn(newDate);
+    newDate.setUTCDate(d.getUTCDate() - i);
+    let fileName = hearaldConfiguration.isoFn(newDate);
     if (fileSuffix) {
       fileName += `-${fileSuffix}`;
     }
-    events = [...events, ...(await readEventsFromFile(fileName))];
+    names.push(fileName);
+  }
+  return names;
+};
+
+// Total size in bytes of the event files for a window, so a window too big for memory can be refused
+// before anything is read.
+export const windowBytes = async (d, n, fileSuffix) => {
+  let total = 0;
+  for (const fileName of windowFileNames(d, n, fileSuffix)) {
+    try {
+      total += (await stat(eventFilePath(fileName))).size;
+    } catch (e) {
+      if (e.code !== "ENOENT") throw e;
+    }
+  }
+  return total;
+};
+
+export const loadEventsOverWindow = async (d, n, fileSuffix) => {
+  const events = [];
+  for (const fileName of windowFileNames(d, n, fileSuffix)) {
+    // push, not spreading into a new array each day, which is quadratic
+    for (const event of await readEventsFromFile(fileName)) events.push(event);
   }
   return events;
 };
+
+// Deletes event logs and cached summaries older than `days` days (by the date in the file name; a
+// file dated exactly `days` days ago is kept). Counts days with the configured clock. Returns the number of files removed. Run it nightly to honour a retention period.
+export const purgeOldEvents = async ({
+  days,
+  now = hearaldConfiguration.nowFn(),
+} = {}) => {
+  if (!Number.isInteger(days) || days < 1) {
+    throw new Error("purgeOldEvents needs a whole number of days, 1 or more");
+  }
+  // UTC fields, like `isoFn`; local-time maths could be off by a day around daylight saving changes
+  const cutoff = new Date(now);
+  cutoff.setUTCDate(cutoff.getUTCDate() - days);
+  const cutoffIso = hearaldConfiguration.isoFn(cutoff);
+
+  let names;
+  try {
+    names = await readdir(hearaldConfiguration.eventsDir);
+  } catch (e) {
+    if (e.code === "ENOENT") return 0;
+    throw e;
+  }
+  let removed = 0;
+  for (const name of names) {
+    const match =
+      /^(\d{4}-\d{2}-\d{2})(?:-[a-z]+)?\.log$/.exec(name) ||
+      /^summary:(\d{4}-\d{2}-\d{2}):\d+(?::[0-9a-f]+)?\.json$/.exec(name);
+    if (match && match[1] < cutoffIso) {
+      await closeEventFile(name.replace(/\.log$/, ""));
+      await unlink(join(hearaldConfiguration.eventsDir, name));
+      removed += 1;
+    }
+  }
+  return removed;
+};

package/src/server/validate.js

@@ -0,0 +1,56 @@
+// Validates events arriving on the public event endpoint against a schema, so the endpoint cannot
+// be used to store arbitrary text or unbounded data.
+//
+//   schema = {
+//     completion_added: {
+//       platform: { type: "string", enum: ["web", "ios"], required: true },
+//       goal_type: { type: "string", enum: ["binary", "counter", "monthly"] },
+//       count: { type: "number", integer: true, min: 0, max: 1000 },
+//     },
+//   };
+//
+// Events not in the schema, and parameters not listed for their event, are dropped. An event with a
+// missing required parameter or a parameter of the wrong type is rejected whole.
+
+const MAX_USER_ID_LENGTH = 64;
+const DEFAULT_MAX_STRING = 64;
+
+function checkParam(spec, value) {
+  switch (spec.type) {
+    case "string":
+      if (typeof value !== "string") return { ok: false };
+      if (value.length > (spec.maxLength ?? DEFAULT_MAX_STRING)) return { ok: false };
+      if (spec.enum && !spec.enum.includes(value)) return { ok: false };
+      return { ok: true, value };
+    case "number":
+      if (typeof value !== "number" || !Number.isFinite(value)) return { ok: false };
+      if (spec.integer && !Number.isInteger(value)) return { ok: false };
+      if (spec.min !== undefined && value < spec.min) return { ok: false };
+      if (spec.max !== undefined && value > spec.max) return { ok: false };
+      return { ok: true, value };
+    case "boolean":
+      return typeof value === "boolean" ? { ok: true, value } : { ok: false };
+    default:
+      throw new Error(`unknown parameter type "${spec.type}"`);
+  }
+}
+
+// Returns the cleaned { e, u, p }, or null when the event should be dropped.
+export function validateEvent(schema, { e, u, p } = {}) {
+  if (typeof e !== "string" || !Object.hasOwn(schema, e)) return null;
+  if (u !== null && u !== undefined) {
+    if (typeof u !== "string" || !u || u.length > MAX_USER_ID_LENGTH) return null;
+  }
+  const given = p && typeof p === "object" && !Array.isArray(p) ? p : {};
+  const clean = {};
+  for (const [name, spec] of Object.entries(schema[e])) {
+    if (!Object.hasOwn(given, name)) {
+      if (spec.required) return null;
+      continue;
+    }
+    const result = checkParam(spec, given[name]);
+    if (!result.ok) return null;
+    clean[name] = result.value;
+  }
+  return { e, u: u ?? null, p: clean };
+}

package/src/shared/utils.js

@@ -10,21 +10,36 @@ export const pacificTimeFormatter = new Intl.DateTimeFormat("en-US", {
 export const pacificTimeOffsetHours = (date) =>
   pacificTimeFormatter.format(date || new Date()).endsWith("PDT") ? -7 : -8;
 
-export const dateStringToPstDate = (dateString) => {
-  const rawDate = dateString ? new Date(dateString) : new Date();
-  const offset = pacificTimeOffsetHours(rawDate);
-  const pstDate = new Date();
-  pstDate.setUTCFullYear(
-    rawDate.getUTCFullYear(),
-    rawDate.getUTCMonth(),
-    rawDate.getUTCDate()
+// A Date whose UTC fields hold the current Pacific wall-clock time, so `iso()` (which reads UTC
+// fields) names the Pacific day. The instant it represents is not "now"; it is only for day buckets.
+export const nowAsPstDate = (now = new Date()) => {
+  const parts = Object.fromEntries(
+    new Intl.DateTimeFormat("en-US", {
+      timeZone: "America/Los_Angeles",
+      hourCycle: "h23",
+      year: "numeric",
+      month: "numeric",
+      day: "numeric",
+      hour: "numeric",
+      minute: "numeric",
+      second: "numeric",
+    })
+      .formatToParts(now)
+      .map((p) => [p.type, p.value])
+  );
+  return new Date(
+    Date.UTC(
+      Number(parts.year),
+      Number(parts.month) - 1,
+      Number(parts.day),
+      Number(parts.hour),
+      Number(parts.minute),
+      Number(parts.second),
+      now.getMilliseconds()
+    )
   );
-  pstDate.setUTCHours(-1 * offset, 0, 0, 0);
-  return pstDate;
 };
 
-export const nowAsPstDate = () => dateStringToPstDate();
-
 export function omitProperties(object, ...keys) {
   let rest = object;
   keys.forEach((key) => {
@@ -43,9 +58,10 @@ export function iso(time) {
   ].join("-");
 }
 
-export function defaultEndDate() {
-  const yesterday = nowAsPstDate();
-  yesterday.setDate(yesterday.getDate() - 1);
-  yesterday.setHours(0, 0, 0, 0);
+// Midnight (UTC fields) at the start of yesterday, in the same day buckets as nowAsPstDate.
+export function defaultEndDate(now = nowAsPstDate()) {
+  const yesterday = new Date(now);
+  yesterday.setUTCDate(yesterday.getUTCDate() - 1);
+  yesterday.setUTCHours(0, 0, 0, 0);
   return yesterday;
 }

package/.eslintrc

@@ -1,14 +0,0 @@
-{
-  "env": {
-    "browser": true,
-    "node": true,
-    "es2020": true
-  },
-  "parserOptions": {
-    "ecmaVersion": 2020,
-    "sourceType": "module"
-  },
-  "globals": {
-    "Intl": "readonly"
-  }
-}

package/TODO.md

@@ -1,34 +0,0 @@
-# hearald TODO
-
-## Status (August 2025)
-
-- Event bus/evented logic, event file helpers, emit logic, admin stats, analytics helpers, and error tracking utilities have all been extracted and modularized.
-- hearald now uses client/server/shared subfolders and has a working .eslintrc for Intl.
-- Lint passes and all functionality is verified.
-- Migration of all helpers to client/server/shared is nearly complete.
-
-## Next Steps for hearald
-
-1. **Tracking Custom Events**
-
-   - Make the event schema/config easy to extend (document how to define custom events)
-   - Provide a clear API for emitting custom events (e.g., `trackEvent('my_event', {...})`)
-
-2. **Generating Statistical Summary Views**
-
-   - Make stat definitions configurable (allow user to provide their own stat_definition.json)
-   - Expose a summary API (e.g., `generateStats(endDate, timeWindow)`)
-   - Document how to add new metrics and computed stats
-
-3. **Admin UI to See Stats**
-   - Provide a basic, embeddable React/Preact/Vue component for admin stats (or a template)
-   - Document how to use the endpoint and client fetcher in any admin UI
-   - Optionally, provide a standalone admin page as part of the package
-
-## General Improvements
-
-- Write clear documentation and usage examples
-- Add configuration options for stat definitions and event types
-- Provide ready-to-use middleware and client helpers for common event types
-- Package a basic admin UI or template
-- Test in a clean, new project to validate ease of adoption

package/rollup.config.js

@@ -1,59 +0,0 @@
-import fs from "fs";
-import path from "path";
-import resolve from "@rollup/plugin-node-resolve";
-import postcss from "rollup-plugin-postcss";
-
-// Custom plugin to inline JS and CSS into HTML
-function inlineAssetsPlugin({ htmlFile, outputFile }) {
-  return {
-    name: "inline-assets",
-    async writeBundle(options, bundle) {
-      const htmlPath = path.resolve(htmlFile);
-      let html = fs.readFileSync(htmlPath, "utf8");
-
-      // Find JS and CSS output from bundle
-      const jsBundle = Object.values(bundle).find(
-        (file) => file.type === "chunk" && file.isEntry
-      );
-      const cssBundle = Object.values(bundle).find(
-        (file) => file.type === "asset" && file.fileName.endsWith(".css")
-      );
-
-      const js = jsBundle ? jsBundle.code : "";
-      const css = cssBundle ? cssBundle.source : "";
-
-      // Replace <script src=...> and <link rel="stylesheet" ...> with inline tags
-      html = html.replace(
-        /<script\s+src="[^"]*"[^>]*><\/script>/,
-        `<script>${js}</script>`
-      );
-      // Replace any <link rel="stylesheet" ...> (with any attributes)
-      html = html.replace(
-        /<link\s+rel=["']stylesheet["'][^>]*>/gi,
-        `<style>${css}</style>`
-      );
-
-      // Write the inlined HTML
-      fs.writeFileSync(path.resolve(options.dir, outputFile), html);
-    },
-  };
-}
-
-export default {
-  input: "src/public/js/index.js", // Use JS entry point
-  output: {
-    dir: "dist",
-    format: "iife",
-  },
-  plugins: [
-    resolve(),
-    postcss({
-      inject: false,
-      extract: true,
-    }),
-    inlineAssetsPlugin({
-      htmlFile: "src/public/analytics.html",
-      outputFile: "analytics.html",
-    }),
-  ],
-};

package/src/server/serverTrackError.js

@@ -1,47 +0,0 @@
-// hearald/src/serverTrackError.js
-// Server-side utility to track errors
-
-export async function trackServerError({
-  error,
-  logEvent,
-  userId = null,
-  extraParams = {},
-  nowFn = () => new Date(),
-}) {
-  let message = "";
-  let stack = "";
-  if (error instanceof Error) {
-    message = error.message;
-    stack = error.stack || "";
-  } else if (typeof error === "string") {
-    message = error;
-  } else if (error && typeof error === "object") {
-    message = error.message || JSON.stringify(error);
-    stack = error.stack || "";
-  }
-  const params = {
-    message,
-    stack,
-    ...extraParams,
-  };
-  await logEvent(nowFn(), { e: "error", u: userId, p: params });
-}
-
-// Express error-handling middleware
-export function errorTrackingMiddleware({
-  logEvent,
-  nowFn = () => new Date(),
-  userIdExtractor = (req) => req.user?.id,
-}) {
-  // eslint-disable-next-line no-unused-vars
-  return function (err, req, res, next) {
-    trackServerError({
-      error: err,
-      logEvent,
-      userId: userIdExtractor(req),
-      extraParams: { url: req.originalUrl, method: req.method, ip: req.ip },
-      nowFn,
-    });
-    next(err);
-  };
-}