fhirsmith 0.18.1 → 1.0.0

package/library/cgroup-memory.js

@@ -110,4 +110,136 @@ function readCgroupMemoryLimit(opts = {}) {
   return { limit: 0, source: null };
 }
 
-module.exports = { readCgroupMemoryLimit };
+/**
+ * The directory of this process's own cgroup (memory controller), and whether it is v2 or
+ * v1. null when there is none to be found (not Linux, or no /proc).
+ * @param {Object} [opts] - procFile, cgroupRoot (tests)
+ * @returns {{dir: string, version: 2|1}|null}
+ */
+function ownCgroupDir(opts = {}) {
+  const procFile = opts.procFile || '/proc/self/cgroup';
+  const cgroupRoot = opts.cgroupRoot || '/sys/fs/cgroup';
+  let lines;
+  try {
+    lines = fs.readFileSync(procFile, 'utf8').split('\n').filter(l => l.trim() !== '');
+  } catch {
+    return null;
+  }
+  const v2 = lines.find(l => l.startsWith('0::'));
+  if (v2) {
+    const dir = path.join(cgroupRoot, path.posix.normalize(v2.substring(3) || '/'));
+    if (fs.existsSync(path.join(dir, 'memory.stat'))) {
+      return { dir, version: 2 };
+    }
+  }
+  for (const l of lines) {
+    const parts = l.split(':');
+    if (parts.length >= 3 && parts[1].split(',').includes('memory')) {
+      const dir = path.join(cgroupRoot, 'memory', path.posix.normalize(parts.slice(2).join(':') || '/'));
+      if (fs.existsSync(path.join(dir, 'memory.stat'))) {
+        return { dir, version: 1 };
+      }
+    }
+  }
+  return null;
+}
+
+/**
+ * What memory.stat says a cgroup holds: its private memory (`anon`: what the kernel can't take
+ * back without killing a process) and its page cache (`file`: the terminology and SQLite files,
+ * mapped or read, which the kernel drops when it needs the room; shared pages counted once).
+ * v2: 'anon' and 'file'; v1: 'total_rss' (else 'rss') and 'total_cache' (else 'cache').
+ * @param {{dir: string, version: 2|1}} cg - from ownCgroupDir
+ * @returns {{anon: number, file: number|null}|null} bytes, or null if it can't be read
+ */
+function readCgroupStat(cg) {
+  let text;
+  try {
+    text = fs.readFileSync(path.join(cg.dir, 'memory.stat'), 'utf8');
+  } catch {
+    return null;
+  }
+  const values = new Map();
+  for (const line of text.split('\n')) {
+    const i = line.indexOf(' ');
+    if (i > 0) {
+      values.set(line.substring(0, i), Number(line.substring(i + 1)));
+    }
+  }
+  const first = keys => {
+    for (const key of keys) {
+      const v = values.get(key);
+      if (Number.isFinite(v) && v >= 0) {
+        return v;
+      }
+    }
+    return null;
+  };
+  const v2 = cg.version === 2;
+  const anon = first(v2 ? ['anon'] : ['total_rss', 'rss']);
+  return anon === null ? null : { anon, file: first(v2 ? ['file'] : ['total_cache', 'cache']) };
+}
+
+/**
+ * The private memory of everything in a cgroup (memory.stat 'anon'; see readCgroupStat). The
+ * page cache isn't counted: the kernel drops it when it needs the room.
+ * @param {{dir: string, version: 2|1}} cg - from ownCgroupDir
+ * @returns {number} bytes, or 0 if it can't be read
+ */
+function readCgroupAnon(cg) {
+  const stat = readCgroupStat(cg);
+  return stat ? stat.anon : 0;
+}
+
+/**
+ * A process's resident memory by kind, from /proc/<pid>/status: RssAnon (its private memory)
+ * and RssFile (the file pages it has mapped and touched, such as the SNOMED CT cache files;
+ * pages shared with other processes are counted in each). Linux only.
+ * @param {number|string} pid - or 'self'
+ * @param {string} [procRoot='/proc'] - (tests)
+ * @returns {{anon: number, file: number}|null} bytes, or null if it can't be read
+ */
+function readProcessMemory(pid, procRoot = '/proc') {
+  let status;
+  try {
+    status = fs.readFileSync(path.join(procRoot, String(pid), 'status'), 'utf8');
+  } catch {
+    return null;
+  }
+  const kb = name => {
+    const m = new RegExp(`^${name}:\\s+(\\d+) kB`, 'm').exec(status);
+    return m ? Number(m[1]) * 1024 : null;
+  };
+  const anon = kb('RssAnon');
+  return anon === null ? null : { anon, file: kb('RssFile') ?? 0 };
+}
+
+/**
+ * The RSS of each process in a cgroup, from /proc/<pid>/status. Processes that have gone,
+ * or can't be read, are left out.
+ * @param {{dir: string}} cg - from ownCgroupDir
+ * @returns {Map<number, number>} pid -> bytes
+ */
+function readCgroupProcessRss(cg) {
+  const result = new Map();
+  let pids;
+  try {
+    pids = fs.readFileSync(path.join(cg.dir, 'cgroup.procs'), 'utf8').split('\n').filter(Boolean).map(Number);
+  } catch {
+    return result;
+  }
+  for (const pid of pids) {
+    try {
+      const status = fs.readFileSync(`/proc/${pid}/status`, 'utf8');
+      const m = /^VmRSS:\s+(\d+) kB/m.exec(status);
+      if (m) {
+        result.set(pid, Number(m[1]) * 1024);
+      }
+    } catch {
+      // gone
+    }
+  }
+  return result;
+}
+
+module.exports = { readCgroupMemoryLimit, ownCgroupDir, readCgroupStat, readCgroupAnon, readProcessMemory, readCgroupProcessRss };

package/library/cluster-counts.js

@@ -0,0 +1,121 @@
+/**
+ * The tx-workers' request counts, for the master's statistics (cluster.md, "Statistics").
+ *
+ * A tx-worker has no statistics database and sends no daily report (library/role.js). Under
+ * the launcher (cluster.js), each worker sends the requests it counted since the last time
+ * (module, endpoint, operation, count, time) over its IPC channel, every minute and when it
+ * stops. The launcher sums what it has not yet delivered and passes it on to the master,
+ * which adds it to its own counters (ServerStats.mergeCounts). The master's statistics
+ * database, and the daily report built from it, then count every process, each request once:
+ * a worker's counts are taken when sent, and only the master writes them.
+ *
+ * Without the launcher (processes started by hand behind nginx) there is no IPC channel, and
+ * a worker's requests are not in the master's statistics.
+ */
+const COUNTS = 'fhirsmith:counts';
+// how often a worker sends its request counts
+const COUNTS_INTERVAL_MS = 60 * 1000;
+// how long a stopping worker waits for its last counts to be sent
+const COUNTS_FLUSH_MS = 2000;
+
+/**
+ * Attach a process with a role to the launcher's IPC channel. A process started without the
+ * launcher has none, and this does nothing.
+ * @param {Object} stats - the ServerStats
+ * @param {Object} [channel] - process, or a stand-in (tests)
+ * @param {Object} [options]
+ * @param {string} [options.role] - 'master' takes the workers' counts; 'tx-worker' sends its own
+ * @param {number} [options.intervalMs]
+ * @returns {Function} disconnect; its flush() sends a worker's last counts (on shutdown)
+ */
+function connectCounts(stats, channel = process, { role = null, intervalMs = COUNTS_INTERVAL_MS } = {}) {
+  if (typeof channel.send !== 'function') {
+    const none = () => {};
+    none.flush = async () => {};
+    return none;
+  }
+  // A worker's counts are taken only when sent, one lot at a time; a lot that can't be sent
+  // is put back, to go with the next
+  let inFlight = null;
+  const send = () => {
+    if (inFlight) return inFlight;
+    if (role !== 'tx-worker' || channel.connected === false) return Promise.resolve();
+    const rows = stats.takeCounts();
+    if (rows.length === 0) return Promise.resolve();
+    inFlight = new Promise(resolve => {
+      const done = error => {
+        if (error) stats.restoreCounts(rows);
+        inFlight = null;
+        resolve();
+      };
+      try {
+        channel.send({ type: COUNTS, rows }, done);
+      } catch (error) {
+        done(error);
+      }
+    });
+    return inFlight;
+  };
+  const timer = role === 'tx-worker' ? setInterval(send, intervalMs) : null;
+  if (timer && typeof timer.unref === 'function') timer.unref();
+
+  const receive = message => {
+    if (role === 'master' && message && message.type === COUNTS) {
+      stats.mergeCounts(message.rows);
+    }
+  };
+  channel.on('message', receive);
+
+  const disconnect = () => {
+    if (timer) clearInterval(timer);
+    channel.removeListener('message', receive);
+  };
+  disconnect.flush = async () => {
+    let wait;
+    const timeout = new Promise(resolve => { wait = setTimeout(resolve, COUNTS_FLUSH_MS); });
+    try {
+      // the lot in flight, then whatever was counted since
+      await Promise.race([(async () => { await inFlight; await send(); })(), timeout]);
+    } finally {
+      clearTimeout(wait);
+    }
+  };
+  return disconnect;
+}
+
+/**
+ * The launcher's side: what the workers sent and the master hasn't had yet, summed by
+ * (module, endpoint, operation).
+ */
+class PendingCounts {
+  constructor() {
+    this.rows = new Map();
+  }
+
+  add(rows) {
+    for (const row of Array.isArray(rows) ? rows : []) {
+      if (!row || !Number.isInteger(row.count) || row.count <= 0 || !Number.isFinite(row.time)) continue;
+      const key = `${row.module}\u001f${row.endpoint}\u001f${row.operation}`;
+      const sum = this.rows.get(key);
+      if (sum) {
+        sum.count += row.count;
+        sum.time += row.time;
+      } else {
+        this.rows.set(key, { module: row.module, endpoint: row.endpoint, operation: row.operation, count: row.count, time: row.time });
+      }
+    }
+  }
+
+  /** Everything pending, now no longer pending (add it back if it can't be delivered). */
+  take() {
+    const rows = [...this.rows.values()];
+    this.rows.clear();
+    return rows;
+  }
+
+  get size() {
+    return this.rows.size;
+  }
+}
+
+module.exports = { connectCounts, PendingCounts, COUNTS, COUNTS_INTERVAL_MS };

package/library/cluster-events.js

@@ -0,0 +1,153 @@
+/**
+ * What the launcher (cluster.js) saw, for the master's daily report (cluster.md, "The daily
+ * report").
+ *
+ * Only the launcher knows when a process exited, was recycled or came back, and what it
+ * refused or abandoned itself. It keeps those as rows: an event (a process exited, was
+ * recycled, was ready again after a time unavailable; the launcher started) is a row of its
+ * own; what the launcher answered itself (503, 504, 502, a cache-id lost to a restart, a
+ * client that took none of a response) is counted, one row per kind and process per lot. It
+ * passes them to the master over its IPC channel, once the master is ready, as it passes the
+ * workers' request counts, and the master writes them to its statistics database
+ * (ServerStats.recordClusterEvents), where the daily report reads them.
+ *
+ * A row is {time, kind, process, detail, count, ms}:
+ *  - time: when it happened (a counted row: the last time in the lot)
+ *  - kind: one of KINDS
+ *  - process: the process's name ('master', 'worker b'), or null for the launcher itself
+ *  - detail: the exit code or signal, or why a process was recycled; otherwise null
+ *  - count: 1 for an event; how many for a counted row
+ *  - ms: for 'ready', how long the process had been unavailable; otherwise 0
+ */
+const CLUSTER_EVENTS = 'fhirsmith:cluster-events';
+
+const KINDS = {
+  // events
+  launcherStarted: 'launcher-started',
+  exited: 'exited',
+  recycled: 'recycled',
+  ready: 'ready',
+  // counted: what the launcher answered itself
+  busy: 'refused-busy',
+  stopping: 'refused-stopping',
+  notReady: 'refused-not-ready',
+  lostToRestart: 'lost-to-restart',
+  timeout: 'upstream-timeout',
+  failed: 'upstream-failed',
+  writeTimeout: 'write-timeout'
+};
+const KNOWN = new Set(Object.values(KINDS));
+// kept one by one; the rest are counted
+const EVENT_KINDS = new Set([KINDS.launcherStarted, KINDS.exited, KINDS.recycled, KINDS.ready]);
+// events kept while the master can't take them (it is down, or not ready yet); the oldest go
+// first. Counted rows are one per kind and process, so they don't grow
+const MAX_PENDING = 5000;
+const MAX_DETAIL = 500;
+
+/**
+ * A row as the master takes it, or null if it isn't one.
+ */
+function validRow(row) {
+  if (!row || !KNOWN.has(row.kind) || !Number.isFinite(row.time) || !Number.isInteger(row.count) || row.count <= 0) {
+    return null;
+  }
+  const text = (v) => (typeof v === 'string' && v ? v.slice(0, MAX_DETAIL) : null);
+  return {
+    time: Math.round(row.time),
+    kind: row.kind,
+    process: text(row.process),
+    detail: text(row.detail),
+    count: row.count,
+    ms: Number.isFinite(row.ms) && row.ms > 0 ? Math.round(row.ms) : 0
+  };
+}
+
+/**
+ * The launcher's side: what hasn't reached the master yet. Shaped like PendingCounts
+ * (library/cluster-counts.js), so the launcher delivers both the same way.
+ */
+class PendingClusterEvents {
+  constructor(now = Date.now) {
+    this.now = now;
+    this.events = [];
+    this.counted = new Map();
+  }
+
+  /** Something that happened once: a process exited, was recycled, was ready again. */
+  event(kind, process = null, { detail = null, ms = 0 } = {}) {
+    this.events.push({ time: this.now(), kind, process, detail: detail == null ? null : String(detail), count: 1, ms });
+    if (this.events.length > MAX_PENDING) {
+      this.events.splice(0, this.events.length - MAX_PENDING);
+    }
+  }
+
+  /** Something the launcher answered itself: counted, not kept one by one. */
+  count(kind, process = null) {
+    const key = `${kind}\u001f${process || ''}`;
+    let row = this.counted.get(key);
+    if (!row) {
+      row = { time: 0, kind, process, detail: null, count: 0, ms: 0 };
+      this.counted.set(key, row);
+    }
+    row.count++;
+    row.time = this.now();
+  }
+
+  /** Put back a lot that couldn't be delivered. */
+  add(rows) {
+    for (const row of Array.isArray(rows) ? rows : []) {
+      if (!EVENT_KINDS.has(row.kind)) {
+        const key = `${row.kind}\u001f${row.process || ''}`;
+        const sum = this.counted.get(key);
+        if (sum) {
+          sum.count += row.count;
+          sum.time = Math.max(sum.time, row.time);
+        } else {
+          this.counted.set(key, { ...row });
+        }
+      } else {
+        this.events.push(row);
+      }
+    }
+    this.events.sort((a, b) => a.time - b.time);
+    if (this.events.length > MAX_PENDING) {
+      this.events.splice(0, this.events.length - MAX_PENDING);
+    }
+  }
+
+  /** Everything pending, now no longer pending (add it back if it can't be delivered). */
+  take() {
+    const rows = [...this.events, ...this.counted.values()];
+    this.events = [];
+    this.counted.clear();
+    return rows;
+  }
+
+  get size() {
+    return this.events.length + this.counted.size;
+  }
+}
+
+/**
+ * The master's side: take what the launcher sends, into the statistics database. A process
+ * without the launcher, or that isn't the master, does nothing.
+ * @param {Object} stats - the ServerStats
+ * @param {Object} [channel] - process, or a stand-in (tests)
+ * @param {Object} [options]
+ * @param {string} [options.role]
+ * @returns {Function} disconnect
+ */
+function connectClusterEvents(stats, channel = process, { role = null } = {}) {
+  if (role !== 'master' || typeof channel.send !== 'function') {
+    return () => {};
+  }
+  const receive = message => {
+    if (message && message.type === CLUSTER_EVENTS) {
+      stats.recordClusterEvents(message.rows);
+    }
+  };
+  channel.on('message', receive);
+  return () => channel.removeListener('message', receive);
+}
+
+module.exports = { CLUSTER_EVENTS, KINDS, EVENT_KINDS, PendingClusterEvents, connectClusterEvents, validRow };

package/library/cluster-metrics.js

@@ -0,0 +1,193 @@
+/**
+ * The launcher's small, in-memory view of each server's existing graph samples.
+ * No request counters are collected by the proxy and no averages are combined.
+ * Histories live until the launcher stops, so a child restart leaves a visible
+ * gap instead of erasing the worker's earlier samples. The one figure the launcher
+ * adds is the memory graph's Total: the cgroup's private memory (or, without a cgroup
+ * of the service's own, the processes' summed), read when the master's sample arrives.
+ */
+const METRIC = 'fhirsmith:metric';
+const SNAPSHOT = 'fhirsmith:metrics';
+const HISTORY_MS = 24 * 60 * 60 * 1000;
+const MAX_POINTS = 1440;
+const FIELDS = ['mem', 'anon', 'rpm', 'tat', 'block', 'blockMax', 'clientConcepts', 'refused', 'intervalStart', 'ext', 'extTat'];
+
+class ClusterMetrics {
+  /**
+   * @param {Object[]} procs - code, role, servesTx
+   * @param {Object} [options] - rejectAt (the memory guard's refusal threshold, bytes), group
+   *   (whether the guard measures the cgroup, which the Total then is), sharedCache
+   */
+  constructor(procs, { rejectAt = null, group = false, sharedCache = false } = {}) {
+    this.rejectAt = rejectAt;
+    this.group = group;
+    this.sharedCache = sharedCache;
+    this.total = [];
+    this.series = new Map(procs.map(p => [p.code, {
+      id: p.code, label: `${p.role === 'master' ? 'Master' : 'Worker'} ${p.code}`,
+      role: p.role, servesTx: p.servesTx, status: 'starting', restarts: 0,
+      startedAt: null, intervalMs: null, history: []
+    }]));
+  }
+
+  /** The memory graph's Total at a time (null: not known, a gap). */
+  recordTotal(time, anon, now = Date.now()) {
+    const last = this.total[this.total.length - 1];
+    if (!Number.isFinite(time) || (last && time <= last.time)) {
+      return;
+    }
+    this.total.push({ time, anon: Number.isFinite(anon) ? anon : null });
+    this.prune(this, now, 'total');
+  }
+
+  prune(series, now, field = 'history') {
+    const cutoff = now - HISTORY_MS;
+    series[field] = series[field].filter(point => point.time > cutoff).slice(-MAX_POINTS);
+  }
+
+  state(code, status, now = Date.now()) {
+    const series = this.series.get(code);
+    if (!series) return;
+    if (status === 'restarting' && series.status !== 'restarting') {
+      series.restarts++;
+      if (series.history.length) {
+        // Chart.js must not connect the last sample to the replacement process.
+        series.history.push({ time: now, ...Object.fromEntries(FIELDS.map(field => [field, null])) });
+      }
+    }
+    series.status = status;
+    this.prune(series, now);
+  }
+
+  record(code, message, now = Date.now()) {
+    const series = this.series.get(code);
+    const point = message && message.point;
+    if (!series || !point || !Number.isFinite(point.time) ||
+        !Number.isFinite(message.startedAt) || !Number.isFinite(message.intervalMs) || message.intervalMs <= 0) {
+      return false;
+    }
+    // Reject malformed or out-of-order samples; a delayed old child is also
+    // rejected by the launcher before it reaches this store.
+    const last = series.history[series.history.length - 1];
+    if (point.time < now - HISTORY_MS || point.time > now + 60000 || (last && point.time <= last.time)) {
+      return false;
+    }
+    const clean = { time: point.time };
+    for (const field of FIELDS) {
+      clean[field] = Number.isFinite(point[field]) ? point[field] : null;
+    }
+    series.startedAt = message.startedAt;
+    series.intervalMs = message.intervalMs;
+    if (last && point.time - last.time > 1.5 * message.intervalMs && FIELDS.some(field => last[field] !== null)) {
+      series.history.push({ time: last.time + message.intervalMs, ...Object.fromEntries(FIELDS.map(field => [field, null])) });
+    }
+    series.history.push(clean);
+    this.prune(series, now);
+    return true;
+  }
+
+  snapshot(now = Date.now()) {
+    this.prune(this, now, 'total');
+    return {
+      cluster: true, sampledAt: now, sharedCache: this.sharedCache,
+      total: { group: this.group, rejectAt: this.rejectAt, history: this.total.map(point => ({ ...point })) },
+      series: [...this.series.values()].map(series => {
+        this.prune(series, now);
+        const last = series.history[series.history.length - 1];
+        const stale = series.status === 'ready' && (!last || now - last.time > Math.max(60000, 2 * series.intervalMs));
+        return { ...series, status: stale ? 'no recent sample' : series.status,
+          history: series.history.map(point => ({ ...point })) };
+      })
+    };
+  }
+}
+
+/** Attach only to a launcher-created IPC channel; standalone servers stay local. */
+function connectMetrics(stats, channel = process) {
+  if (typeof channel.send !== 'function') return () => {};
+  let sending = false;
+  const publish = point => {
+    // Statistics must never block request work or build an IPC backlog.
+    if (channel.connected === false || sending) return;
+    sending = true;
+    try {
+      channel.send({ type: METRIC, point, startedAt: stats.startTime, intervalMs: stats.intervalMs }, () => {
+        sending = false;
+      });
+    } catch {
+      sending = false;
+    }
+  };
+  const receive = message => {
+    if (message && message.type === SNAPSHOT && Array.isArray(message.series)) {
+      stats.clusterMetrics = { cluster: true, sampledAt: message.sampledAt, sharedCache: message.sharedCache === true,
+        total: message.total || null, series: message.series };
+      stats.clusterMetricsUnavailable = false;
+    }
+  };
+  const disconnected = () => { stats.clusterMetricsUnavailable = true; };
+  stats.onMetric = publish;
+  channel.on('message', receive);
+  channel.on('disconnect', disconnected);
+  return () => {
+    if (stats.onMetric === publish) stats.onMetric = null;
+    channel.removeListener('message', receive);
+    channel.removeListener('disconnect', disconnected);
+  };
+}
+
+/**
+ * Serialize only graph data, safely inside an inline script in either page. Without the
+ * launcher's snapshot (no role, a worker's own page, a master before its first), this process's
+ * own samples as one series, with its memory guard's refusal threshold (guard: rejectAt, group:
+ * whether the guard measures the cgroup) and no Total.
+ */
+function graphData(stats, label = 'Server', now = Date.now(), guard = {}) {
+  return JSON.stringify(graphSeries(stats, label, now, guard)).replace(/</g, '\\u003c');
+}
+
+/**
+ * The graphs' data (as graphData, not serialized): the launcher's snapshot, or this process's
+ * own samples as one series
+ */
+function graphSeries(stats, label = 'Server', now = Date.now(), guard = {}) {
+  const data = stats.clusterMetrics || {
+    cluster: false, sampledAt: now,
+    total: Number.isFinite(guard.rejectAt) && guard.rejectAt > 0
+      ? { group: Boolean(guard.group), rejectAt: guard.rejectAt, history: [], local: true } : null,
+    series: [{ id: 'local', label, status: 'ready', restarts: 0, startedAt: stats.startTime,
+      intervalMs: stats.intervalMs, history: stats.history }]
+  };
+  const series = data.series.map(item => {
+    const last = item.history[item.history.length - 1];
+    const stale = item.status === 'ready' && (!last || now - last.time > Math.max(60000, 2 * item.intervalMs));
+    return { ...item, status: data.cluster && stats.clusterMetricsUnavailable ? 'unavailable' : stale ? 'no recent sample' : item.status };
+  });
+  return { ...data, series };
+}
+
+/**
+ * The external requests (those that had to ask another server: ServerStats.countExternalRequest)
+ * in the graphs' samples - every process's, on the master under the launcher: {count, average
+ * (ms), hours (how far back the samples go)}, or null if there were none
+ */
+function externalSummary(stats, now = Date.now()) {
+  let count = 0;
+  let time = 0;
+  let earliest = now;
+  for (const series of graphSeries(stats, 'external', now).series || []) {
+    for (const point of series.history || []) {
+      const from = Number.isFinite(point.intervalStart) ? point.intervalStart : point.time;
+      if (Number.isFinite(from)) {
+        earliest = Math.min(earliest, from);
+      }
+      if (point.ext > 0 && Number.isFinite(point.extTat)) {
+        count += point.ext;
+        time += point.ext * point.extTat;
+      }
+    }
+  }
+  return count > 0 ? { count, average: time / count, hours: (now - earliest) / 3600000 } : null;
+}
+
+module.exports = { ClusterMetrics, connectMetrics, graphData, graphSeries, externalSummary, METRIC, SNAPSHOT, HISTORY_MS, MAX_POINTS };