heyquilt 0.0.1 → 0.3.106

package/src/relay-metrics.js

@@ -0,0 +1,140 @@
+// How busy and how big each room is, for placement (relay-cluster.js) and the relay's /stats:
+// WebSocket messages and bytes in and out, counted per minute; and the room's bytes in memory
+// and on disk. The accounts API reads them every minute and gives a heavy room a machine of its
+// own (src/api/relay-cluster.js), with heavyReasons deciding what heavy is. The machine's own
+// load (memory, event-loop lag, CPU) is MachineMeter's, for scaling the cluster on load.
+import fs from 'node:fs'
+import os from 'node:os'
+import path from 'node:path'
+import { monitorEventLoopDelay } from 'node:perf_hooks'
+
+export const METER_WINDOW_MS = 60 * 1000
+const zero = () => ({ msgsIn: 0, msgsOut: 0, bytesIn: 0, bytesOut: 0 })
+
+/** Counts a room's traffic: this minute's, and the last whole minute's. */
+export class RoomMeter {
+  constructor ({ now = Date.now } = {}) {
+    this.now = now
+    this.at = now() // when this minute started
+    this.cur = zero()
+    this.last = zero()
+    this.sum = zero()
+  }
+
+  roll () {
+    const t = this.now()
+    const n = Math.floor((t - this.at) / METER_WINDOW_MS)
+    if (n <= 0) return
+    this.last = n === 1 ? this.cur : zero()
+    this.cur = zero()
+    this.at += n * METER_WINDOW_MS
+  }
+
+  in (bytes = 0) { this.roll(); this.cur.msgsIn++; this.cur.bytesIn += bytes; this.sum.msgsIn++; this.sum.bytesIn += bytes }
+  out (bytes = 0) { this.roll(); this.cur.msgsOut++; this.cur.bytesOut += bytes; this.sum.msgsOut++; this.sum.bytesOut += bytes }
+
+  /** The last whole minute's counts. */
+  lastMinute () { this.roll(); return { ...this.last } }
+  /** Everything since the room loaded. */
+  totals () { return { ...this.sum } }
+}
+
+function treeBytes (p) {
+  let st
+  try { st = fs.statSync(p) } catch { return 0 }
+  if (!st.isDirectory()) return st.size
+  let n = 0
+  try { for (const f of fs.readdirSync(p)) n += treeBytes(path.join(p, f)) } catch {}
+  return n
+}
+
+/** What a room keeps on the relay's disk: its document, metadata, kept chat, branch documents and chat files. */
+export function roomDiskBytes (dataDir, name) {
+  if (!dataDir) return 0
+  return treeBytes(path.join(dataDir, `${name}.ydoc`)) + treeBytes(path.join(dataDir, `${name}.json`)) +
+    treeBytes(path.join(dataDir, `${name}.chat.jsonl`)) + treeBytes(path.join(dataDir, 'branches', name)) +
+    treeBytes(path.join(dataDir, 'files', name))
+}
+
+// A room this big or this busy gets a machine of its own (settable on the accounts API).
+export const HEAVY = {
+  diskBytes: 128 * 1024 * 1024,
+  docBytes: 48 * 1024 * 1024, // in memory: several times this, on a 512 MB machine
+  ioBytesPerMin: 20 * 1024 * 1024,
+  msgsPerMin: 6000,
+  ioMinutes: 3 // traffic must last this many minutes running; a burst is not a reason to move
+}
+
+/** Whether this minute's traffic is over the limits (one minute of `ioMinutes`). */
+export const busyMinute = (s, limits = HEAVY) => ((s.bytesIn || 0) + (s.bytesOut || 0) >= limits.ioBytesPerMin) || ((s.msgsIn || 0) + (s.msgsOut || 0) >= limits.msgsPerMin)
+
+/** Why a room is heavy ('disk', 'memory', 'io'), or [] when it isn't. `busyMinutes`: busy minutes running, this one included. */
+export function heavyReasons (s, limits = HEAVY, busyMinutes = 0) {
+  const out = []
+  if ((s.diskBytes || 0) >= limits.diskBytes) out.push('disk')
+  if ((s.docBytes || 0) >= limits.docBytes) out.push('memory')
+  if (busyMinute(s, limits) && busyMinutes >= limits.ioMinutes) out.push('io')
+  return out
+}
+
+// How often the event loop is sampled for lag; every sample includes this much on its own.
+export const LAG_RESOLUTION_MS = 20
+
+/** The memory this machine (or its container's limit, when lower) has. */
+export function machineMemory () {
+  const total = os.totalmem()
+  const limit = typeof process.constrainedMemory === 'function' ? Number(process.constrainedMemory()) || 0 : 0
+  return limit > 0 && limit < total ? limit : total
+}
+
+/**
+ * The machine's own load, a minute at a time: event-loop lag (the minute's 99th percentile and
+ * mean, in ms, beyond the sampling interval) and CPU (share of one core). `roll()` closes the
+ * minute (the report does, every minute); `last` is the last closed one.
+ */
+export class MachineMeter {
+  constructor ({ now = Date.now, cpuUsage = (prev) => process.cpuUsage(prev), delay = null, resolutionMs = LAG_RESOLUTION_MS } = {}) {
+    this.now = now
+    this.cpuUsage = cpuUsage
+    this.resolutionMs = resolutionMs
+    this.delay = delay || monitorEventLoopDelay({ resolution: resolutionMs })
+    if (this.delay.enable) this.delay.enable()
+    this.at = now()
+    this.cpu0 = cpuUsage()
+    this.last = { lagMs: 0, lagMeanMs: 0, cpu: 0 }
+  }
+
+  roll () {
+    const t = this.now()
+    const used = this.cpuUsage(this.cpu0)
+    const wallUs = Math.max(1, t - this.at) * 1000
+    const ms = (ns) => Math.max(0, Math.round((Number(ns) || 0) / 1e6 - this.resolutionMs))
+    const sampled = (this.delay.count ?? 1) > 0
+    this.last = {
+      lagMs: sampled ? ms(this.delay.percentile(99)) : 0,
+      lagMeanMs: sampled ? ms(this.delay.mean) : 0,
+      cpu: Math.round(((used.user + used.system) / wallUs) * 1000) / 1000
+    }
+    this.delay.reset()
+    this.at = t
+    this.cpu0 = this.cpuUsage()
+    return this.last
+  }
+
+  stop () { if (this.delay.disable) this.delay.disable() }
+}
+
+/**
+ * A machine's load for the accounts API: memory used (the relay's resident memory) against the
+ * machine's, the last minute's event-loop lag and CPU, and every loaded room's traffic together.
+ */
+export function machineLoad ({ meter, rooms = [], memTotal = machineMemory(), mem = process.memoryUsage() }) {
+  let bytesPerMin = 0
+  let msgsPerMin = 0
+  for (const r of rooms) {
+    bytesPerMin += (r.bytesIn || 0) + (r.bytesOut || 0)
+    msgsPerMin += (r.msgsIn || 0) + (r.msgsOut || 0)
+  }
+  const last = meter ? meter.last : { lagMs: 0, lagMeanMs: 0, cpu: 0 }
+  return { memUsed: mem.rss, memTotal, lagMs: last.lagMs, lagMeanMs: last.lagMeanMs, cpu: last.cpu, bytesPerMin, msgsPerMin }
+}