@adalo/metrics 0.1.173 → 0.1.175

package/src/metrics/httpMetricsRedisStore.js

@@ -0,0 +1,208 @@
+/**
+ * Record separator for hash fields (avoids collisions when route contains "|").
+ * @type {string}
+ */
+const FIELD_SEP = '\x1e'
+
+/**
+ * Default Redis key TTL in seconds (sliding: refreshed on each `record` write).
+ * @type {number}
+ */
+const DEFAULT_HTTP_METRICS_REDIS_TTL_SEC = 120
+
+const DRAIN_LUA = `
+local function drain(key)
+  local v = redis.call('HGETALL', key)
+  redis.call('DEL', key)
+  return v
+end
+return {drain(KEYS[1]), drain(KEYS[2])}
+`
+
+/**
+ * @param {string} method
+ * @param {string} route
+ * @param {number} statusCode
+ * @returns {string}
+ */
+function buildFieldKey(method, route, statusCode) {
+  return [method, route, String(statusCode)].join(FIELD_SEP)
+}
+
+function hgetallPairsToObject(pairs) {
+  const o = {}
+  if (!pairs || !pairs.length) {
+    return o
+  }
+  for (let i = 0; i < pairs.length; i += 2) {
+    o[pairs[i]] = pairs[i + 1]
+  }
+  return o
+}
+
+/**
+ * @param {unknown} raw redis eval result [countPairs, durPairs]
+ * @returns {{ labels: { method: string, route: string, status_code: string }, count: number, dur: number }[]}
+ */
+function rowsFromDrainRaw(raw) {
+  const rows = []
+  if (!raw || !Array.isArray(raw) || raw.length < 2) {
+    return rows
+  }
+  const counts = hgetallPairsToObject(raw[0])
+  const durs = hgetallPairsToObject(raw[1])
+  const fieldKeys = Object.keys(counts)
+  for (const field of fieldKeys) {
+    const count = parseInt(counts[field], 10)
+    if (!count || count < 1) {
+      continue
+    }
+    const dur = parseInt(durs[field] || '0', 10) || 0
+    const parts = field.split(FIELD_SEP)
+    let labels = null
+    if (parts.length === 3) {
+      const [m, route, statusStr] = parts
+      labels = { method: m, route, status_code: statusStr }
+    } else if (parts.length === 5 || parts.length === 6) {
+      const [m, route, statusStr] = parts
+      labels = { method: m, route, status_code: statusStr }
+    }
+    if (!labels) {
+      continue
+    }
+    rows.push({ labels, count, dur })
+  }
+  return rows
+}
+
+/**
+ * Redis HTTP aggregate store. Uses an **injected** client (same pattern as {@link RedisMetricsClient}).
+ * Expects `redis` v3-style API: `multi().hincrby().expire().exec(cb)`, `eval`.
+ *
+ * **Structure:** `countKey` / `durKey` are each **one Redis hash**. Each **field name** encodes
+ * `(method, route, status_code)`. Values: `:count` is HINCRBY 1 per request; `:dur` is summed ms.
+ */
+class HttpMetricsRedisStore {
+  /**
+   * @param {Object} opts
+   * @param {import('redis').RedisClient} opts.redisClient
+   * @param {string} opts.appName BUILD_APP_NAME (key segment)
+   * @param {string} opts.processType logical process for key (e.g. web)
+   * @param {number} [opts.ttlSec] Expire keys after this many seconds (default 120)
+   */
+  constructor({ redisClient, appName, processType, ttlSec }) {
+    if (redisClient == null) {
+      throw new Error('HttpMetricsRedisStore: redisClient is required')
+    }
+    this._client = redisClient
+    this.ttlSec =
+      typeof ttlSec === 'number' && ttlSec > 0
+        ? ttlSec
+        : DEFAULT_HTTP_METRICS_REDIS_TTL_SEC
+    const keySeg = `${encodeURIComponent(appName)}:${encodeURIComponent(
+      processType
+    )}`
+    this.countKey = `metrics:http:v2:${keySeg}:count`
+    this.durKey = `metrics:http:v2:${keySeg}:dur`
+  }
+
+  /**
+   * @returns {import('redis').RedisClient}
+   * @private
+   */
+  _ensureClient() {
+    return this._client
+  }
+
+  /**
+   * @param {string} method
+   * @param {string} route
+   * @param {number} statusCode
+   * @param {number} durationMs
+   */
+  record(method, route, statusCode, durationMs) {
+    try {
+      const client = this._ensureClient()
+      const field = buildFieldKey(method, route, statusCode)
+      const dur = Math.max(0, Math.round(Number(durationMs) || 0))
+      client
+        .multi()
+        .hincrby(this.countKey, field, 1)
+        .hincrby(this.durKey, field, dur)
+        .expire(this.countKey, this.ttlSec)
+        .expire(this.durKey, this.ttlSec)
+        .exec(err => {
+          if (err) {
+            console.error('[HttpMetricsRedisStore] record failed:', err.message)
+          }
+        })
+    } catch (e) {
+      console.error('[HttpMetricsRedisStore] record:', e.message)
+    }
+  }
+
+  /**
+   * Atomically drain Redis hashes (same Lua as `flushToCounters`) and return parsed rows.
+   *
+   * @returns {Promise<{ ok: boolean, rows: { labels: Object, count: number, dur: number }[] }>}
+   */
+  drainRows() {
+    let client
+    try {
+      client = this._ensureClient()
+    } catch (e) {
+      console.error('[HttpMetricsRedisStore] drainRows:', e.message)
+      return Promise.resolve({ ok: false, rows: [] })
+    }
+    return new Promise(resolve => {
+      client.eval(DRAIN_LUA, 2, this.countKey, this.durKey, (evalErr, raw) => {
+        if (evalErr) {
+          console.error(
+            '[HttpMetricsRedisStore] drain failed:',
+            evalErr.message
+          )
+          resolve({ ok: false, rows: [] })
+          return
+        }
+        try {
+          const rows = rowsFromDrainRaw(raw)
+          resolve({ ok: true, rows })
+        } catch (e) {
+          console.error(
+            '[HttpMetricsRedisStore] drainRows parse failed:',
+            e.message
+          )
+          resolve({ ok: false, rows: [] })
+        }
+      })
+    })
+  }
+
+  /**
+   * @param {(labels: Object, value: number) => void} applyCount
+   * @param {(labels: Object, value: number) => void} applyDuration
+   * @returns {Promise<boolean>}
+   */
+  flushToCounters(applyCount, applyDuration) {
+    return this.drainRows().then(({ ok, rows }) => {
+      if (!ok) {
+        return false
+      }
+      for (const row of rows) {
+        applyCount(row.labels, row.count)
+        if (row.dur > 0) {
+          applyDuration(row.labels, row.dur)
+        }
+      }
+      return true
+    })
+  }
+}
+
+module.exports = {
+  HttpMetricsRedisStore,
+  buildFieldKey,
+  FIELD_SEP,
+  DEFAULT_HTTP_METRICS_REDIS_TTL_SEC,
+  rowsFromDrainRaw,
+}

package/src/metrics/metricsClient.js

@@ -4,32 +4,38 @@ const { BaseMetricsClient } = require('./baseMetricsClient')
 
 /**
  * MetricsClient handles Prometheus metrics collection and push.
- * Supports gauges, counters, default metrics, and custom metrics.
- * Extends BaseMetricsClient for common functionality.
+ * Supports gauges, default process metrics, optional HTTP counters, and custom metrics.
+ *
+ * **HTTP metrics:** In-process counters only (`app_requests_*`), gated by `httpMetricsEnabled` /
+ * `METRICS_HTTP_ENABLED`. For Redis-backed HTTP aggregation (multi-web / cluster), use
+ * {@link HttpMetricsRedisRecorder} and {@link HttpMetricsRedisCollector} — not this class.
+ *
+ * @extends BaseMetricsClient
  */
 class MetricsClient extends BaseMetricsClient {
   /**
-   * @param {Object} config
+   * @param {Object} [config]
    * @param {string} [config.appName] Name of the application
    * @param {string} [config.dynoId] Dyno/instance ID
    * @param {string} [config.processType] Process type (web, worker, etc.)
    * @param {boolean} [config.enabled] Enable metrics collection
-   * @param {boolean} [config.httpMetricsEnabled=false] Enable HTTP request metrics (app_requests_total, app_requests_total_duration)
+   * @param {boolean} [config.httpMetricsEnabled] Enable HTTP request metrics (`app_requests_total`, `app_requests_total_duration`); defaults from `METRICS_HTTP_ENABLED === 'true'`
    * @param {boolean} [config.logValues] Log metrics values to console
-   * @param {string} [config.pushgatewayUrl] PushGateway URL
-   * @param {string} [config.pushgatewaySecret] PushGateway secret token
+   * @param {string} [config.pushgatewayUrl] Push URL (VM-agent import endpoint, e.g. .../api/v1/import/prometheus). /metrics is for GET (scrape), not POST (push).
+   * @param {string} [config.pushgatewaySecret] Basic auth secret (Base64 of `user:password`)
    * @param {number} [config.intervalSec] Interval in seconds for pushing metrics
    * @param {boolean} [config.removeOldMetrics] Enable to clear metrics by service name
-   * @param {function} [config.startupValidation] Add to validate on start push.
-   * @param {boolean} [config.disablePushgateway] Disable pushing to Pushgateway (use HTTP scraping instead)
+   * @param {function} [config.startupValidation] Add to validate on start push
+   * @param {boolean} [config.disablePushgateway] Disable pushing to VM-agent (use HTTP scraping instead)
+   * @param {boolean} [config.blockNodeDefaultMetrics] When true, skip prom-client default process metrics (rare; see {@link BaseMetricsClient})
    */
   constructor(config = {}) {
     super(config)
 
     this.httpMetricsEnabled =
-      config.httpMetricsEnabled ??
-      process.env.METRICS_HTTP_ENABLED === 'true' ??
-      false
+      config.httpMetricsEnabled !== undefined
+        ? config.httpMetricsEnabled
+        : process.env.METRICS_HTTP_ENABLED === 'true'
 
     this._lastUsageMicros = 0
     this._lastCheckTime = Date.now()
@@ -85,8 +91,6 @@ class MetricsClient extends BaseMetricsClient {
         labelNames: this.withDefaultLabelsWithoutDynoId([
           'method',
           'route',
-          'appId',
-          'databaseId',
           'status_code',
         ]),
         useLabelsWithoutDynoId: true,
@@ -98,8 +102,6 @@ class MetricsClient extends BaseMetricsClient {
         labelNames: this.withDefaultLabelsWithoutDynoId([
           'method',
           'route',
-          'appId',
-          'databaseId',
           'status_code',
         ]),
         useLabelsWithoutDynoId: true,
@@ -139,7 +141,7 @@ class MetricsClient extends BaseMetricsClient {
   }
 
   /**
-   * Get available CPU cores.
+   * Available CPU cores (cgroup quota or `os.cpus().length`).
    * @returns {number}
    */
   getAvailableCPUs() {
@@ -160,7 +162,7 @@ class MetricsClient extends BaseMetricsClient {
   }
 
   /**
-   * Get container memory usage in bytes.
+   * Container memory usage in bytes (`memory.current` or RSS fallback).
    * @returns {number}
    */
   getContainerMemoryUsage() {
@@ -175,7 +177,7 @@ class MetricsClient extends BaseMetricsClient {
   }
 
   /**
-   * Get container memory limit in bytes.
+   * Container memory limit in bytes (`memory.max` or host total).
    * @returns {number}
    */
   getContainerMemoryLimit() {
@@ -195,7 +197,7 @@ class MetricsClient extends BaseMetricsClient {
   }
 
   /**
-   * Measure event loop lag in ms.
+   * Event loop lag sample in milliseconds.
    * @returns {Promise<number>}
    */
   measureLag() {
@@ -206,48 +208,39 @@ class MetricsClient extends BaseMetricsClient {
   }
 
   /**
-   * Increment the HTTP requests counter with detailed request information.
+   * Increment HTTP request counters (in-process). No-op if `httpMetricsEnabled` is false.
    *
-   * @param {Object} params - The parameters for the request counter.
-   * @param {string} params.method - HTTP method (GET, POST, etc.).
-   * @param {string} params.route - The full requested URL or route.
-   * @param {number} params.status_code - HTTP response status code.
-   * @param {string} [params.appId=''] - Optional application identifier.
-   * @param {string} [params.databaseId=''] - Optional database identifier.
-   * @param {number} params.duration - Request duration in milliseconds.
+   * @param {Object} params
+   * @param {string} params.method HTTP method
+   * @param {string} params.route Route or path pattern
+   * @param {number} params.status_code HTTP status code
+   * @param {number} params.duration Duration in milliseconds
    */
-  trackHttpRequest({
-    method,
-    route,
-    status_code,
-    appId = '',
-    databaseId = '',
-    duration,
-  }) {
+  trackHttpRequest({ method, route, status_code, duration }) {
     if (!this.httpMetricsEnabled) return
 
     this.countersFunctions?.app_requests_total({
       method,
       route,
       status_code,
-      appId,
-      databaseId,
     })
     this.countersFunctions?.app_requests_total_duration(
       {
         method,
         route,
         status_code,
-        appId,
-        databaseId,
       },
       duration
     )
   }
 
   /**
-   * Express middleware to track HTTP requests.
-   * Track the `app_requests_total` and `app_requests_total_duration` metric.
+   * Express middleware: records `app_requests_*` on response finish.
+   * Skips when disabled, HTTP metrics off, or `OPTIONS`.
+   *
+   * @param {import('http').IncomingMessage} req
+   * @param {import('http').ServerResponse} res
+   * @param {function} next
    */
   trackHttpRequestMiddleware = (req, res, next) => {
     if (!this.enabled || !this.httpMetricsEnabled || req.method === 'OPTIONS') {
@@ -258,23 +251,11 @@ class MetricsClient extends BaseMetricsClient {
     const start = Date.now()
     res.on('finish', () => {
       const route = req.route?.path || req.path || 'unknown'
-      const appId =
-        req.params?.appId || req.body?.appId || req.query?.appId || ''
-      const databaseId =
-        req.params?.databaseId ||
-        req.body?.databaseId ||
-        req.query?.databaseId ||
-        req.params?.datasourceId ||
-        req.body?.datasourceId ||
-        req.query?.datasourceId ||
-        ''
 
       this.trackHttpRequest({
         method: req.method,
         route,
         status_code: res.statusCode,
-        appId,
-        databaseId,
         duration: Date.now() - start,
       })
     })

package/src/metrics/metricsDatabaseClient.js

@@ -1,5 +1,9 @@
 const { Pool } = require('pg')
 const { BaseMetricsClient } = require('./baseMetricsClient')
+const {
+  exitUnlessProcessTypeIs,
+  METRICS_PROCESS_TYPE_DATABASE,
+} = require('./metricsProcessTypeUtils')
 
 /**
  * DatabaseMetricsClient collects Postgres connection metrics
@@ -31,6 +35,8 @@ class DatabaseMetricsClient extends BaseMetricsClient {
     additional_database_urls = {},
     ...metricsConfig
   } = {}) {
+    exitUnlessProcessTypeIs(metricsConfig, METRICS_PROCESS_TYPE_DATABASE)
+
     const intervalSec =
       metricsConfig.intervalSec ||
       parseInt(process.env.METRICS_DATABASE_INTERVAL_SEC || '', 10) ||
@@ -78,7 +84,7 @@ class DatabaseMetricsClient extends BaseMetricsClient {
 
     super({
       ...metricsConfig,
-      processType: metricsConfig.processType || 'database-metrics',
+      processType: metricsConfig.processType || METRICS_PROCESS_TYPE_DATABASE,
       intervalSec,
       startupValidation,
     })

package/src/metrics/metricsProcessTypeUtils.js

@@ -0,0 +1,98 @@
+/**
+ * Helpers for resolving `processType` and silently exiting when a specialized metrics client
+ * is constructed on the wrong dyno / process (no log output).
+ *
+ * **Canonical names** align with typical Procfile processes (e.g. backend: `web`, `worker`,
+ * `queue-metrics`, `database-metrics`, `http-metrics`). The compile **worker** dyno serves queues/builds
+ * — it does not run HTTP request metrics. HTTP Redis **key segment** for API traffic is fixed **`web`**
+ * (`HttpMetricsRedisCollector` / `HttpMetricsRedisRecorder`), not `BUILD_DYNO_PROCESS_TYPE`.
+ *
+ * @module metrics/metricsProcessTypeUtils
+ */
+
+/** DB-only metrics dyno (`database-metrics` in Procfile). */
+const METRICS_PROCESS_TYPE_DATABASE = 'database-metrics'
+
+/** Queue + Redis metrics dyno (`queue-metrics` in Procfile). */
+const METRICS_PROCESS_TYPE_QUEUE = 'queue-metrics'
+
+/** Redis-only metrics dyno (no Bee Queue; optional separate process). */
+const METRICS_PROCESS_TYPE_REDIS = 'redis-metrics'
+
+/** Web servers — HTTP traffic, HTTP Redis **writers** typically use this in Redis key segment. */
+const METRICS_PROCESS_TYPE_WEB = 'web'
+
+/** Build/compile workers and similar (e.g. backend `worker:`) — no HTTP server metrics here. */
+const METRICS_PROCESS_TYPE_WORKER = 'worker'
+
+/**
+ * Parent {@link RedisMetricsClient} allows either redis-only or queue stack (`QueueRedisMetricsClient`).
+ * @type {readonly string[]}
+ */
+const REDIS_METRICS_CLIENT_ALLOWED_PROCESS_TYPES = Object.freeze([
+  METRICS_PROCESS_TYPE_REDIS,
+  METRICS_PROCESS_TYPE_QUEUE,
+])
+
+/**
+ * Resolve logical process type the same way specialized metrics clients do.
+ *
+ * @param {{ processType?: string }} metricsConfig - Remainder of constructor options (e.g. after destructuring `redisClient`, `databaseUrl`, …)
+ * @param {string} defaultProcessType - Fallback when `metricsConfig.processType` and `BUILD_DYNO_PROCESS_TYPE` are unset
+ * @returns {string}
+ */
+function resolveMetricsProcessType(metricsConfig, defaultProcessType) {
+  return (
+    metricsConfig.processType ||
+    process.env.BUILD_DYNO_PROCESS_TYPE ||
+    defaultProcessType
+  )
+}
+
+/**
+ * Exit with no logs if the resolved process type is not exactly `expectedProcessType`.
+ *
+ * @param {{ processType?: string }} metricsConfig
+ * @param {string} expectedProcessType - Single allowed value (use a module constant, e.g. {@link METRICS_PROCESS_TYPE_DATABASE})
+ * @returns {void}
+ */
+function exitUnlessProcessTypeIs(metricsConfig, expectedProcessType) {
+  const resolved = resolveMetricsProcessType(metricsConfig, expectedProcessType)
+  if (resolved !== expectedProcessType) {
+    process.exit(0)
+  }
+}
+
+/**
+ * Exit with no logs if the resolved process type is not in `allowedProcessTypes`.
+ *
+ * @param {{ processType?: string }} metricsConfig
+ * @param {readonly string[]} allowedProcessTypes
+ * @param {string} defaultWhenUnspecified - Used only to resolve when config/env omit `processType` (e.g. {@link METRICS_PROCESS_TYPE_QUEUE} for {@link RedisMetricsClient})
+ * @returns {void}
+ */
+function exitUnlessProcessTypeIn(
+  metricsConfig,
+  allowedProcessTypes,
+  defaultWhenUnspecified
+) {
+  const resolved = resolveMetricsProcessType(
+    metricsConfig,
+    defaultWhenUnspecified
+  )
+  if (!allowedProcessTypes.includes(resolved)) {
+    process.exit(0)
+  }
+}
+
+module.exports = {
+  resolveMetricsProcessType,
+  exitUnlessProcessTypeIs,
+  exitUnlessProcessTypeIn,
+  METRICS_PROCESS_TYPE_DATABASE,
+  METRICS_PROCESS_TYPE_QUEUE,
+  METRICS_PROCESS_TYPE_REDIS,
+  METRICS_PROCESS_TYPE_WEB,
+  METRICS_PROCESS_TYPE_WORKER,
+  REDIS_METRICS_CLIENT_ALLOWED_PROCESS_TYPES,
+}

package/src/metrics/metricsQueueRedisClient.js

@@ -1,5 +1,9 @@
 const { RedisMetricsClient } = require('./metricsRedisClient')
 const { IOREDIS, REDIS_V3, REDIS_V4 } = require('../redisUtils')
+const {
+  exitUnlessProcessTypeIs,
+  METRICS_PROCESS_TYPE_QUEUE,
+} = require('./metricsProcessTypeUtils')
 
 /**
  * QueueRedisMetricsClient extends RedisMetricsClient to collect
@@ -24,6 +28,8 @@ class QueueRedisMetricsClient extends RedisMetricsClient {
    * @param {boolean} [options.disablePushgateway] - Disable pushing to Pushgateway (use HTTP scraping instead)
    */
   constructor({ redisClient, ...metricsConfig } = {}) {
+    exitUnlessProcessTypeIs(metricsConfig, METRICS_PROCESS_TYPE_QUEUE)
+
     const getConfiguredQueueNames = () => {
       if (!process.env.METRICS_APP_REDIS_BQ) {
         throw new Error(

package/src/metrics/metricsRedisClient.js

@@ -5,6 +5,11 @@ const {
   IOREDIS,
   REDIS_V3,
 } = require('../redisUtils')
+const {
+  exitUnlessProcessTypeIn,
+  METRICS_PROCESS_TYPE_QUEUE,
+  REDIS_METRICS_CLIENT_ALLOWED_PROCESS_TYPES,
+} = require('./metricsProcessTypeUtils')
 
 const redisConnectionStableFields = ['name', 'flags', 'cmd']
 const redisConnectionFields = ['name', 'flags', 'tot-mem', 'cmd']
@@ -36,6 +41,12 @@ class RedisMetricsClient extends BaseMetricsClient {
       throw new Error('RedisMetricsClient requires redisClient')
     }
 
+    exitUnlessProcessTypeIn(
+      metricsConfig,
+      REDIS_METRICS_CLIENT_ALLOWED_PROCESS_TYPES,
+      METRICS_PROCESS_TYPE_QUEUE
+    )
+
     const intervalSec =
       metricsConfig.intervalSec ||
       parseInt(process.env.METRICS_QUEUE_INTERVAL_SEC || '', 10) ||
@@ -43,7 +54,7 @@ class RedisMetricsClient extends BaseMetricsClient {
 
     super({
       ...metricsConfig,
-      processType: metricsConfig.processType || 'queue-metrics',
+      processType: metricsConfig.processType || METRICS_PROCESS_TYPE_QUEUE,
       intervalSec,
     })