@naturalcycles/backend-lib 9.84.0 → 9.84.2

package/dist/server/eventLoop.util.d.ts

@@ -10,13 +10,19 @@ import type { Integer, NonNegativeInteger, NumberOfMilliseconds, NumberOfPercent
  * expect p50/mean near 0 on a healthy service, instead of the ~`resolution` baseline
  * reported by older interval-based sampling.
  *
- * @experimental
+ * Beware of CPU-throttled processes (e.g. idle CloudRun instances with request-based billing):
+ * the time the process spent runnable-but-unscheduled is recorded as event loop delay,
+ * and the interval timer fires late (see `intervalMs`).
+ * Use `pause()` / `resume()` to keep such periods out of the delay samples,
+ * e.g. pause when the last in-flight request finishes and resume when the next one arrives.
  */
 export declare class EventLoopMonitor {
     constructor(cfg?: EventLoopMonitorCfg);
     private interval;
     private eld;
     private lastElu;
+    private lastReportedAt;
+    private paused;
     /**
      * Undefined until the first interval has completed.
      */
@@ -25,6 +31,22 @@ export declare class EventLoopMonitor {
     gcCount: number;
     gcTotalTime: number;
     stop(): void;
+    /**
+     * Stops sampling event loop delay until `resume()` is called.
+     * GC and ELU keep being tracked, and the samples recorded so far are kept.
+     *
+     * Meant for periods when the process is known to be CPU-throttled, e.g. a CloudRun instance
+     * (request-based billing) with no request in flight: a delay recorded there measures
+     * the throttling, not the application.
+     */
+    pause(): void;
+    /**
+     * Resumes sampling event loop delay after `pause()`.
+     * Safe to call from within the event that ended the throttled period (e.g. the middleware of
+     * the request that woke the instance): the histogram restarts its clock from this moment,
+     * so the stall that preceded it in the same loop iteration is not sampled.
+     */
+    resume(): void;
 }
 export interface EventLoopMonitorCfg {
     /**
@@ -53,15 +75,15 @@ export interface EventLoopStats {
      */
     elu: NumberOfPercent;
     /**
-     * Number of times gc ran during the last `measureInterval`
+     * Number of times gc ran during the last interval
      */
     gcCount: NonNegativeInteger;
     /**
-     * Total time spent on gc during the last `measureInterval`
+     * Total time spent on gc during the last interval
      */
     gcTotalTime: NumberOfMilliseconds;
     /**
-     * CPU time spent on GC in the last `measureInterval`.
+     * CPU time spent on GC in the last interval, relative to `intervalMs`.
      * In "basis points", 100 basis points == 1%.
      * E.g 125 means 1.25% of CPU spent on GC.
      * It's reported in BPS to be able to do more accurate aggregate reporting
@@ -69,4 +91,12 @@ export interface EventLoopStats {
      * too "lossy")
      */
     gcCPU: Integer;
+    /**
+     * Actual wall-clock duration of the interval covered by these stats.
+     * Normally equals `measureInterval`, but is longer when the timer fired late,
+     * e.g. on a CloudRun instance that was CPU-throttled while idle (request-based billing).
+     * An interval much longer than `measureInterval` means the delay stats are dominated
+     * by that throttling rather than by the application.
+     */
+    intervalMs: NumberOfMilliseconds;
 }

package/dist/server/eventLoop.util.js

@@ -26,7 +26,11 @@ import { monitorEventLoopDelay, performance, PerformanceObserver } from 'node:pe
  * expect p50/mean near 0 on a healthy service, instead of the ~`resolution` baseline
  * reported by older interval-based sampling.
  *
- * @experimental
+ * Beware of CPU-throttled processes (e.g. idle CloudRun instances with request-based billing):
+ * the time the process spent runnable-but-unscheduled is recorded as event loop delay,
+ * and the interval timer fires late (see `intervalMs`).
+ * Use `pause()` / `resume()` to keep such periods out of the delay samples,
+ * e.g. pause when the last in-flight request finishes and resume when the next one arrives.
  */
 export class EventLoopMonitor {
     constructor(cfg = {}) {
@@ -41,6 +45,7 @@ export class EventLoopMonitor {
         this.eld = monitorEventLoopDelay(eldOptions);
         this.eld.enable();
         this.lastElu = performance.eventLoopUtilization();
+        this.lastReportedAt = performance.now();
         this.po = new PerformanceObserver(list => {
             for (const entry of list.getEntries()) {
                 this.gcCount++;
@@ -56,11 +61,17 @@ export class EventLoopMonitor {
             const p99 = Math.round(eld.percentile(99) / 1e6);
             const max = Math.round(eld.max / 1e6);
             const mean = Math.round(eld.mean / 1e6);
+            // Actual wall-clock length of this interval, as opposed to the nominal `measureInterval`.
+            // Timers fire late when the process is starved of CPU, so this can be much longer.
+            // Floored at 1 to never divide by zero (only reachable with fake timers).
+            const now = performance.now();
+            const intervalMs = Math.max(1, Math.round(now - this.lastReportedAt));
+            this.lastReportedAt = now;
             const currentElu = performance.eventLoopUtilization();
             const deltaElu = performance.eventLoopUtilization(this.lastElu, currentElu);
             this.lastElu = currentElu;
             const elu = Math.round(deltaElu.utilization * 100);
-            const gcCPU = Math.round((gcTotalTime / measureInterval) * 10_000);
+            const gcCPU = Math.round((gcTotalTime / intervalMs) * 10_000);
             this.lastStats = {
                 p50,
                 p90,
@@ -71,6 +82,7 @@ export class EventLoopMonitor {
                 gcCount,
                 gcTotalTime,
                 gcCPU,
+                intervalMs,
             };
             cfg.onStats?.(this.lastStats);
             eld.reset();
@@ -81,6 +93,8 @@ export class EventLoopMonitor {
     interval;
     eld;
     lastElu;
+    lastReportedAt;
+    paused = false;
     /**
      * Undefined until the first interval has completed.
      */
@@ -93,4 +107,28 @@ export class EventLoopMonitor {
         this.eld.disable();
         this.po.disconnect();
     }
+    /**
+     * Stops sampling event loop delay until `resume()` is called.
+     * GC and ELU keep being tracked, and the samples recorded so far are kept.
+     *
+     * Meant for periods when the process is known to be CPU-throttled, e.g. a CloudRun instance
+     * (request-based billing) with no request in flight: a delay recorded there measures
+     * the throttling, not the application.
+     */
+    pause() {
+        this.paused = true;
+        this.eld.disable();
+    }
+    /**
+     * Resumes sampling event loop delay after `pause()`.
+     * Safe to call from within the event that ended the throttled period (e.g. the middleware of
+     * the request that woke the instance): the histogram restarts its clock from this moment,
+     * so the stall that preceded it in the same loop iteration is not sampled.
+     */
+    resume() {
+        if (!this.paused)
+            return;
+        this.paused = false;
+        this.eld.enable();
+    }
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/backend-lib",
   "type": "module",
-  "version": "9.84.0",
+  "version": "9.84.2",
   "dependencies": {
     "@naturalcycles/db-lib": "^10",
     "@naturalcycles/js-lib": "^15",

package/src/server/eventLoop.util.ts

@@ -35,7 +35,11 @@ import type {
  * expect p50/mean near 0 on a healthy service, instead of the ~`resolution` baseline
  * reported by older interval-based sampling.
  *
- * @experimental
+ * Beware of CPU-throttled processes (e.g. idle CloudRun instances with request-based billing):
+ * the time the process spent runnable-but-unscheduled is recorded as event loop delay,
+ * and the interval timer fires late (see `intervalMs`).
+ * Use `pause()` / `resume()` to keep such periods out of the delay samples,
+ * e.g. pause when the last in-flight request finishes and resume when the next one arrives.
  */
 export class EventLoopMonitor {
   constructor(cfg: EventLoopMonitorCfg = {}) {
@@ -52,6 +56,7 @@ export class EventLoopMonitor {
     this.eld.enable()
 
     this.lastElu = performance.eventLoopUtilization()
+    this.lastReportedAt = performance.now()
 
     this.po = new PerformanceObserver(list => {
       for (const entry of list.getEntries()) {
@@ -71,12 +76,19 @@ export class EventLoopMonitor {
       const max = Math.round(eld.max / 1e6)
       const mean = Math.round(eld.mean / 1e6)
 
+      // Actual wall-clock length of this interval, as opposed to the nominal `measureInterval`.
+      // Timers fire late when the process is starved of CPU, so this can be much longer.
+      // Floored at 1 to never divide by zero (only reachable with fake timers).
+      const now = performance.now()
+      const intervalMs = Math.max(1, Math.round(now - this.lastReportedAt))
+      this.lastReportedAt = now
+
       const currentElu = performance.eventLoopUtilization()
       const deltaElu = performance.eventLoopUtilization(this.lastElu, currentElu)
       this.lastElu = currentElu
 
       const elu = Math.round(deltaElu.utilization * 100)
-      const gcCPU = Math.round((gcTotalTime / measureInterval) * 10_000)
+      const gcCPU = Math.round((gcTotalTime / intervalMs) * 10_000)
 
       this.lastStats = {
         p50,
@@ -88,6 +100,7 @@ export class EventLoopMonitor {
         gcCount,
         gcTotalTime,
         gcCPU,
+        intervalMs,
       }
 
       cfg.onStats?.(this.lastStats)
@@ -101,6 +114,8 @@ export class EventLoopMonitor {
   private interval: NodeJS.Timeout
   private eld: ELDHistogram
   private lastElu: EventLoopUtilization
+  private lastReportedAt: NumberOfMilliseconds
+  private paused = false
   /**
    * Undefined until the first interval has completed.
    */
@@ -115,6 +130,31 @@ export class EventLoopMonitor {
     this.po.disconnect()
   }
 
+  /**
+   * Stops sampling event loop delay until `resume()` is called.
+   * GC and ELU keep being tracked, and the samples recorded so far are kept.
+   *
+   * Meant for periods when the process is known to be CPU-throttled, e.g. a CloudRun instance
+   * (request-based billing) with no request in flight: a delay recorded there measures
+   * the throttling, not the application.
+   */
+  pause(): void {
+    this.paused = true
+    this.eld.disable()
+  }
+
+  /**
+   * Resumes sampling event loop delay after `pause()`.
+   * Safe to call from within the event that ended the throttled period (e.g. the middleware of
+   * the request that woke the instance): the histogram restarts its clock from this moment,
+   * so the stall that preceded it in the same loop iteration is not sampled.
+   */
+  resume(): void {
+    if (!this.paused) return
+    this.paused = false
+    this.eld.enable()
+  }
+
   // cfg: Required<EventLoopMonitorCfg>
 }
 
@@ -146,15 +186,15 @@ export interface EventLoopStats {
    */
   elu: NumberOfPercent
   /**
-   * Number of times gc ran during the last `measureInterval`
+   * Number of times gc ran during the last interval
    */
   gcCount: NonNegativeInteger
   /**
-   * Total time spent on gc during the last `measureInterval`
+   * Total time spent on gc during the last interval
    */
   gcTotalTime: NumberOfMilliseconds
   /**
-   * CPU time spent on GC in the last `measureInterval`.
+   * CPU time spent on GC in the last interval, relative to `intervalMs`.
    * In "basis points", 100 basis points == 1%.
    * E.g 125 means 1.25% of CPU spent on GC.
    * It's reported in BPS to be able to do more accurate aggregate reporting
@@ -162,4 +202,12 @@ export interface EventLoopStats {
    * too "lossy")
    */
   gcCPU: Integer
+  /**
+   * Actual wall-clock duration of the interval covered by these stats.
+   * Normally equals `measureInterval`, but is longer when the timer fired late,
+   * e.g. on a CloudRun instance that was CPU-throttled while idle (request-based billing).
+   * An interval much longer than `measureInterval` means the delay stats are dominated
+   * by that throttling rather than by the application.
+   */
+  intervalMs: NumberOfMilliseconds
 }