@naturalcycles/backend-lib 9.83.0 → 9.84.1

package/dist/server/eventLoop.util.d.ts

@@ -10,13 +10,16 @@ import type { Integer, NonNegativeInteger, NumberOfMilliseconds, NumberOfPercent
  * expect p50/mean near 0 on a healthy service, instead of the ~`resolution` baseline
  * reported by older interval-based sampling.
  *
- * @experimental
+ * Beware of CPU-throttled processes (e.g. idle CloudRun instances with request-based billing):
+ * the interval timer fires late there, and the time the process spent runnable-but-unscheduled
+ * is recorded as event loop delay. Use `intervalMs` to detect such intervals.
  */
 export declare class EventLoopMonitor {
     constructor(cfg?: EventLoopMonitorCfg);
     private interval;
     private eld;
     private lastElu;
+    private lastReportedAt;
     /**
      * Undefined until the first interval has completed.
      */
@@ -53,15 +56,15 @@ export interface EventLoopStats {
      */
     elu: NumberOfPercent;
     /**
-     * Number of times gc ran during the last `measureInterval`
+     * Number of times gc ran during the last interval
      */
     gcCount: NonNegativeInteger;
     /**
-     * Total time spent on gc during the last `measureInterval`
+     * Total time spent on gc during the last interval
      */
     gcTotalTime: NumberOfMilliseconds;
     /**
-     * CPU time spent on GC in the last `measureInterval`.
+     * CPU time spent on GC in the last interval, relative to `intervalMs`.
      * In "basis points", 100 basis points == 1%.
      * E.g 125 means 1.25% of CPU spent on GC.
      * It's reported in BPS to be able to do more accurate aggregate reporting
@@ -69,4 +72,12 @@ export interface EventLoopStats {
      * too "lossy")
      */
     gcCPU: Integer;
+    /**
+     * Actual wall-clock duration of the interval covered by these stats.
+     * Normally equals `measureInterval`, but is longer when the timer fired late,
+     * e.g. on a CloudRun instance that was CPU-throttled while idle (request-based billing).
+     * An interval much longer than `measureInterval` means the delay stats are dominated
+     * by that throttling rather than by the application.
+     */
+    intervalMs: NumberOfMilliseconds;
 }

package/dist/server/eventLoop.util.js

@@ -26,7 +26,9 @@ import { monitorEventLoopDelay, performance, PerformanceObserver } from 'node:pe
  * expect p50/mean near 0 on a healthy service, instead of the ~`resolution` baseline
  * reported by older interval-based sampling.
  *
- * @experimental
+ * Beware of CPU-throttled processes (e.g. idle CloudRun instances with request-based billing):
+ * the interval timer fires late there, and the time the process spent runnable-but-unscheduled
+ * is recorded as event loop delay. Use `intervalMs` to detect such intervals.
  */
 export class EventLoopMonitor {
     constructor(cfg = {}) {
@@ -41,6 +43,7 @@ export class EventLoopMonitor {
         this.eld = monitorEventLoopDelay(eldOptions);
         this.eld.enable();
         this.lastElu = performance.eventLoopUtilization();
+        this.lastReportedAt = performance.now();
         this.po = new PerformanceObserver(list => {
             for (const entry of list.getEntries()) {
                 this.gcCount++;
@@ -56,11 +59,17 @@ export class EventLoopMonitor {
             const p99 = Math.round(eld.percentile(99) / 1e6);
             const max = Math.round(eld.max / 1e6);
             const mean = Math.round(eld.mean / 1e6);
+            // Actual wall-clock length of this interval, as opposed to the nominal `measureInterval`.
+            // Timers fire late when the process is starved of CPU, so this can be much longer.
+            // Floored at 1 to never divide by zero (only reachable with fake timers).
+            const now = performance.now();
+            const intervalMs = Math.max(1, Math.round(now - this.lastReportedAt));
+            this.lastReportedAt = now;
             const currentElu = performance.eventLoopUtilization();
             const deltaElu = performance.eventLoopUtilization(this.lastElu, currentElu);
             this.lastElu = currentElu;
             const elu = Math.round(deltaElu.utilization * 100);
-            const gcCPU = Math.round((gcTotalTime / measureInterval) * 10_000);
+            const gcCPU = Math.round((gcTotalTime / intervalMs) * 10_000);
             this.lastStats = {
                 p50,
                 p90,
@@ -71,6 +80,7 @@ export class EventLoopMonitor {
                 gcCount,
                 gcTotalTime,
                 gcCPU,
+                intervalMs,
             };
             cfg.onStats?.(this.lastStats);
             eld.reset();
@@ -81,6 +91,7 @@ export class EventLoopMonitor {
     interval;
     eld;
     lastElu;
+    lastReportedAt;
     /**
      * Undefined until the first interval has completed.
      */

package/dist/testing/express.test.service.d.ts

@@ -7,7 +7,7 @@ export interface ExpressApp extends Fetcher, AsyncDisposable {
 declare class ExpressTestService {
     createAppFromResource(resource: BackendRequestHandlerCfg, opt?: FetcherOptions, defaultAppCfg?: DefaultAppCfg): Promise<ExpressApp>;
     createAppFromResources(resources: BackendRequestHandlerCfg[], opt?: FetcherOptions): Promise<ExpressApp>;
-    createApp(app: BackendApplication, opt?: FetcherOptions): ExpressApp;
+    createApp(app: BackendApplication, opt?: FetcherOptions): Promise<ExpressApp>;
     /**
      * Creates a "Default Express App" with provided resources.
      * Starts an http server on '127.0.0.1' and random available port.

package/dist/testing/express.test.service.js

@@ -1,9 +1,10 @@
 import { getFetcher } from '@naturalcycles/js-lib/http';
 import { pDelay } from '@naturalcycles/js-lib/promise/pDelay.js';
 import { createDefaultApp } from '../express/createDefaultApp.js';
+const TEST_SERVER_HOST = '127.0.0.1';
 const nativeFetchFn = async (url, init) => await globalThis.fetch(url, init);
 // Example:
-// const app = expressTestService.createApp([ debugResource ])
+// const app = await expressTestService.createAppFromResource(debugResource)
 // afterAll(async () => {
 //   await app.close()
 // })
@@ -19,10 +20,10 @@ class ExpressTestService {
             resources,
         }), opt);
     }
-    createApp(app, opt) {
-        const server = this.createTestServer(app);
+    async createApp(app, opt) {
+        const server = await this.createTestServer(app);
         const { port } = server.address();
-        const baseUrl = `http://127.0.0.1:${port}`;
+        const baseUrl = `http://${TEST_SERVER_HOST}:${port}`;
         const fetcher = getFetcher({
             baseUrl,
             responseType: 'json',
@@ -63,14 +64,23 @@ class ExpressTestService {
      * const { port } = server.address() as AddressInfo
      * const url = `http://127.0.0.1:${port}`
      */
-    createTestServer(app) {
+    async createTestServer(app) {
         // Important!
-        // Only with this syntax `app.listen(0)` it allocates a port synchronously
-        // Trying to specify a hostname will make server.address() return null.
-        // This is how `supertest` is doing it.
-        const server = app.listen(0);
+        // Listening on the same host that the Fetcher connects to.
+        // `app.listen(0)` (without a hostname) listens on all interfaces (`::`), and the OS (e.g macOS)
+        // may give it a port that some other process (e.g IntelliJ on 63342) already listens on
+        // at 127.0.0.1. Requests to 127.0.0.1 would then reach that other process instead.
+        // Listening on 127.0.0.1 makes the OS pick a port that is free on 127.0.0.1.
+        // The downside is that the port is only known after the async `listening` callback
+        // (server.address() returns null until then).
         // enableDestroy(server)
-        return server;
+        return await new Promise((resolve, reject) => {
+            const server = app.listen(0, TEST_SERVER_HOST, (err) => {
+                if (err)
+                    return reject(err);
+                resolve(server);
+            });
+        });
     }
 }
 export const expressTestService = new ExpressTestService();

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/backend-lib",
   "type": "module",
-  "version": "9.83.0",
+  "version": "9.84.1",
   "dependencies": {
     "@naturalcycles/db-lib": "^10",
     "@naturalcycles/js-lib": "^15",

package/src/server/eventLoop.util.ts

@@ -35,7 +35,9 @@ import type {
  * expect p50/mean near 0 on a healthy service, instead of the ~`resolution` baseline
  * reported by older interval-based sampling.
  *
- * @experimental
+ * Beware of CPU-throttled processes (e.g. idle CloudRun instances with request-based billing):
+ * the interval timer fires late there, and the time the process spent runnable-but-unscheduled
+ * is recorded as event loop delay. Use `intervalMs` to detect such intervals.
  */
 export class EventLoopMonitor {
   constructor(cfg: EventLoopMonitorCfg = {}) {
@@ -52,6 +54,7 @@ export class EventLoopMonitor {
     this.eld.enable()
 
     this.lastElu = performance.eventLoopUtilization()
+    this.lastReportedAt = performance.now()
 
     this.po = new PerformanceObserver(list => {
       for (const entry of list.getEntries()) {
@@ -71,12 +74,19 @@ export class EventLoopMonitor {
       const max = Math.round(eld.max / 1e6)
       const mean = Math.round(eld.mean / 1e6)
 
+      // Actual wall-clock length of this interval, as opposed to the nominal `measureInterval`.
+      // Timers fire late when the process is starved of CPU, so this can be much longer.
+      // Floored at 1 to never divide by zero (only reachable with fake timers).
+      const now = performance.now()
+      const intervalMs = Math.max(1, Math.round(now - this.lastReportedAt))
+      this.lastReportedAt = now
+
       const currentElu = performance.eventLoopUtilization()
       const deltaElu = performance.eventLoopUtilization(this.lastElu, currentElu)
       this.lastElu = currentElu
 
       const elu = Math.round(deltaElu.utilization * 100)
-      const gcCPU = Math.round((gcTotalTime / measureInterval) * 10_000)
+      const gcCPU = Math.round((gcTotalTime / intervalMs) * 10_000)
 
       this.lastStats = {
         p50,
@@ -88,6 +98,7 @@ export class EventLoopMonitor {
         gcCount,
         gcTotalTime,
         gcCPU,
+        intervalMs,
       }
 
       cfg.onStats?.(this.lastStats)
@@ -101,6 +112,7 @@ export class EventLoopMonitor {
   private interval: NodeJS.Timeout
   private eld: ELDHistogram
   private lastElu: EventLoopUtilization
+  private lastReportedAt: NumberOfMilliseconds
   /**
    * Undefined until the first interval has completed.
    */
@@ -146,15 +158,15 @@ export interface EventLoopStats {
    */
   elu: NumberOfPercent
   /**
-   * Number of times gc ran during the last `measureInterval`
+   * Number of times gc ran during the last interval
    */
   gcCount: NonNegativeInteger
   /**
-   * Total time spent on gc during the last `measureInterval`
+   * Total time spent on gc during the last interval
    */
   gcTotalTime: NumberOfMilliseconds
   /**
-   * CPU time spent on GC in the last `measureInterval`.
+   * CPU time spent on GC in the last interval, relative to `intervalMs`.
    * In "basis points", 100 basis points == 1%.
    * E.g 125 means 1.25% of CPU spent on GC.
    * It's reported in BPS to be able to do more accurate aggregate reporting
@@ -162,4 +174,12 @@ export interface EventLoopStats {
    * too "lossy")
    */
   gcCPU: Integer
+  /**
+   * Actual wall-clock duration of the interval covered by these stats.
+   * Normally equals `measureInterval`, but is longer when the timer fired late,
+   * e.g. on a CloudRun instance that was CPU-throttled while idle (request-based billing).
+   * An interval much longer than `measureInterval` means the delay stats are dominated
+   * by that throttling rather than by the application.
+   */
+  intervalMs: NumberOfMilliseconds
 }

package/src/testing/express.test.service.ts

@@ -7,6 +7,8 @@ import { createDefaultApp } from '../express/createDefaultApp.js'
 import type { BackendRequestHandlerCfg, DefaultAppCfg } from '../express/createDefaultApp.js'
 import type { BackendApplication } from '../server/server.model.js'
 
+const TEST_SERVER_HOST = '127.0.0.1'
+
 const nativeFetchFn: FetchFunction = async (url, init) => await globalThis.fetch(url, init)
 
 export interface ExpressApp extends Fetcher, AsyncDisposable {
@@ -14,7 +16,7 @@ export interface ExpressApp extends Fetcher, AsyncDisposable {
 }
 
 // Example:
-// const app = expressTestService.createApp([ debugResource ])
+// const app = await expressTestService.createAppFromResource(debugResource)
 // afterAll(async () => {
 //   await app.close()
 // })
@@ -46,10 +48,10 @@ class ExpressTestService {
     )
   }
 
-  createApp(app: BackendApplication, opt?: FetcherOptions): ExpressApp {
-    const server = this.createTestServer(app)
+  async createApp(app: BackendApplication, opt?: FetcherOptions): Promise<ExpressApp> {
+    const server = await this.createTestServer(app)
     const { port } = server.address() as AddressInfo
-    const baseUrl = `http://127.0.0.1:${port}`
+    const baseUrl = `http://${TEST_SERVER_HOST}:${port}`
 
     const fetcher = getFetcher({
       baseUrl,
@@ -95,14 +97,22 @@ class ExpressTestService {
    * const { port } = server.address() as AddressInfo
    * const url = `http://127.0.0.1:${port}`
    */
-  private createTestServer(app: BackendApplication): Server {
+  private async createTestServer(app: BackendApplication): Promise<Server> {
     // Important!
-    // Only with this syntax `app.listen(0)` it allocates a port synchronously
-    // Trying to specify a hostname will make server.address() return null.
-    // This is how `supertest` is doing it.
-    const server = app.listen(0)
+    // Listening on the same host that the Fetcher connects to.
+    // `app.listen(0)` (without a hostname) listens on all interfaces (`::`), and the OS (e.g macOS)
+    // may give it a port that some other process (e.g IntelliJ on 63342) already listens on
+    // at 127.0.0.1. Requests to 127.0.0.1 would then reach that other process instead.
+    // Listening on 127.0.0.1 makes the OS pick a port that is free on 127.0.0.1.
+    // The downside is that the port is only known after the async `listening` callback
+    // (server.address() returns null until then).
     // enableDestroy(server)
-    return server
+    return await new Promise<Server>((resolve, reject) => {
+      const server = app.listen(0, TEST_SERVER_HOST, (err?: Error) => {
+        if (err) return reject(err)
+        resolve(server)
+      })
+    })
   }
 }