@superblocksteam/sdk 2.0.130-next.3 → 2.0.130-next.5

package/src/telemetry/memory-metrics.ts

@@ -1,7 +1,12 @@
+import { readFileSync } from "node:fs";
 import {
+  constants,
+  monitorEventLoopDelay,
+  performance,
   PerformanceObserver,
   type PerformanceObserverEntryList,
 } from "node:perf_hooks";
+import { getHeapStatistics } from "node:v8";
 
 import type { BatchObservableResult } from "@opentelemetry/api";
 
@@ -9,77 +14,341 @@ import { getMeter } from "./index.js";
 
 let cleanupFn: (() => void) | undefined;
 
+interface CgroupMemory {
+  currentBytes: number;
+  limitBytes: number;
+  usageRatio: number;
+}
+
+interface CgroupNumber {
+  exists: boolean;
+  value?: number;
+}
+
+interface GcPerformanceEntry {
+  duration: number;
+  kind?: number;
+  detail?: {
+    kind?: number;
+  };
+}
+
+const NS_PER_MS = 1_000_000;
+const US_PER_SECOND = 1_000_000;
+
+function finiteNonNegative(value: number): number {
+  return Number.isFinite(value) && value > 0 ? value : 0;
+}
+
+function nsToMs(value: number): number {
+  return finiteNonNegative(value) / NS_PER_MS;
+}
+
+function ratio(numerator: number, denominator: number): number {
+  if (!Number.isFinite(numerator) || !Number.isFinite(denominator)) {
+    return 0;
+  }
+  if (denominator <= 0) {
+    return 0;
+  }
+  return Math.max(0, numerator / denominator);
+}
+
+function readCgroupNumber(filePath: string): CgroupNumber {
+  try {
+    const value = readFileSync(filePath, "utf-8").trim();
+    if (!value || value === "max") {
+      return { exists: true };
+    }
+    const parsed = Number(value);
+    if (
+      Number.isFinite(parsed) &&
+      parsed >= 0 &&
+      parsed < Number.MAX_SAFE_INTEGER
+    ) {
+      return { exists: true, value: parsed };
+    }
+    return { exists: true };
+  } catch {
+    return { exists: false };
+  }
+}
+
+export function readCgroupMemory(
+  root = "/sys/fs/cgroup",
+): CgroupMemory | undefined {
+  const v2Current = readCgroupNumber(`${root}/memory.current`);
+  const v2Limit = readCgroupNumber(`${root}/memory.max`);
+  const usesCgroupV2 = v2Current.exists || v2Limit.exists;
+  const currentBytes = usesCgroupV2
+    ? v2Current.value
+    : readCgroupNumber(`${root}/memory/memory.usage_in_bytes`).value;
+  const limitBytes = usesCgroupV2
+    ? v2Limit.value
+    : readCgroupNumber(`${root}/memory/memory.limit_in_bytes`).value;
+
+  if (
+    currentBytes === undefined ||
+    limitBytes === undefined ||
+    limitBytes <= 0
+  ) {
+    return undefined;
+  }
+
+  return {
+    currentBytes,
+    limitBytes,
+    usageRatio: ratio(currentBytes, limitBytes),
+  };
+}
+
+function gcKind(kind: number | undefined): string {
+  switch (kind) {
+    case constants.NODE_PERFORMANCE_GC_MINOR:
+      return "minor";
+    case constants.NODE_PERFORMANCE_GC_MAJOR:
+      return "major";
+    case constants.NODE_PERFORMANCE_GC_INCREMENTAL:
+      return "incremental";
+    case constants.NODE_PERFORMANCE_GC_WEAKCB:
+      return "weakcb";
+    default:
+      return "unknown";
+  }
+}
+
 export function startMemoryMetrics(): void {
   if (cleanupFn) {
     return;
   }
 
   const meter = getMeter();
+  const eventLoopDelay = monitorEventLoopDelay({ resolution: 20 });
+  let previousEventLoopUtilization = performance.eventLoopUtilization();
+  let removeBatchCallback: (() => void) | undefined;
+  let disconnectGcObserver: (() => void) | undefined;
 
-  const heapUsedGauge = meter.createObservableGauge(
-    "nodejs.memory.heap_used_bytes",
-    { description: "V8 heap memory used (bytes)", unit: "By" },
-  );
-  const heapTotalGauge = meter.createObservableGauge(
-    "nodejs.memory.heap_total_bytes",
-    { description: "V8 heap memory allocated (bytes)", unit: "By" },
-  );
-  const rssGauge = meter.createObservableGauge("nodejs.memory.rss_bytes", {
-    description: "Resident Set Size - total memory allocated for the process",
-    unit: "By",
-  });
-  const externalGauge = meter.createObservableGauge(
-    "nodejs.memory.external_bytes",
-    {
-      description: "Memory used by C++ objects bound to JavaScript objects",
-      unit: "By",
-    },
-  );
-  const arrayBuffersGauge = meter.createObservableGauge(
-    "nodejs.memory.array_buffers_bytes",
-    {
-      description:
-        "Memory allocated for ArrayBuffers and SharedArrayBuffers (also included in external)",
+  try {
+    const heapUsedGauge = meter.createObservableGauge(
+      "nodejs.memory.heap_used_bytes",
+      { description: "V8 heap memory used (bytes)", unit: "By" },
+    );
+    const heapTotalGauge = meter.createObservableGauge(
+      "nodejs.memory.heap_total_bytes",
+      { description: "V8 heap memory allocated (bytes)", unit: "By" },
+    );
+    const rssGauge = meter.createObservableGauge("nodejs.memory.rss_bytes", {
+      description: "Resident Set Size - total memory allocated for the process",
       unit: "By",
-    },
-  );
-
-  const gcCountCounter = meter.createCounter("nodejs.gc.count", {
-    description: "Number of garbage collection events",
-    unit: "{event}",
-  });
-
-  const gauges = [
-    heapUsedGauge,
-    heapTotalGauge,
-    rssGauge,
-    externalGauge,
-    arrayBuffersGauge,
-  ];
-
-  // Single memoryUsage() call per export interval for all 5 gauges (idiomatic OTEL batch pattern).
-  const batchCb = (result: BatchObservableResult) => {
-    const mem = process.memoryUsage();
-    result.observe(heapUsedGauge, mem.heapUsed);
-    result.observe(heapTotalGauge, mem.heapTotal);
-    result.observe(rssGauge, mem.rss);
-    result.observe(externalGauge, mem.external);
-    result.observe(arrayBuffersGauge, mem.arrayBuffers);
-  };
+    });
+    const externalGauge = meter.createObservableGauge(
+      "nodejs.memory.external_bytes",
+      {
+        description: "Memory used by C++ objects bound to JavaScript objects",
+        unit: "By",
+      },
+    );
+    const arrayBuffersGauge = meter.createObservableGauge(
+      "nodejs.memory.array_buffers_bytes",
+      {
+        description:
+          "Memory allocated for ArrayBuffers and SharedArrayBuffers (also included in external)",
+        unit: "By",
+      },
+    );
+    const heapLimitGauge = meter.createObservableGauge(
+      "nodejs.memory.heap_limit_bytes",
+      { description: "Configured V8 heap size limit (bytes)", unit: "By" },
+    );
+    const heapUsedRatioGauge = meter.createObservableGauge(
+      "nodejs.memory.heap_used_ratio",
+      { description: "V8 heap used divided by V8 heap size limit", unit: "1" },
+    );
+    const cgroupCurrentGauge = meter.createObservableGauge(
+      "nodejs.memory.cgroup_current_bytes",
+      { description: "Current cgroup memory usage when available", unit: "By" },
+    );
+    const cgroupLimitGauge = meter.createObservableGauge(
+      "nodejs.memory.cgroup_limit_bytes",
+      { description: "Cgroup memory limit when available", unit: "By" },
+    );
+    const cgroupUsageRatioGauge = meter.createObservableGauge(
+      "nodejs.memory.cgroup_usage_ratio",
+      {
+        description: "Cgroup memory usage divided by cgroup memory limit",
+        unit: "1",
+      },
+    );
+    const eventLoopDelayMeanGauge = meter.createObservableGauge(
+      "nodejs.event_loop.delay.mean_ms",
+      {
+        description: "Mean event-loop delay for the last collection window",
+        unit: "ms",
+      },
+    );
+    const eventLoopDelayMaxGauge = meter.createObservableGauge(
+      "nodejs.event_loop.delay.max_ms",
+      {
+        description: "Max event-loop delay for the last collection window",
+        unit: "ms",
+      },
+    );
+    const eventLoopDelayP50Gauge = meter.createObservableGauge(
+      "nodejs.event_loop.delay.p50_ms",
+      {
+        description: "p50 event-loop delay for the last collection window",
+        unit: "ms",
+      },
+    );
+    const eventLoopDelayP90Gauge = meter.createObservableGauge(
+      "nodejs.event_loop.delay.p90_ms",
+      {
+        description: "p90 event-loop delay for the last collection window",
+        unit: "ms",
+      },
+    );
+    const eventLoopDelayP99Gauge = meter.createObservableGauge(
+      "nodejs.event_loop.delay.p99_ms",
+      {
+        description: "p99 event-loop delay for the last collection window",
+        unit: "ms",
+      },
+    );
+    const eventLoopUtilizationGauge = meter.createObservableGauge(
+      "nodejs.event_loop.utilization",
+      {
+        description:
+          "Event-loop utilization delta for the last collection window",
+        unit: "1",
+      },
+    );
+    const processCpuUserCounter = meter.createObservableCounter(
+      "nodejs.process.cpu.user_seconds",
+      { description: "Process user CPU time since start", unit: "s" },
+    );
+    const processCpuSystemCounter = meter.createObservableCounter(
+      "nodejs.process.cpu.system_seconds",
+      { description: "Process system CPU time since start", unit: "s" },
+    );
 
-  meter.addBatchObservableCallback(batchCb, gauges);
+    const gcCountCounter = meter.createCounter("nodejs.gc.count", {
+      description: "Number of garbage collection events",
+      unit: "{event}",
+    });
+    const gcDurationHistogram = meter.createHistogram("nodejs.gc.duration_ms", {
+      description: "Garbage collection duration by collection kind",
+      unit: "ms",
+      advice: {
+        explicitBucketBoundaries: [0.5, 1, 2, 5, 10, 25, 50, 100, 250, 500],
+      },
+    });
 
-  const gcObserver = new PerformanceObserver(
-    (list: PerformanceObserverEntryList) => {
-      gcCountCounter.add(list.getEntries().length);
-    },
-  );
-  gcObserver.observe({ entryTypes: ["gc"], buffered: false });
+    const observableInstruments = [
+      heapUsedGauge,
+      heapTotalGauge,
+      rssGauge,
+      externalGauge,
+      arrayBuffersGauge,
+      heapLimitGauge,
+      heapUsedRatioGauge,
+      cgroupCurrentGauge,
+      cgroupLimitGauge,
+      cgroupUsageRatioGauge,
+      eventLoopDelayMeanGauge,
+      eventLoopDelayMaxGauge,
+      eventLoopDelayP50Gauge,
+      eventLoopDelayP90Gauge,
+      eventLoopDelayP99Gauge,
+      eventLoopUtilizationGauge,
+      processCpuUserCounter,
+      processCpuSystemCounter,
+    ];
 
-  cleanupFn = () => {
-    gcObserver.disconnect();
-    meter.removeBatchObservableCallback(batchCb, gauges);
-  };
+    // Single callback per export interval keeps runtime stats temporally aligned.
+    const batchCb = (result: BatchObservableResult) => {
+      try {
+        const mem = process.memoryUsage();
+        const heap = getHeapStatistics();
+        const cgroup = readCgroupMemory();
+        const elu = performance.eventLoopUtilization(
+          previousEventLoopUtilization,
+        );
+        previousEventLoopUtilization = performance.eventLoopUtilization();
+        const cpu = process.cpuUsage();
+
+        result.observe(heapUsedGauge, mem.heapUsed);
+        result.observe(heapTotalGauge, mem.heapTotal);
+        result.observe(rssGauge, mem.rss);
+        result.observe(externalGauge, mem.external);
+        result.observe(arrayBuffersGauge, mem.arrayBuffers);
+        result.observe(heapLimitGauge, heap.heap_size_limit);
+        result.observe(
+          heapUsedRatioGauge,
+          ratio(mem.heapUsed, heap.heap_size_limit),
+        );
+        // Omit cgroup gauges when no finite cgroup limit is available; absence means host/unlimited memory, not telemetry setup failure.
+        if (cgroup) {
+          result.observe(cgroupCurrentGauge, cgroup.currentBytes);
+          result.observe(cgroupLimitGauge, cgroup.limitBytes);
+          result.observe(cgroupUsageRatioGauge, cgroup.usageRatio);
+        }
+        result.observe(eventLoopDelayMeanGauge, nsToMs(eventLoopDelay.mean));
+        result.observe(eventLoopDelayMaxGauge, nsToMs(eventLoopDelay.max));
+        result.observe(
+          eventLoopDelayP50Gauge,
+          nsToMs(eventLoopDelay.percentile(50)),
+        );
+        result.observe(
+          eventLoopDelayP90Gauge,
+          nsToMs(eventLoopDelay.percentile(90)),
+        );
+        result.observe(
+          eventLoopDelayP99Gauge,
+          nsToMs(eventLoopDelay.percentile(99)),
+        );
+        result.observe(
+          eventLoopUtilizationGauge,
+          finiteNonNegative(elu.utilization),
+        );
+        result.observe(processCpuUserCounter, cpu.user / US_PER_SECOND);
+        result.observe(processCpuSystemCounter, cpu.system / US_PER_SECOND);
+      } finally {
+        eventLoopDelay.reset();
+      }
+    };
+
+    meter.addBatchObservableCallback(batchCb, observableInstruments);
+    removeBatchCallback = () => {
+      meter.removeBatchObservableCallback(batchCb, observableInstruments);
+    };
+
+    const gcObserver = new PerformanceObserver(
+      (list: PerformanceObserverEntryList) => {
+        for (const entry of list.getEntries() as GcPerformanceEntry[]) {
+          const kind = gcKind(entry.detail?.kind ?? entry.kind);
+          const attrs = { kind };
+          gcCountCounter.add(1, attrs);
+          gcDurationHistogram.record(finiteNonNegative(entry.duration), attrs);
+        }
+      },
+    );
+    disconnectGcObserver = () => {
+      gcObserver.disconnect();
+    };
+    gcObserver.observe({ entryTypes: ["gc"], buffered: false });
+
+    eventLoopDelay.enable();
+    cleanupFn = () => {
+      eventLoopDelay.disable();
+      disconnectGcObserver?.();
+      removeBatchCallback?.();
+    };
+  } catch (error) {
+    eventLoopDelay.disable();
+    disconnectGcObserver?.();
+    removeBatchCallback?.();
+    throw error;
+  }
 }
 
 export function stopMemoryMetrics(): void {