@saidsef/tracing-node 4.2.0 → 4.4.0

package/README.md

@@ -21,6 +21,9 @@ Effortlessly supercharge your applications with world-class distributed tracing!
 | IORedis client | Cache instrumentation |
 | AWS SDK | Cloud service instrumentation |
 | Pino logger | Integration with trace/span IDs |
+| Node runtime metrics | Event loop, garbage collection, heap |
+| Log export | Pino records over OTLP, correlated by trace |
+| RED metrics | Request duration histograms over OTLP |
 | DNS/FS instrumentation | Optional monitoring |
 | Resource detection | Host, OS, process, container |
 | W3C Trace Context | Standard propagation |
@@ -43,6 +46,10 @@ setupTracing({serviceName: 'my-service', url: 'http://alloy:4317'});
 
 The W3C Trace Context propagation this library registers is what lets Tempo pair a caller's client span with the callee's server span, which is what a service graph is built from.
 
+Metrics go to the same endpoint by default and land in Mimir. They are recorded before the sampler runs, so they stay complete however far trace volume is turned down.
+
+Pino log records go to the same endpoint and land in Loki, each carrying the trace and span id of the request that wrote it. No log agent or file scraping sits in between.
+
 ## Instalation
 
 ```
@@ -79,6 +86,11 @@ setupTracing({hostname: 'hostname', serviceName: 'service_name', url: 'endpoint'
 | url | string | tracing endpoint i.e. `<schema>://<host>:<port>` | Yes | `n/a` |
 | enableFsInstrumentation | boolean | enable FS instrumentation | No | `false` |
 | enableDnsInstrumentation | boolean | enable DNS instrumentation | No | `false`  |
+| enableMetrics | boolean | export metrics as well as traces | No | `true` |
+| metricsUrl | string | metrics endpoint, when it differs from `url` | No | `url` |
+| metricExportIntervalMillis | number | how often metrics are exported | No | `60000` |
+| enableLogs | boolean | send Pino log records over OTLP | No | `true` |
+| logsUrl | string | logs endpoint, when it differs from `url` | No | `url` |
 
 ## Documentation
 

package/libs/index.mjs

@@ -17,17 +17,23 @@
 import {AwsInstrumentation} from '@opentelemetry/instrumentation-aws-sdk';
 import {BatchSpanProcessor} from '@opentelemetry/sdk-trace-base';
 import {ConnectInstrumentation} from '@opentelemetry/instrumentation-connect';
-import {diag, DiagConsoleLogger, DiagLogLevel} from '@opentelemetry/api';
+import {diag, DiagConsoleLogger, DiagLogLevel, metrics} from '@opentelemetry/api';
 import {HttpInstrumentation} from '@opentelemetry/instrumentation-http';
 import {DnsInstrumentation} from '@opentelemetry/instrumentation-dns';
 import {ElasticsearchInstrumentation} from 'opentelemetry-instrumentation-elasticsearch';
 import {ExpressInstrumentation} from '@opentelemetry/instrumentation-express';
+import {logs} from '@opentelemetry/api-logs';
 import {NodeTracerProvider} from '@opentelemetry/sdk-trace-node';
+import {OTLPLogExporter} from '@opentelemetry/exporter-logs-otlp-grpc';
+import {OTLPMetricExporter} from '@opentelemetry/exporter-metrics-otlp-grpc';
 import {OTLPTraceExporter} from '@opentelemetry/exporter-trace-otlp-grpc';
 import {PinoInstrumentation} from '@opentelemetry/instrumentation-pino';
 import {UndiciInstrumentation} from '@opentelemetry/instrumentation-undici';
 import {IORedisInstrumentation} from '@opentelemetry/instrumentation-ioredis';
 import {registerInstrumentations} from '@opentelemetry/instrumentation';
+import {RuntimeNodeInstrumentation} from '@opentelemetry/instrumentation-runtime-node';
+import {MeterProvider, PeriodicExportingMetricReader} from '@opentelemetry/sdk-metrics';
+import {BatchLogRecordProcessor, LoggerProvider} from '@opentelemetry/sdk-logs';
 import {FsInstrumentation} from '@opentelemetry/instrumentation-fs';
 import {resourceFromAttributes, detectResources, envDetector, hostDetector, osDetector, processDetector, serviceInstanceIdDetector} from '@opentelemetry/resources';
 import {ATTR_SERVICE_NAME} from '@opentelemetry/semantic-conventions';
@@ -67,6 +73,8 @@ const setPeerService = (span, host) => {
 };
 
 let tracerProvider = null; // Declare provider in module scope for access in stopTracing
+let meterProvider = null;
+let loggerProvider = null;
 
 /**
 * Sets up tracing for the application using OpenTelemetry.
@@ -77,6 +85,10 @@ let tracerProvider = null; // Declare provider in module scope for access in sto
 * The IORedis instrumentation includes peer.service attributes for proper
 * service map visualization in distributed tracing tools like Tempo.
 *
+* A MeterProvider is registered alongside it, which is what makes the
+* instrumentations record the request duration histograms they already
+* compute, and adds the Node runtime metrics.
+*
 * @param {Object} options - Configuration options for tracing.
 * @param {string} [options.hostname=process.env.CONTAINER_NAME || process.env.HOSTNAME] - The hostname of the service.
 * @param {string} [options.serviceName=process.env.SERVICE_NAME] - The name of the service.
@@ -84,6 +96,11 @@ let tracerProvider = null; // Declare provider in module scope for access in sto
 * @param {number} [options.concurrencyLimit=10] - The concurrency limit for the exporter.
 * @param {boolean} [options.enableFsInstrumentation=false] - Enable file system instrumentation.
 * @param {boolean} [options.enableDnsInstrumentation=false] - Enable DNS instrumentation.
+* @param {boolean} [options.enableMetrics=true] - Export metrics as well as traces.
+* @param {string} [options.metricsUrl=options.url] - Endpoint for metrics, when it differs from the trace endpoint.
+* @param {number} [options.metricExportIntervalMillis=60000] - How often metrics are exported.
+* @param {boolean} [options.enableLogs=true] - Send Pino log records over OTLP.
+* @param {string} [options.logsUrl=options.url] - Endpoint for logs, when it differs from the trace endpoint.
 *
 * @returns {Tracer} - The tracer for the service.
 */
@@ -101,6 +118,11 @@ export function setupTracing(options = {}) {
     concurrencyLimit = 10,
     enableFsInstrumentation = false,
     enableDnsInstrumentation = false,
+    enableMetrics = true,
+    metricsUrl = url,
+    metricExportIntervalMillis = 60000,
+    enableLogs = true,
+    logsUrl = url,
   } = options;
 
   // Validate required parameters
@@ -135,13 +157,46 @@ export function setupTracing(options = {}) {
     explicitAttributes[ATTR_CONTAINER_NAME] = hostname;
   }
 
+  // One resource for every signal. Grafana pairs a metric and a log line with
+  // a trace on service.name, so the providers carry an identical resource.
+  const resource = detectResources({
+    detectors: [envDetector, hostDetector, osDetector, processDetector, serviceInstanceIdDetector],
+  }).merge(resourceFromAttributes(explicitAttributes));
+
   tracerProvider = new NodeTracerProvider({
     spanProcessors: [spanProcessor],
-    resource: detectResources({
-      detectors: [envDetector, hostDetector, osDetector, processDetector, serviceInstanceIdDetector],
-    }).merge(resourceFromAttributes(explicitAttributes)),
+    resource,
   });
 
+  if (enableMetrics) {
+    meterProvider = new MeterProvider({
+      resource,
+      readers: [
+        new PeriodicExportingMetricReader({
+          exporter: new OTLPMetricExporter({...exportOptions, url: metricsUrl}),
+          exportIntervalMillis: metricExportIntervalMillis,
+        }),
+      ],
+    });
+    metrics.setGlobalMeterProvider(meterProvider);
+  }
+
+  if (enableLogs) {
+    loggerProvider = new LoggerProvider({
+      resource,
+      processors: [
+        new BatchLogRecordProcessor({
+          exporter: new OTLPLogExporter({...exportOptions, url: logsUrl}),
+          maxQueueSize: 4096,
+          maxExportBatchSize: 1024,
+          scheduledDelayMillis: 2000,
+          exportTimeoutMillis: 10000,
+        }),
+      ],
+    });
+    logs.setGlobalLoggerProvider(loggerProvider);
+  }
+
   // Register globally. With no overrides, register() installs the modern
   // AsyncLocalStorageContextManager and a CompositePropagator of
   // W3CTraceContext + W3CBaggage - identical propagation to the previous
@@ -214,6 +269,10 @@ export function setupTracing(options = {}) {
       },
     }),
     new PinoInstrumentation({
+      // Log sending is on by default, and every record is parsed and rebuilt as
+      // a LogRecord before it reaches a logger. With no logger provider that
+      // work is done for a no-op, so turn it off rather than pay for nothing.
+      disableLogSending: !enableLogs,
       logHook: (span, record) => {
         // trace_id/span_id/trace_flags are injected by the instrumentation by
         // default; only add service name for better log correlation.
@@ -282,6 +341,10 @@ export function setupTracing(options = {}) {
       },
     }),
     new ElasticsearchInstrumentation(),
+    // Event loop delay, GC pauses and heap occupancy are metric-only, and they
+    // are what explains a whole service slowing at once. Constructed only with
+    // metrics on, since the collectors start sampling on construction.
+    ...(enableMetrics ? [new RuntimeNodeInstrumentation()] : []),
     // Spread so the optional instrumentations are constructed only when enabled:
     // FsInstrumentation patches fs on construction.
     ...(enableFsInstrumentation ? [new FsInstrumentation()] : []),
@@ -289,9 +352,12 @@ export function setupTracing(options = {}) {
     ...(enableDnsInstrumentation ? [new DnsInstrumentation({ignoreHostnames: ['localhost', '127.0.0.1', '::1']})] : []),
   ];
 
-  // Register instrumentations
+  // Register instrumentations. Without meterProvider the instrumentations get
+  // the no-op meter, and the histograms they already record are discarded.
   registerInstrumentations({
     tracerProvider,
+    meterProvider,
+    loggerProvider,
     instrumentations,
   });
 
@@ -300,11 +366,11 @@ export function setupTracing(options = {}) {
 }
 
 /**
-* Gracefully stops the tracing by shutting down the tracer provider.
+* Gracefully stops the tracing by shutting down every provider it registered.
 *
-* This function ensures that all pending spans are exported and resources are
-* cleaned up properly. It is recommended to call this function during the
-* application's shutdown process.
+* This function ensures that all pending spans, metrics and log records are
+* exported and resources are cleaned up properly. It is recommended to call
+* this function during the application's shutdown process.
 *
 * @returns {Promise<void>} - A promise that resolves when shutdown is complete.
 */
@@ -320,6 +386,35 @@ export async function stopTracing() {
   } else {
     diag.warn('Tracer provider is not initialized.');
   }
+
+  // Separate from the trace shutdown, so a failing exporter on one signal
+  // still lets the other flush.
+  if (meterProvider) {
+    try {
+      await meterProvider.shutdown();
+      meterProvider = null;
+      // The API refuses a second setGlobalMeterProvider, so unregister here or
+      // a later setupTracing leaves the global pointing at a dead provider.
+      metrics.disable();
+      diag.info('Metrics have been successfully shut down.');
+    } catch (error) {
+      diag.error('Error during metrics shutdown:', error);
+    }
+  }
+
+  if (loggerProvider) {
+    try {
+      await loggerProvider.shutdown();
+      diag.info('Logs have been successfully shut down.');
+    } catch (error) {
+      diag.error('Error during logs shutdown:', error);
+    } finally {
+      // A second setGlobalLoggerProvider is ignored, so unregister whatever the
+      // flush did, or a later setupTracing keeps writing to a dead provider.
+      loggerProvider = null;
+      logs.disable();
+    }
+  }
 }
 
 /**
@@ -329,4 +424,6 @@ export async function stopTracing() {
  */
 export function __resetTracingForTesting() {
   tracerProvider = null;
+  meterProvider = null;
+  loggerProvider = null;
 }

package/libs/index.test.mjs

@@ -1,6 +1,10 @@
 // index.test.mjs
 import { describe, it, beforeEach, afterEach } from 'node:test';
 import assert from 'node:assert';
+import { metrics } from '@opentelemetry/api';
+import { logs } from '@opentelemetry/api-logs';
+import { MeterProvider } from '@opentelemetry/sdk-metrics';
+import { LoggerProvider } from '@opentelemetry/sdk-logs';
 import { setupTracing, stopTracing, __resetTracingForTesting } from './index.mjs';
 
 describe('setupTracing', () => {
@@ -72,4 +76,99 @@ describe('setupTracing', () => {
     });
     assert.ok(tracer, 'tracer should be defined');
   });
+
+  // The http and undici instrumentations record their duration histograms
+  // whether or not a meter provider exists. Without one the API hands them the
+  // no-op meter and every measurement is dropped.
+  it('should register a global meter provider by default', () => {
+    setupTracing({
+      serviceName: 'test-service',
+      url: 'http://localhost:4317',
+    });
+    assert.ok(metrics.getMeterProvider() instanceof MeterProvider, 'global meter provider should be the SDK one');
+  });
+
+  it('should leave the no-op meter provider in place when metrics are disabled', () => {
+    setupTracing({
+      serviceName: 'test-service',
+      url: 'http://localhost:4317',
+      enableMetrics: false,
+    });
+    assert.ok(!(metrics.getMeterProvider() instanceof MeterProvider), 'no meter provider should be registered');
+  });
+
+  it('should accept a separate metrics endpoint', () => {
+    const tracer = setupTracing({
+      serviceName: 'test-service',
+      url: 'http://localhost:4317',
+      metricsUrl: 'http://localhost:4318',
+    });
+    assert.ok(tracer, 'tracer should be defined');
+    assert.ok(metrics.getMeterProvider() instanceof MeterProvider, 'global meter provider should be the SDK one');
+  });
+
+  // The Pino instrumentation sends every log record to the Logs API whether or
+  // not a provider is registered. Without one the record is built and dropped.
+  it('should register a global logger provider by default', () => {
+    setupTracing({
+      serviceName: 'test-service',
+      url: 'http://localhost:4317',
+    });
+    assert.ok(logs.getLoggerProvider() instanceof LoggerProvider, 'global logger provider should be the SDK one');
+  });
+
+  it('should leave the no-op logger provider in place when logs are disabled', () => {
+    setupTracing({
+      serviceName: 'test-service',
+      url: 'http://localhost:4317',
+      enableLogs: false,
+    });
+    assert.ok(!(logs.getLoggerProvider() instanceof LoggerProvider), 'no logger provider should be registered');
+  });
+
+  it('should accept a separate logs endpoint', () => {
+    const tracer = setupTracing({
+      serviceName: 'test-service',
+      url: 'http://localhost:4317',
+      logsUrl: 'http://localhost:4318',
+    });
+    assert.ok(tracer, 'tracer should be defined');
+    assert.ok(logs.getLoggerProvider() instanceof LoggerProvider, 'global logger provider should be the SDK one');
+  });
+
+  // Without the unregister in stopTracing the API keeps the first provider and
+  // silently ignores the second registration.
+  it('should unregister the logger provider on shutdown', async () => {
+    setupTracing({
+      serviceName: 'test-service',
+      url: 'http://localhost:4317',
+    });
+    await stopTracing();
+    assert.ok(!(logs.getLoggerProvider() instanceof LoggerProvider), 'logger provider should be unregistered');
+
+    __resetTracingForTesting();
+    setupTracing({
+      serviceName: 'test-service',
+      url: 'http://localhost:4317',
+    });
+    assert.ok(logs.getLoggerProvider() instanceof LoggerProvider, 'a later setup should register again');
+  });
+
+  // Without the unregister in stopTracing the API refuses the second
+  // registration and the global keeps pointing at the shut-down provider.
+  it('should unregister the meter provider on shutdown', async () => {
+    setupTracing({
+      serviceName: 'test-service',
+      url: 'http://localhost:4317',
+    });
+    await stopTracing();
+    assert.ok(!(metrics.getMeterProvider() instanceof MeterProvider), 'meter provider should be unregistered');
+
+    __resetTracingForTesting();
+    setupTracing({
+      serviceName: 'test-service',
+      url: 'http://localhost:4317',
+    });
+    assert.ok(metrics.getMeterProvider() instanceof MeterProvider, 'a later setup should register again');
+  });
 });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@saidsef/tracing-node",
-  "version": "4.2.0",
+  "version": "4.4.0",
   "description": "tracing NodeJS - Wrapper for OpenTelemetry instrumentation packages",
   "main": "libs/index.mjs",
   "scripts": {
@@ -33,6 +33,9 @@
   "homepage": "https://github.com/saidsef/tracing-node#readme",
   "dependencies": {
     "@opentelemetry/api": "^1.9.1",
+    "@opentelemetry/api-logs": "^0.222.0",
+    "@opentelemetry/exporter-logs-otlp-grpc": "^0.222.0",
+    "@opentelemetry/exporter-metrics-otlp-grpc": "^0.222.0",
     "@opentelemetry/exporter-trace-otlp-grpc": "^0.222.0",
     "@opentelemetry/instrumentation": "^0.222.0",
     "@opentelemetry/instrumentation-aws-sdk": "^0.77.0",
@@ -43,8 +46,11 @@
     "@opentelemetry/instrumentation-http": "^0.222.0",
     "@opentelemetry/instrumentation-ioredis": "^0.70.0",
     "@opentelemetry/instrumentation-pino": "^0.68.0",
+    "@opentelemetry/instrumentation-runtime-node": "^0.35.0",
     "@opentelemetry/instrumentation-undici": "^0.32.0",
     "@opentelemetry/resources": "^2.11.0",
+    "@opentelemetry/sdk-logs": "^0.222.0",
+    "@opentelemetry/sdk-metrics": "^2.11.0",
     "@opentelemetry/sdk-trace-base": "^2.11.0",
     "@opentelemetry/sdk-trace-node": "^2.11.0",
     "@opentelemetry/semantic-conventions": "^1.43.0",