@saidsef/tracing-node 4.1.1 → 4.3.0

package/README.md

@@ -21,6 +21,8 @@ Effortlessly supercharge your applications with world-class distributed tracing!
 | IORedis client | Cache instrumentation |
 | AWS SDK | Cloud service instrumentation |
 | Pino logger | Integration with trace/span IDs |
+| Node runtime metrics | Event loop, garbage collection, heap |
+| RED metrics | Request duration histograms over OTLP |
 | DNS/FS instrumentation | Optional monitoring |
 | Resource detection | Host, OS, process, container |
 | W3C Trace Context | Standard propagation |
@@ -43,6 +45,8 @@ setupTracing({serviceName: 'my-service', url: 'http://alloy:4317'});
 
 The W3C Trace Context propagation this library registers is what lets Tempo pair a caller's client span with the callee's server span, which is what a service graph is built from.
 
+Metrics go to the same endpoint by default and land in Mimir. They are recorded before the sampler runs, so they stay complete however far trace volume is turned down.
+
 ## Instalation
 
 ```
@@ -79,6 +83,26 @@ setupTracing({hostname: 'hostname', serviceName: 'service_name', url: 'endpoint'
 | url | string | tracing endpoint i.e. `<schema>://<host>:<port>` | Yes | `n/a` |
 | enableFsInstrumentation | boolean | enable FS instrumentation | No | `false` |
 | enableDnsInstrumentation | boolean | enable DNS instrumentation | No | `false`  |
+| enableMetrics | boolean | export metrics as well as traces | No | `true` |
+| metricsUrl | string | metrics endpoint, when it differs from `url` | No | `url` |
+| metricExportIntervalMillis | number | how often metrics are exported | No | `60000` |
+
+## Documentation
+
+Full documentation is in the [`docs/`](./docs) directory and built with [MkDocs Material](https://squidfunk.github.io/mkdocs-material/).
+
+Live docs: [tracing-node.readthedocs.io](https://tracing-node.readthedocs.io/)
+
+| Page | Contents |
+|------|----------|
+| [Architecture](./docs/architecture.md) | The pipeline `setupTracing` builds, and how the service graph is fed |
+| [Configuration](./docs/usage.md) | Options, environment variables, initialisation order and shutdown |
+| [Instrumentation](./docs/instrumentation.md) | Each instrumentation, and the attributes it emits |
+| [Deployment](./docs/deployment.md) | Running instrumented services in containers and Kubernetes |
+| [Testing](./docs/testing.md) | The unit tests and the end to end harness |
+| [Troubleshooting](./docs/troubleshooting.md) | Symptoms, causes and fixes |
+
+Build them locally with `npm run build-docs`, which renders the site into `site/`.
 
 ## Source
 

package/libs/index.mjs

@@ -17,17 +17,20 @@
 import {AwsInstrumentation} from '@opentelemetry/instrumentation-aws-sdk';
 import {BatchSpanProcessor} from '@opentelemetry/sdk-trace-base';
 import {ConnectInstrumentation} from '@opentelemetry/instrumentation-connect';
-import {diag, DiagConsoleLogger, DiagLogLevel} from '@opentelemetry/api';
+import {diag, DiagConsoleLogger, DiagLogLevel, metrics} from '@opentelemetry/api';
 import {HttpInstrumentation} from '@opentelemetry/instrumentation-http';
 import {DnsInstrumentation} from '@opentelemetry/instrumentation-dns';
 import {ElasticsearchInstrumentation} from 'opentelemetry-instrumentation-elasticsearch';
 import {ExpressInstrumentation} from '@opentelemetry/instrumentation-express';
 import {NodeTracerProvider} from '@opentelemetry/sdk-trace-node';
+import {OTLPMetricExporter} from '@opentelemetry/exporter-metrics-otlp-grpc';
 import {OTLPTraceExporter} from '@opentelemetry/exporter-trace-otlp-grpc';
 import {PinoInstrumentation} from '@opentelemetry/instrumentation-pino';
 import {UndiciInstrumentation} from '@opentelemetry/instrumentation-undici';
 import {IORedisInstrumentation} from '@opentelemetry/instrumentation-ioredis';
 import {registerInstrumentations} from '@opentelemetry/instrumentation';
+import {RuntimeNodeInstrumentation} from '@opentelemetry/instrumentation-runtime-node';
+import {MeterProvider, PeriodicExportingMetricReader} from '@opentelemetry/sdk-metrics';
 import {FsInstrumentation} from '@opentelemetry/instrumentation-fs';
 import {resourceFromAttributes, detectResources, envDetector, hostDetector, osDetector, processDetector, serviceInstanceIdDetector} from '@opentelemetry/resources';
 import {ATTR_SERVICE_NAME} from '@opentelemetry/semantic-conventions';
@@ -67,6 +70,7 @@ const setPeerService = (span, host) => {
 };
 
 let tracerProvider = null; // Declare provider in module scope for access in stopTracing
+let meterProvider = null;
 
 /**
 * Sets up tracing for the application using OpenTelemetry.
@@ -77,6 +81,10 @@ let tracerProvider = null; // Declare provider in module scope for access in sto
 * The IORedis instrumentation includes peer.service attributes for proper
 * service map visualization in distributed tracing tools like Tempo.
 *
+* A MeterProvider is registered alongside it, which is what makes the
+* instrumentations record the request duration histograms they already
+* compute, and adds the Node runtime metrics.
+*
 * @param {Object} options - Configuration options for tracing.
 * @param {string} [options.hostname=process.env.CONTAINER_NAME || process.env.HOSTNAME] - The hostname of the service.
 * @param {string} [options.serviceName=process.env.SERVICE_NAME] - The name of the service.
@@ -84,6 +92,9 @@ let tracerProvider = null; // Declare provider in module scope for access in sto
 * @param {number} [options.concurrencyLimit=10] - The concurrency limit for the exporter.
 * @param {boolean} [options.enableFsInstrumentation=false] - Enable file system instrumentation.
 * @param {boolean} [options.enableDnsInstrumentation=false] - Enable DNS instrumentation.
+* @param {boolean} [options.enableMetrics=true] - Export metrics as well as traces.
+* @param {string} [options.metricsUrl=options.url] - Endpoint for metrics, when it differs from the trace endpoint.
+* @param {number} [options.metricExportIntervalMillis=60000] - How often metrics are exported.
 *
 * @returns {Tracer} - The tracer for the service.
 */
@@ -101,6 +112,9 @@ export function setupTracing(options = {}) {
     concurrencyLimit = 10,
     enableFsInstrumentation = false,
     enableDnsInstrumentation = false,
+    enableMetrics = true,
+    metricsUrl = url,
+    metricExportIntervalMillis = 60000,
   } = options;
 
   // Validate required parameters
@@ -135,13 +149,30 @@ export function setupTracing(options = {}) {
     explicitAttributes[ATTR_CONTAINER_NAME] = hostname;
   }
 
+  // One resource for both signals. Grafana pairs a metric with a trace on
+  // service.name, so the two providers have to carry an identical resource.
+  const resource = detectResources({
+    detectors: [envDetector, hostDetector, osDetector, processDetector, serviceInstanceIdDetector],
+  }).merge(resourceFromAttributes(explicitAttributes));
+
   tracerProvider = new NodeTracerProvider({
     spanProcessors: [spanProcessor],
-    resource: detectResources({
-      detectors: [envDetector, hostDetector, osDetector, processDetector, serviceInstanceIdDetector],
-    }).merge(resourceFromAttributes(explicitAttributes)),
+    resource,
   });
 
+  if (enableMetrics) {
+    meterProvider = new MeterProvider({
+      resource,
+      readers: [
+        new PeriodicExportingMetricReader({
+          exporter: new OTLPMetricExporter({...exportOptions, url: metricsUrl}),
+          exportIntervalMillis: metricExportIntervalMillis,
+        }),
+      ],
+    });
+    metrics.setGlobalMeterProvider(meterProvider);
+  }
+
   // Register globally. With no overrides, register() installs the modern
   // AsyncLocalStorageContextManager and a CompositePropagator of
   // W3CTraceContext + W3CBaggage - identical propagation to the previous
@@ -282,6 +313,10 @@ export function setupTracing(options = {}) {
       },
     }),
     new ElasticsearchInstrumentation(),
+    // Event loop delay, GC pauses and heap occupancy are metric-only, and they
+    // are what explains a whole service slowing at once. Constructed only with
+    // metrics on, since the collectors start sampling on construction.
+    ...(enableMetrics ? [new RuntimeNodeInstrumentation()] : []),
     // Spread so the optional instrumentations are constructed only when enabled:
     // FsInstrumentation patches fs on construction.
     ...(enableFsInstrumentation ? [new FsInstrumentation()] : []),
@@ -289,9 +324,11 @@ export function setupTracing(options = {}) {
     ...(enableDnsInstrumentation ? [new DnsInstrumentation({ignoreHostnames: ['localhost', '127.0.0.1', '::1']})] : []),
   ];
 
-  // Register instrumentations
+  // Register instrumentations. Without meterProvider the instrumentations get
+  // the no-op meter, and the histograms they already record are discarded.
   registerInstrumentations({
     tracerProvider,
+    meterProvider,
     instrumentations,
   });
 
@@ -300,11 +337,11 @@ export function setupTracing(options = {}) {
 }
 
 /**
-* Gracefully stops the tracing by shutting down the tracer provider.
+* Gracefully stops the tracing by shutting down the tracer and meter providers.
 *
-* This function ensures that all pending spans are exported and resources are
-* cleaned up properly. It is recommended to call this function during the
-* application's shutdown process.
+* This function ensures that all pending spans and metrics are exported and
+* resources are cleaned up properly. It is recommended to call this function
+* during the application's shutdown process.
 *
 * @returns {Promise<void>} - A promise that resolves when shutdown is complete.
 */
@@ -320,6 +357,21 @@ export async function stopTracing() {
   } else {
     diag.warn('Tracer provider is not initialized.');
   }
+
+  // Separate from the trace shutdown, so a failing exporter on one signal
+  // still lets the other flush.
+  if (meterProvider) {
+    try {
+      await meterProvider.shutdown();
+      meterProvider = null;
+      // The API refuses a second setGlobalMeterProvider, so unregister here or
+      // a later setupTracing leaves the global pointing at a dead provider.
+      metrics.disable();
+      diag.info('Metrics have been successfully shut down.');
+    } catch (error) {
+      diag.error('Error during metrics shutdown:', error);
+    }
+  }
 }
 
 /**
@@ -329,4 +381,5 @@ export async function stopTracing() {
  */
 export function __resetTracingForTesting() {
   tracerProvider = null;
+  meterProvider = null;
 }

package/libs/index.test.mjs

@@ -1,6 +1,8 @@
 // index.test.mjs
 import { describe, it, beforeEach, afterEach } from 'node:test';
 import assert from 'node:assert';
+import { metrics } from '@opentelemetry/api';
+import { MeterProvider } from '@opentelemetry/sdk-metrics';
 import { setupTracing, stopTracing, __resetTracingForTesting } from './index.mjs';
 
 describe('setupTracing', () => {
@@ -72,4 +74,52 @@ describe('setupTracing', () => {
     });
     assert.ok(tracer, 'tracer should be defined');
   });
+
+  // The http and undici instrumentations record their duration histograms
+  // whether or not a meter provider exists. Without one the API hands them the
+  // no-op meter and every measurement is dropped.
+  it('should register a global meter provider by default', () => {
+    setupTracing({
+      serviceName: 'test-service',
+      url: 'http://localhost:4317',
+    });
+    assert.ok(metrics.getMeterProvider() instanceof MeterProvider, 'global meter provider should be the SDK one');
+  });
+
+  it('should leave the no-op meter provider in place when metrics are disabled', () => {
+    setupTracing({
+      serviceName: 'test-service',
+      url: 'http://localhost:4317',
+      enableMetrics: false,
+    });
+    assert.ok(!(metrics.getMeterProvider() instanceof MeterProvider), 'no meter provider should be registered');
+  });
+
+  it('should accept a separate metrics endpoint', () => {
+    const tracer = setupTracing({
+      serviceName: 'test-service',
+      url: 'http://localhost:4317',
+      metricsUrl: 'http://localhost:4318',
+    });
+    assert.ok(tracer, 'tracer should be defined');
+    assert.ok(metrics.getMeterProvider() instanceof MeterProvider, 'global meter provider should be the SDK one');
+  });
+
+  // Without the unregister in stopTracing the API refuses the second
+  // registration and the global keeps pointing at the shut-down provider.
+  it('should unregister the meter provider on shutdown', async () => {
+    setupTracing({
+      serviceName: 'test-service',
+      url: 'http://localhost:4317',
+    });
+    await stopTracing();
+    assert.ok(!(metrics.getMeterProvider() instanceof MeterProvider), 'meter provider should be unregistered');
+
+    __resetTracingForTesting();
+    setupTracing({
+      serviceName: 'test-service',
+      url: 'http://localhost:4317',
+    });
+    assert.ok(metrics.getMeterProvider() instanceof MeterProvider, 'a later setup should register again');
+  });
 });

package/package.json

@@ -1,12 +1,14 @@
 {
   "name": "@saidsef/tracing-node",
-  "version": "4.1.1",
+  "version": "4.3.0",
   "description": "tracing NodeJS - Wrapper for OpenTelemetry instrumentation packages",
   "main": "libs/index.mjs",
   "scripts": {
     "test": "node --trace-warnings --test --report-uncaught-exception libs/index.test.mjs",
     "lint": "eslint .",
-    "rebuild": "rm -rfv node_modules/ package-lock.json && npm install --prod --omit=dev"
+    "rebuild": "rm -rfv node_modules/ package-lock.json && npm install --prod --omit=dev",
+    "build-docs": "podman run --rm -v .:/docs docker.io/squidfunk/mkdocs-material:9 build",
+    "clean-docs": "rm -rf site"
   },
   "type": "module",
   "private": false,
@@ -30,32 +32,35 @@
   },
   "homepage": "https://github.com/saidsef/tracing-node#readme",
   "dependencies": {
-    "@opentelemetry/api": "^1.9.0",
-    "@opentelemetry/exporter-trace-otlp-grpc": "^0.221.0",
-    "@opentelemetry/instrumentation": "^0.221.0",
-    "@opentelemetry/instrumentation-aws-sdk": "^0.76.0",
-    "@opentelemetry/instrumentation-connect": "^0.64.0",
-    "@opentelemetry/instrumentation-dns": "^0.64.0",
-    "@opentelemetry/instrumentation-express": "^0.69.0",
-    "@opentelemetry/instrumentation-fs": "^0.40.0",
-    "@opentelemetry/instrumentation-http": "^0.221.0",
-    "@opentelemetry/instrumentation-ioredis": "^0.69.0",
-    "@opentelemetry/instrumentation-pino": "^0.67.0",
-    "@opentelemetry/instrumentation-undici": "^0.31.0",
-    "@opentelemetry/resources": "^2.10.0",
-    "@opentelemetry/sdk-trace-base": "^2.10.0",
-    "@opentelemetry/sdk-trace-node": "^2.10.0",
+    "@opentelemetry/api": "^1.9.1",
+    "@opentelemetry/exporter-metrics-otlp-grpc": "^0.222.0",
+    "@opentelemetry/exporter-trace-otlp-grpc": "^0.222.0",
+    "@opentelemetry/instrumentation": "^0.222.0",
+    "@opentelemetry/instrumentation-aws-sdk": "^0.77.0",
+    "@opentelemetry/instrumentation-connect": "^0.65.0",
+    "@opentelemetry/instrumentation-dns": "^0.65.0",
+    "@opentelemetry/instrumentation-express": "^0.70.0",
+    "@opentelemetry/instrumentation-fs": "^0.41.0",
+    "@opentelemetry/instrumentation-http": "^0.222.0",
+    "@opentelemetry/instrumentation-ioredis": "^0.70.0",
+    "@opentelemetry/instrumentation-pino": "^0.68.0",
+    "@opentelemetry/instrumentation-runtime-node": "^0.35.0",
+    "@opentelemetry/instrumentation-undici": "^0.32.0",
+    "@opentelemetry/resources": "^2.11.0",
+    "@opentelemetry/sdk-metrics": "^2.11.0",
+    "@opentelemetry/sdk-trace-base": "^2.11.0",
+    "@opentelemetry/sdk-trace-node": "^2.11.0",
     "@opentelemetry/semantic-conventions": "^1.43.0",
     "opentelemetry-instrumentation-elasticsearch": "^0.41.0"
   },
   "devDependencies": {
-    "eslint": "^10.8.0"
+    "eslint": "^10.10.0"
   },
   "overrides": {
     "protobufjs": "^7.5.3",
-    "@opentelemetry/core": "^2.10.0"
+    "@opentelemetry/core": "^2.11.0"
   },
   "allowScripts": {
-    "protobufjs@7.6.5": true
+    "protobufjs@7.6.6": true
   }
 }