@open-mercato/telemetry 0.10.1-develop.7827.1.5c631908e4 → 0.10.1-develop.7836.1.0ef0f453c6

package/src/__tests__/env-load-order.test.ts

@@ -3,7 +3,7 @@ import {
   getTelemetryRuntime,
   resetTelemetryRuntime,
 } from '@open-mercato/shared/lib/telemetry/runtime'
-import { initTelemetry, resetTelemetryInit } from '../init'
+import { initTelemetry, resetTelemetryInit, shutdownTelemetry } from '../init'
 import {
   getActiveProvider,
   registerProvider,
@@ -12,6 +12,7 @@ import {
 import { resetTelemetryEnvCache } from '../env'
 import type { TelemetryProvider } from '../types'
 import { NOOP_SPAN } from '../provider/noop-provider'
+import * as errorReporting from '../facade/report-error'
 
 function provider(name = 'noop'): TelemetryProvider {
   return {
@@ -31,6 +32,7 @@ function provider(name = 'noop'): TelemetryProvider {
 
 describe('telemetry explicit opt-in boundary', () => {
   beforeEach(() => {
+    jest.useFakeTimers()
     delete process.env.TELEMETRY_BACKEND
     resetTelemetryEnvCache()
     resetTelemetryInit()
@@ -39,6 +41,12 @@ describe('telemetry explicit opt-in boundary', () => {
     resetTelemetryRuntime()
   })
 
+  afterEach(() => {
+    jest.restoreAllMocks()
+    resetTelemetryInit()
+    jest.useRealTimers()
+  })
+
   it('does not resolve a custom provider or register runtime hooks while off', async () => {
     const customNoop = provider()
     registerProvider(customNoop)
@@ -49,6 +57,7 @@ describe('telemetry explicit opt-in boundary', () => {
     expect(getActiveProvider().name).toBe('noop')
     expect(getLoggerExtension()).toBeUndefined()
     expect(getTelemetryRuntime()).toBeUndefined()
+    expect(jest.getTimerCount()).toBe(0)
   })
 
   it('can initialize after dotenv sets an enabled backend following an off call', async () => {
@@ -63,6 +72,7 @@ describe('telemetry explicit opt-in boundary', () => {
     expect(getActiveProvider()).toBe(customConsole)
     expect(getLoggerExtension()).toBeDefined()
     expect(getTelemetryRuntime()).toBeDefined()
+    expect(jest.getTimerCount()).toBe(1)
   })
 
   it('initializes an explicitly registered custom backend', async () => {
@@ -78,6 +88,69 @@ describe('telemetry explicit opt-in boundary', () => {
     expect(getTelemetryRuntime()).toBeDefined()
   })
 
+  it('shares concurrent initialization and releases the sampler on shutdown', async () => {
+    const customConsole = provider('console')
+    registerProvider(customConsole)
+    process.env.TELEMETRY_BACKEND = 'console'
+
+    await Promise.all([initTelemetry(), initTelemetry()])
+
+    expect(customConsole.start).toHaveBeenCalledTimes(1)
+    expect(jest.getTimerCount()).toBe(1)
+
+    await shutdownTelemetry()
+
+    expect(customConsole.shutdown).toHaveBeenCalledTimes(1)
+    expect(getTelemetryRuntime()).toBeUndefined()
+    expect(jest.getTimerCount()).toBe(0)
+  })
+
+  it.each([false, true])('reports rejected initialization during shutdown and permits recovery (reporting throws: %s)', async (reportingThrows) => {
+    const failure = new Error('[internal] telemetry provider startup failed')
+    let rejectStart!: (reason: unknown) => void
+    const startup = new Promise<void>((_resolve, reject) => { rejectStart = reject })
+    const customConsole = provider('console')
+    customConsole.start = jest.fn(() => startup)
+    registerProvider(customConsole)
+    process.env.TELEMETRY_BACKEND = 'console'
+    const reportError = jest.spyOn(errorReporting, 'reportError')
+    if (reportingThrows) {
+      reportError.mockImplementation(() => { throw new Error('[internal] reporting unavailable') })
+    }
+
+    const initialization = initTelemetry()
+    const initializationFailure = expect(initialization).rejects.toBe(failure)
+    const shutdown = shutdownTelemetry()
+    expect(customConsole.start).toHaveBeenCalledTimes(1)
+    expect(reportError).not.toHaveBeenCalled()
+
+    rejectStart(failure)
+    await initializationFailure
+    await expect(shutdown).resolves.toBeUndefined()
+
+    expect(reportError).toHaveBeenCalledTimes(1)
+    expect(reportError).toHaveBeenCalledWith(failure, {
+      module: 'telemetry',
+      code: 'telemetry.initialization_failed',
+    })
+    expect(customConsole.shutdown).not.toHaveBeenCalled()
+    expect(getActiveProvider().name).toBe('noop')
+    expect(getLoggerExtension()).toBeUndefined()
+    expect(getTelemetryRuntime()).toBeUndefined()
+    expect(jest.getTimerCount()).toBe(0)
+
+    const recoveredProvider = provider('console')
+    registerProvider(recoveredProvider)
+    await initTelemetry()
+    expect(recoveredProvider.start).toHaveBeenCalledTimes(1)
+    expect(jest.getTimerCount()).toBe(1)
+    await shutdownTelemetry()
+    expect(recoveredProvider.shutdown).toHaveBeenCalledTimes(1)
+    expect(getLoggerExtension()).toBeUndefined()
+    expect(getTelemetryRuntime()).toBeUndefined()
+    expect(jest.getTimerCount()).toBe(0)
+  })
+
   it('wires shared runtime histograms to the active provider', async () => {
     const customConsole = provider('console')
     registerProvider(customConsole)

package/src/__tests__/otlp-integration.test.ts

@@ -8,9 +8,15 @@ import { SpanStatusCode, propagation, trace, context, defaultTextMapSetter, ROOT
 import { W3CTraceContextPropagator } from '@opentelemetry/core'
 import { InMemorySpanExporter, SimpleSpanProcessor, type ReadableSpan } from '@opentelemetry/sdk-trace-base'
 import { InMemoryLogRecordExporter, SimpleLogRecordProcessor } from '@opentelemetry/sdk-logs'
-import { InMemoryMetricExporter, PeriodicExportingMetricReader, AggregationTemporality } from '@opentelemetry/sdk-metrics'
-
-import { OtlpProvider } from '../provider/otlp-provider'
+import {
+  InMemoryMetricExporter,
+  PeriodicExportingMetricReader,
+  AggregationTemporality,
+  DataPointType,
+  type Histogram,
+} from '@opentelemetry/sdk-metrics'
+
+import { OtlpProvider, SECONDS_HISTOGRAM_BUCKETS } from '../provider/otlp-provider'
 import { createBackupHeaderPropagator } from '../browser/propagator'
 import {
   createLogger,
@@ -20,7 +26,7 @@ import {
 import { setActiveProvider, resetActiveProvider } from '../provider/registry'
 import { resetTelemetryEnvCache } from '../env'
 import { registerTelemetryLogger } from '../facade/logger-bridge'
-import { withSpan, captureTraceContext, continueTrace, reportError, counter } from '../index'
+import { withSpan, captureTraceContext, continueTrace, reportError, counter, histogram } from '../index'
 
 const spanExporter = new InMemorySpanExporter()
 const logExporter = new InMemoryLogRecordExporter()
@@ -357,4 +363,28 @@ describe('OtlpProvider (in-memory exporters)', () => {
       .flatMap((rm) => rm.scopeMetrics.flatMap((sm) => sm.metrics.map((m) => m.descriptor.name)))
     expect(metricNames).toContain('om.errors')
   })
+
+  it('aggregates second-unit histograms with second-scaled buckets', async () => {
+    histogram('om.test.wait_time', 0.003, { pool: 'primary' }, 's')
+    histogram('om.test.wait_time', 0.0002, { pool: 'primary' }, 's')
+    histogram('om.test.payload_size', 3, undefined, 'By')
+    await metricReader.forceFlush()
+
+    const histogramPoint = (name: string) => {
+      const metric = metricExporter
+        .getMetrics()
+        .flatMap((rm) => rm.scopeMetrics.flatMap((sm) => sm.metrics))
+        .find((candidate) => candidate.descriptor.name === name)
+      expect(metric?.dataPointType).toBe(DataPointType.HISTOGRAM)
+      return metric?.dataPoints[0]?.value as Histogram | undefined
+    }
+
+    const seconds = histogramPoint('om.test.wait_time')
+    expect(seconds?.buckets.boundaries).toEqual([...SECONDS_HISTOGRAM_BUCKETS])
+    expect(seconds?.buckets.counts.slice(0, 3)).toEqual([1, 1, 0])
+    expect(seconds?.count).toBe(2)
+
+    const bytes = histogramPoint('om.test.payload_size')
+    expect(bytes?.buckets.boundaries).not.toEqual([...SECONDS_HISTOGRAM_BUCKETS])
+  })
 })

package/src/__tests__/runtime-metrics.test.ts

@@ -0,0 +1,239 @@
+import {
+  registerTelemetryMetricCollector,
+  registerTelemetryRuntime,
+  resetTelemetryMetricCollectors,
+  resetTelemetryRuntime,
+  type TelemetryMetricPoint,
+  type TelemetryRuntime,
+} from '@open-mercato/shared/lib/telemetry/runtime'
+import {
+  createRuntimeMetricsSampler,
+  resetRuntimeMetrics,
+  startRuntimeMetrics,
+  type EventLoopDelayMonitor,
+  type RuntimeMetricSources,
+} from '../runtime-metrics'
+import type { EventLoopUtilization } from 'node:perf_hooks'
+
+function createRuntime(recordMetric: (point: TelemetryMetricPoint) => void): TelemetryRuntime {
+  return {
+    canUseGlobalTracePropagation: () => false,
+    captureTraceContext: () => ({}),
+    continueTrace: (_carrier, _name, fn) => fn(),
+    recordMetric,
+    recordHttpDuration: () => {},
+    reportError: () => {},
+    shutdown: async () => {},
+  }
+}
+
+describe('runtime metrics sampler', () => {
+  afterEach(() => {
+    resetRuntimeMetrics()
+    resetTelemetryMetricCollectors()
+    resetTelemetryRuntime()
+  })
+
+  it('records interval runtime values and invokes shared metric collectors', () => {
+    const points: TelemetryMetricPoint[] = []
+    registerTelemetryRuntime(createRuntime((point) => points.push(point)))
+    const collectPool = jest.fn()
+    registerTelemetryMetricCollector(collectPool)
+
+    const delay: EventLoopDelayMonitor = {
+      enable: jest.fn(),
+      disable: jest.fn(),
+      percentile: jest.fn((percentile) => percentile * 1_000_000),
+      reset: jest.fn(),
+    }
+    const utilization: EventLoopUtilization[] = [
+      { idle: 10, active: 10, utilization: 0.5 },
+      { idle: 15, active: 25, utilization: 0.625 },
+      { idle: 5, active: 15, utilization: 0.75 },
+    ]
+    let sample = () => {}
+    const interval = { unref: jest.fn() } as unknown as ReturnType<typeof setInterval>
+    const sources: RuntimeMetricSources = {
+      createEventLoopDelayMonitor: () => delay,
+      eventLoopUtilization: jest.fn(() => utilization.shift() ?? {
+        idle: 0,
+        active: 0,
+        utilization: 0,
+      }),
+      memoryUsage: () => ({ rss: 123_456 }),
+      heapSpaceStatistics: () => [{
+        space_name: 'old_space',
+        space_used_size: 78_900,
+      }],
+      setInterval: (handler, intervalMs) => {
+        expect(intervalMs).toBe(10_000)
+        sample = handler
+        return interval
+      },
+      clearInterval: jest.fn(),
+    }
+
+    const dispose = createRuntimeMetricsSampler(sources)
+    sample()
+
+    expect(delay.enable).toHaveBeenCalledTimes(1)
+    expect(interval.unref).toHaveBeenCalledTimes(1)
+    expect(points).toEqual([
+      {
+        kind: 'gauge',
+        name: 'nodejs.eventloop.utilization',
+        value: 0.75,
+        unit: '1',
+      },
+      {
+        kind: 'gauge',
+        name: 'nodejs.eventloop.delay.p50',
+        value: 0.05,
+        unit: 's',
+      },
+      {
+        kind: 'gauge',
+        name: 'nodejs.eventloop.delay.p90',
+        value: 0.09,
+        unit: 's',
+      },
+      {
+        kind: 'gauge',
+        name: 'nodejs.eventloop.delay.p99',
+        value: 0.099,
+        unit: 's',
+      },
+      {
+        kind: 'gauge',
+        name: 'process.memory.usage',
+        value: 123_456,
+        unit: 'By',
+      },
+      {
+        kind: 'gauge',
+        name: 'v8js.memory.heap.used',
+        value: 78_900,
+        labels: { 'v8js.heap.space.name': 'old_space' },
+        unit: 'By',
+      },
+    ])
+    expect(collectPool).toHaveBeenCalledTimes(1)
+    expect(delay.reset).toHaveBeenCalledTimes(1)
+
+    dispose()
+
+    expect(sources.clearInterval).toHaveBeenCalledWith(interval)
+    expect(delay.disable).toHaveBeenCalledTimes(1)
+  })
+
+  it('normalizes non-finite delay percentiles to zero', () => {
+    const points: TelemetryMetricPoint[] = []
+    registerTelemetryRuntime(createRuntime((point) => points.push(point)))
+    let sample = () => {}
+    const interval = { unref: jest.fn() } as unknown as ReturnType<typeof setInterval>
+    const delay: EventLoopDelayMonitor = {
+      enable: jest.fn(),
+      disable: jest.fn(),
+      percentile: () => Number.NaN,
+      reset: jest.fn(),
+    }
+    const sources: RuntimeMetricSources = {
+      createEventLoopDelayMonitor: () => delay,
+      eventLoopUtilization: () => ({ idle: 0, active: 0, utilization: 0 }),
+      memoryUsage: () => ({ rss: 0 }),
+      heapSpaceStatistics: () => [],
+      setInterval: (handler) => {
+        sample = handler
+        return interval
+      },
+      clearInterval: jest.fn(),
+    }
+
+    const dispose = createRuntimeMetricsSampler(sources)
+    sample()
+    dispose()
+
+    expect(points.filter((point) => point.name.startsWith('nodejs.eventloop.delay.')))
+      .toEqual([
+        { kind: 'gauge', name: 'nodejs.eventloop.delay.p50', value: 0, unit: 's' },
+        { kind: 'gauge', name: 'nodejs.eventloop.delay.p90', value: 0, unit: 's' },
+        { kind: 'gauge', name: 'nodejs.eventloop.delay.p99', value: 0, unit: 's' },
+      ])
+  })
+
+  it.each([false, true])('reports sampling failures, recovers, and disposes once (reporting throws: %s)', (reportingThrows) => {
+    const failure = new Error('[internal] memory sample failed')
+    const reportError = jest.fn(() => {
+      if (reportingThrows) throw new Error('[internal] reporting sink failed')
+    })
+    const points: TelemetryMetricPoint[] = []
+    registerTelemetryRuntime({ ...createRuntime((point) => points.push(point)), reportError })
+    let sample = () => {}
+    const interval = { unref: jest.fn() } as unknown as ReturnType<typeof setInterval>
+    const delay: EventLoopDelayMonitor = {
+      enable: jest.fn(),
+      disable: jest.fn(),
+      percentile: () => 0,
+      reset: jest.fn(),
+    }
+    const sources: RuntimeMetricSources = {
+      createEventLoopDelayMonitor: () => delay,
+      eventLoopUtilization: () => ({ idle: 0, active: 0, utilization: 0 }),
+      memoryUsage: jest.fn().mockImplementationOnce(() => { throw failure }).mockReturnValue({ rss: 123 }),
+      heapSpaceStatistics: () => [],
+      setInterval: (handler) => {
+        sample = handler
+        return interval
+      },
+      clearInterval: jest.fn(),
+    }
+
+    const dispose = createRuntimeMetricsSampler(sources)
+    expect(sample).not.toThrow()
+    expect(sample).not.toThrow()
+    dispose()
+    dispose()
+
+    expect(reportError).toHaveBeenCalledWith(failure, {
+      module: 'telemetry',
+      code: 'telemetry.runtime_sampling_failed',
+    })
+    expect(points).toContainEqual({ kind: 'gauge', name: 'process.memory.usage', value: 123, unit: 'By' })
+    expect(delay.reset).toHaveBeenCalledTimes(2)
+    expect(delay.disable).toHaveBeenCalledTimes(1)
+    expect(sources.clearInterval).toHaveBeenCalledTimes(1)
+  })
+
+  it('shares one process sampler until every owner releases it', () => {
+    const interval = { unref: jest.fn() } as unknown as ReturnType<typeof setInterval>
+    const delay: EventLoopDelayMonitor = {
+      enable: jest.fn(),
+      disable: jest.fn(),
+      percentile: () => 0,
+      reset: jest.fn(),
+    }
+    const sources: RuntimeMetricSources = {
+      createEventLoopDelayMonitor: jest.fn(() => delay),
+      eventLoopUtilization: () => ({ idle: 0, active: 0, utilization: 0 }),
+      memoryUsage: () => ({ rss: 0 }),
+      heapSpaceStatistics: () => [],
+      setInterval: jest.fn(() => interval),
+      clearInterval: jest.fn(),
+    }
+
+    const releaseFirst = startRuntimeMetrics(sources)
+    const releaseSecond = startRuntimeMetrics(sources)
+
+    expect(sources.createEventLoopDelayMonitor).toHaveBeenCalledTimes(1)
+    expect(sources.setInterval).toHaveBeenCalledTimes(1)
+
+    releaseFirst()
+    expect(sources.clearInterval).not.toHaveBeenCalled()
+
+    releaseSecond()
+    releaseSecond()
+
+    expect(sources.clearInterval).toHaveBeenCalledTimes(1)
+    expect(delay.disable).toHaveBeenCalledTimes(1)
+  })
+})

package/src/init.ts

@@ -17,11 +17,14 @@ import { withSpan } from './facade/tracer'
 import { recordHttpDuration } from './facade/http'
 import { histogram } from './facade/meter'
 import { reportError } from './facade/report-error'
+import { startRuntimeMetrics } from './runtime-metrics'
 
 let initialized = false
+let initializing: Promise<void> | undefined
 let activeProvider: TelemetryProvider | undefined
 let disposeLoggerExtension: (() => void) | undefined
 let disposeRuntime: (() => void) | undefined
+let disposeRuntimeMetrics: (() => void) | undefined
 
 const logger = createLogger('telemetry')
 const OTLP_DEPENDENCY_UNAVAILABLE = Symbol.for('open-mercato.telemetry.otlp-dependency-unavailable')
@@ -46,7 +49,17 @@ class OtlpDependencyUnavailableError extends Error {
  */
 export async function initTelemetry(): Promise<void> {
   if (initialized) return
+  if (initializing) return initializing
 
+  let pending: Promise<void>
+  pending = initializeTelemetry().finally(() => {
+    if (initializing === pending) initializing = undefined
+  })
+  initializing = pending
+  return pending
+}
+
+async function initializeTelemetry(): Promise<void> {
   // Drop any env snapshot memoized BEFORE the host loaded its `.env` (the CLI
   // binary imports this package before dotenv runs) so init resolves the backend
   // from the actual, fully-loaded environment.
@@ -75,6 +88,7 @@ export async function initTelemetry(): Promise<void> {
   activeProvider = provider
   disposeLoggerExtension = registerTelemetryLogger(provider)
   disposeRuntime = registerTelemetryRuntime(createRuntime(provider))
+  disposeRuntimeMetrics = startRuntimeMetrics()
   initialized = true
 
   logger.info('Telemetry initialized', {
@@ -111,11 +125,26 @@ async function loadOtlpProvider(backend: TelemetryBackendName): Promise<Telemetr
 
 /** Flush + tear down the active backend (shutdown hook / `after()`). */
 export async function shutdownTelemetry(): Promise<void> {
+  if (initializing) {
+    try {
+      await initializing
+    } catch (err) {
+      try {
+        reportError(err, {
+          module: 'telemetry',
+          code: 'telemetry.initialization_failed',
+        })
+      } catch {}
+      logger.warn('Telemetry initialization failed before shutdown', { err })
+    }
+  }
   const provider = activeProvider
   activeProvider = undefined
   initialized = false
   disposeLoggerExtension?.()
   disposeLoggerExtension = undefined
+  disposeRuntimeMetrics?.()
+  disposeRuntimeMetrics = undefined
   disposeRuntime?.()
   disposeRuntime = undefined
   clearActiveProvider()
@@ -131,6 +160,7 @@ function createRuntime(provider: TelemetryProvider): TelemetryRuntime {
     continueTrace: (carrier, name, fn, options) =>
       continueTrace(carrier, name, () => fn(), options),
     withSpan: (name, fn, options) => withSpan(name, fn, options),
+    recordMetric: (point) => provider.recordMetric(point),
     recordHistogram: histogram,
     recordHttpDuration,
     reportError,
@@ -141,9 +171,12 @@ function createRuntime(provider: TelemetryProvider): TelemetryRuntime {
 /** Test-only: allow re-init in jest. */
 export function resetTelemetryInit(): void {
   initialized = false
+  initializing = undefined
   activeProvider = undefined
   disposeLoggerExtension?.()
   disposeLoggerExtension = undefined
+  disposeRuntimeMetrics?.()
+  disposeRuntimeMetrics = undefined
   disposeRuntime?.()
   disposeRuntime = undefined
 }

package/src/modules/telemetry/__integration__/TC-TELEMETRY-002.spec.ts

@@ -68,6 +68,12 @@ async function otelApiIsRegistered(page: Page): Promise<boolean> {
 
 test.describe('TC-TELEMETRY-002: browser RUM runtime', () => {
   test('exports document-load and fetch spans without breaking either fetch path', async ({ page, baseURL }) => {
+    // The server ignores the opt-in cookie unless the app runs with this variable; lanes that
+    // start the app from their own env (the standalone app) do not set it, so there is no SDK to boot.
+    test.skip(
+      process.env.OM_TEST_BROWSER_TELEMETRY_MODE !== 'opt-in',
+      'OM_TEST_BROWSER_TELEMETRY_MODE=opt-in is not configured for this lane.',
+    )
     const origin = new URL(baseURL ?? 'http://localhost:3000').origin
     await page.context().addCookies([{ name: OPT_IN_COOKIE, value: 'on', url: origin }])
     const ingest = recordIngest(page)

package/src/provider/otlp-provider.ts

@@ -23,6 +23,7 @@ import {
   type Counter,
   type Histogram,
   type Gauge,
+  type MetricOptions,
 } from '@opentelemetry/api'
 import { W3CTraceContextPropagator } from '@opentelemetry/core'
 import { logs, SeverityNumber } from '@opentelemetry/api-logs'
@@ -229,6 +230,22 @@ export type OtlpProviderOptions = {
  */
 export const PG_INSTRUMENTATION_OPTIONS = { enhancedDatabaseReporting: false } as const
 
+/**
+ * Bucket boundaries for every histogram recorded in seconds. Without explicit
+ * advice the SDK falls back to millisecond-scaled defaults (0, 5, 10 … 10000),
+ * which puts every realistic latency in the first bucket and leaves percentiles
+ * uncomputable. A superset of the semconv HTTP-duration advice, plus a 1 ms
+ * bucket for sub-millisecond pool acquisitions.
+ */
+export const SECONDS_HISTOGRAM_BUCKETS: readonly number[] = [
+  0.001, 0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10,
+]
+
+function histogramOptions(unit: string | undefined): MetricOptions | undefined {
+  if (unit !== 's') return unit ? { unit } : undefined
+  return { unit, advice: { explicitBucketBoundaries: [...SECONDS_HISTOGRAM_BUCKETS] } }
+}
+
 export class OtlpProvider implements TelemetryProvider {
   /** The configured backend name (signoz | newrelic | otlp) — all OTLP, vendor differs only by endpoint. */
   readonly name: string
@@ -339,7 +356,7 @@ export class OtlpProvider implements TelemetryProvider {
         point.kind === 'counter'
           ? meter.createCounter(point.name, opts)
           : point.kind === 'histogram'
-            ? meter.createHistogram(point.name, opts)
+            ? meter.createHistogram(point.name, histogramOptions(point.unit))
             : meter.createGauge(point.name, opts)
       this.instruments.set(key, instrument)
     }