@ogcio/o11y-sdk-node 0.1.0-beta.9 → 0.3.0

package/lib/exporter/http.ts

@@ -1,13 +1,17 @@
-import { metrics } from "@opentelemetry/sdk-node";
-import { CompressionAlgorithm } from "@opentelemetry/otlp-exporter-base";
-import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
 import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-http";
 import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-http";
+import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
+import { CompressionAlgorithm } from "@opentelemetry/otlp-exporter-base";
+import { metrics } from "@opentelemetry/sdk-node";
+import { NodeSDKConfig } from "../index.js";
 import { LogRecordProcessorMap, SpanProcessorMap } from "../utils.js";
-import { EnrichSpanProcessor } from "../processor/enrich-span-processor.js";
+import {
+  _logsProcessorConfig,
+  _spansProcessorConfig,
+} from "./processor-config.js";
 import { Exporters } from "./index.js";
-import { NodeSDKConfig } from "../index.js";
-import { EnrichLogProcessor } from "../processor/enrich-logger-processor.js";
+import { getNodeSdkConfig } from "../config-manager.js";
+import { PIIExporterDecorator } from "./pii-exporter-decorator.js";
 
 export default function buildHttpExporters(config: NodeSDKConfig): Exporters {
   if (config.collectorUrl.endsWith("/")) {
@@ -17,26 +21,35 @@ export default function buildHttpExporters(config: NodeSDKConfig): Exporters {
   return {
     spans: [
       new SpanProcessorMap[config.collectorMode ?? "batch"](
-        new OTLPTraceExporter({
-          url: `${config.collectorUrl}/v1/traces`,
-          compression: CompressionAlgorithm.GZIP,
-        }),
+        new PIIExporterDecorator(
+          new OTLPTraceExporter({
+            url: `${config.collectorUrl}/v1/traces`,
+            compression: CompressionAlgorithm.GZIP,
+          }),
+          getNodeSdkConfig(),
+        ),
       ),
-      new EnrichSpanProcessor(config.spanAttributes),
+      ..._spansProcessorConfig(config),
     ],
     metrics: new metrics.PeriodicExportingMetricReader({
-      exporter: new OTLPMetricExporter({
-        url: `${config.collectorUrl}/v1/metrics`,
-        compression: CompressionAlgorithm.GZIP,
-      }),
+      exporter: new PIIExporterDecorator(
+        new OTLPMetricExporter({
+          url: `${config.collectorUrl}/v1/metrics`,
+          compression: CompressionAlgorithm.GZIP,
+        }),
+        getNodeSdkConfig(),
+      ),
     }),
     logs: [
-      new EnrichLogProcessor(config.spanAttributes),
+      ..._logsProcessorConfig(config),
       new LogRecordProcessorMap[config.collectorMode ?? "batch"](
-        new OTLPLogExporter({
-          url: `${config.collectorUrl}/v1/logs`,
-          compression: CompressionAlgorithm.GZIP,
-        }),
+        new PIIExporterDecorator(
+          new OTLPLogExporter({
+            url: `${config.collectorUrl}/v1/logs`,
+            compression: CompressionAlgorithm.GZIP,
+          }),
+          getNodeSdkConfig(),
+        ),
       ),
     ],
   };

package/lib/exporter/pii-exporter-decorator.ts

@@ -0,0 +1,152 @@
+import { ExportResult } from "@opentelemetry/core";
+import { OTLPExporterBase } from "@opentelemetry/otlp-exporter-base";
+import { ReadableLogRecord } from "@opentelemetry/sdk-logs";
+import { ReadableSpan, SpanExporter } from "@opentelemetry/sdk-trace-base";
+import { NodeSDKConfig } from "../index.js";
+import {
+  _cleanLogBodyPII,
+  _cleanObjectPII,
+  _cleanStringPII,
+} from "../internals/pii-detection.js";
+import {
+  PushMetricExporter,
+  ResourceMetrics,
+} from "@opentelemetry/sdk-metrics";
+
+export class PIIExporterDecorator
+  extends OTLPExporterBase<
+    (ReadableSpan | ReadableLogRecord)[] | ResourceMetrics
+  >
+  implements SpanExporter, PushMetricExporter
+{
+  private readonly _exporter;
+  private readonly _config;
+
+  constructor(
+    exporter: OTLPExporterBase<
+      (ReadableSpan | ReadableLogRecord)[] | ResourceMetrics
+    >,
+    config: NodeSDKConfig,
+  ) {
+    super(exporter["_delegate"]);
+    this._exporter = exporter;
+    this._config = config;
+  }
+
+  forceFlush(): Promise<void> {
+    return this._exporter.forceFlush();
+  }
+
+  shutdown(): Promise<void> {
+    return this._exporter.shutdown();
+  }
+
+  export(
+    items: (ReadableSpan | ReadableLogRecord)[] | ResourceMetrics,
+    resultCallback: (result: ExportResult) => void,
+  ): void {
+    if (!this._config.detection?.email) {
+      this._exporter.export(items, resultCallback);
+      return;
+    }
+
+    if (Array.isArray(items)) {
+      const redactedItem = items.map((item) => {
+        if (this._isReadableSpan(item)) {
+          this._redactSpan(item);
+        } else if (this._isReadableLogRecord(item)) {
+          this._redactLogRecord(item);
+        }
+        return item;
+      });
+
+      this._exporter.export(redactedItem, resultCallback);
+      return;
+    }
+
+    if (this._isResourceMetrics(items)) {
+      this._redactResourceMetrics(items);
+      this._exporter.export(items, resultCallback);
+    }
+  }
+
+  private _isReadableSpan(span: unknown): span is ReadableSpan {
+    return (
+      typeof span === "object" &&
+      span !== null &&
+      "name" in span &&
+      "kind" in span &&
+      "spanContext" in span &&
+      "attributes" in span
+    );
+  }
+
+  private _isReadableLogRecord(span: unknown): span is ReadableLogRecord {
+    return (
+      typeof span === "object" &&
+      span !== null &&
+      "body" in span &&
+      "attributes" in span &&
+      "severityText" in span &&
+      "severityNumber" in span
+    );
+  }
+
+  private _isResourceMetrics(obj: unknown): obj is ResourceMetrics {
+    return (
+      typeof obj === "object" &&
+      obj !== null &&
+      !Array.isArray(obj) &&
+      "resource" in obj &&
+      "scopeMetrics" in obj
+    );
+  }
+
+  private _redactSpan(span: ReadableSpan) {
+    Object.assign(span, {
+      name: _cleanStringPII(span.name, "trace"),
+      attributes: span.attributes && _cleanObjectPII(span.attributes, "trace"),
+      resource: {
+        attributes:
+          span?.resource?.attributes &&
+          _cleanObjectPII(span.resource.attributes, "trace"),
+      },
+      links: span?.links?.map((link) => {
+        Object.assign(link, {
+          attributes:
+            link?.attributes && _cleanObjectPII(link.attributes, "trace"),
+        });
+      }),
+      events: span?.events?.map((event) => {
+        Object.assign(event, {
+          name: _cleanStringPII(event.name, "trace"),
+          attributes:
+            event?.attributes && _cleanObjectPII(event.attributes, "trace"),
+        });
+        return event;
+      }),
+    });
+  }
+
+  private _redactLogRecord(log: ReadableLogRecord) {
+    Object.assign(log, {
+      body: _cleanLogBodyPII(log.body),
+      attributes: log.attributes && _cleanObjectPII(log.attributes, "log"),
+      resource: {
+        attributes:
+          log?.resource?.attributes &&
+          _cleanObjectPII(log.resource.attributes, "log"),
+      },
+    });
+  }
+
+  private _redactResourceMetrics(metric: ResourceMetrics) {
+    Object.assign(metric, {
+      resource: {
+        attributes:
+          metric?.resource?.attributes &&
+          _cleanObjectPII(metric.resource.attributes, "metric"),
+      },
+    });
+  }
+}

package/lib/exporter/processor-config.ts

@@ -0,0 +1,23 @@
+import { NodeSDKConfig } from "../index.js";
+import { EnrichLogProcessor } from "../processor/enrich-logger-processor.js";
+import { EnrichSpanProcessor } from "../processor/enrich-span-processor.js";
+
+export function _spansProcessorConfig(config: NodeSDKConfig) {
+  const _processor = [];
+
+  if (config.spanAttributes) {
+    _processor.push(new EnrichSpanProcessor(config.spanAttributes));
+  }
+
+  return _processor;
+}
+
+export function _logsProcessorConfig(config: NodeSDKConfig) {
+  const _processor = [];
+
+  if (config.spanAttributes) {
+    _processor.push(new EnrichLogProcessor(config.spanAttributes));
+  }
+
+  return _processor;
+}

package/lib/index.ts

@@ -1,14 +1,20 @@
+import type { Metadata } from "@grpc/grpc-js";
+
 export interface NodeSDKConfig {
   /**
    * The opentelemetry collector entrypoint GRPC url.
-   * If the collectoUrl is null or undefined, the instrumentation will not be activated.
+   * If the collectorUrl is null or undefined, the instrumentation will not be activated.
    * @example http://alloy:4317
    */
   collectorUrl: string;
   /**
-   * Name of your application used for the collector to group logs
+   * Name of your application used for the collector to group logs and naming traces
    */
   serviceName?: string;
+  /**
+   * Version of your application used for the collector to group logs and naming traces
+   */
+  serviceVersion?: string;
   /**
    * Diagnostic log level for the internal runtime instrumentation
    *
@@ -18,7 +24,7 @@ export interface NodeSDKConfig {
   diagLogLevel?: SDKLogLevel;
   /**
    * Collector signals processing mode.
-   * signle: makes an http/grpc request for each signal and it is immediately processed inside grafana
+   * single: makes an http/grpc request for each signal, and it is immediately processed inside grafana
    * batch: sends multiple signals within a time window, optimized to reduce http/grpc calls in production
    *
    * @type string
@@ -34,7 +40,7 @@ export interface NodeSDKConfig {
   ignoreUrls?: SamplerCondition[];
 
   /**
-   * Object containing static properties or functions used to evaluate custom attributes for every logs and traces.
+   * Object containing static properties or functions used to evaluate custom attributes for all logs and traces.
    */
   spanAttributes?: Record<
     string,
@@ -66,6 +72,24 @@ export interface NodeSDKConfig {
    * @default grpc
    */
   protocol?: SDKProtocol;
+
+  /**
+   * Grpc Metadata for the grpc-js client.
+   *
+   * @default { waitForReady: true }
+   */
+  grpcMetadata?: Metadata;
+
+  /**
+   * Enable/Disable PII detection for GDPR data
+   */
+  detection?: {
+    /**
+     * Redact email address
+     * @default true
+     */
+    email?: boolean;
+  };
 }
 
 export interface SamplerCondition {

package/lib/instrumentation.node.ts

@@ -7,17 +7,19 @@ import {
   ParentBasedSampler,
   TraceIdRatioBasedSampler,
 } from "@opentelemetry/sdk-trace-base";
+import { setNodeSdkConfig } from "./config-manager.js";
 import buildConsoleExporters from "./exporter/console.js";
 import buildGrpcExporters from "./exporter/grpc.js";
 import buildHttpExporters from "./exporter/http.js";
 import type { Exporters } from "./exporter/index.js";
 import type { NodeSDKConfig } from "./index.js";
+import { _shutdownHook } from "./internals/hooks.js";
 import { ObservabilityResourceDetector } from "./resource.js";
 import { UrlSampler } from "./url-sampler.js";
 
-export default function buildNodeInstrumentation(
+export default async function buildNodeInstrumentation(
   config?: NodeSDKConfig,
-): NodeSDK | undefined {
+): Promise<NodeSDK | undefined> {
   if (!config) {
     console.warn(
       "observability config not set. Skipping NodeJS OpenTelemetry instrumentation.",
@@ -39,16 +41,19 @@ export default function buildNodeInstrumentation(
     return;
   }
 
-  let exporter: Exporters;
+  if (!config.detection) {
+    config.detection = {
+      email: true,
+    };
+  }
 
-  if (config.protocol === "http") {
-    exporter = buildHttpExporters(config);
-  } else if (config.protocol === "console") {
-    exporter = buildConsoleExporters(config);
-  } else {
-    exporter = buildGrpcExporters(config);
+  if (config.detection.email === undefined) {
+    config.detection.email = true;
   }
 
+  // Init configManager to make it available to all o11y utils.
+  setNodeSdkConfig(config);
+
   const urlSampler = new UrlSampler(
     config.ignoreUrls,
     new TraceIdRatioBasedSampler(config.traceRatio ?? 1),
@@ -62,13 +67,27 @@ export default function buildNodeInstrumentation(
     localParentNotSampled: new AlwaysOffSampler(),
   });
 
+  diag.setLogger(
+    new DiagConsoleLogger(),
+    config.diagLogLevel ? DiagLogLevel[config.diagLogLevel] : DiagLogLevel.INFO,
+  );
+
   try {
-    diag.setLogger(
-      new DiagConsoleLogger(),
-      config.diagLogLevel
-        ? DiagLogLevel[config.diagLogLevel]
-        : DiagLogLevel.INFO,
-    );
+    const nodeSdkInstrumentation = getNodeAutoInstrumentations({
+      "@opentelemetry/instrumentation-fs": {
+        enabled: config.enableFS ?? false,
+      },
+    });
+
+    let exporter: Exporters;
+
+    if (config.protocol === "http") {
+      exporter = buildHttpExporters(config);
+    } else if (config.protocol === "console") {
+      exporter = buildConsoleExporters(config);
+    } else {
+      exporter = await buildGrpcExporters(config);
+    }
 
     const sdk = new NodeSDK({
       resourceDetectors: [
@@ -80,17 +99,13 @@ export default function buildNodeInstrumentation(
       logRecordProcessors: exporter.logs,
       sampler: mainSampler,
       textMapPropagator: new W3CTraceContextPropagator(),
-      instrumentations: [
-        getNodeAutoInstrumentations({
-          "@opentelemetry/instrumentation-fs": {
-            enabled: config.enableFS ?? false,
-          },
-        }),
-      ],
+      instrumentations: [nodeSdkInstrumentation],
     });
 
     sdk.start();
     console.log("NodeJS OpenTelemetry instrumentation started successfully.");
+
+    _shutdownHook(sdk);
     return sdk;
   } catch (error) {
     console.error(

package/lib/internals/hooks.ts

@@ -0,0 +1,14 @@
+export function _shutdownHook(sdk: { shutdown: () => Promise<void> }) {
+  process.on("SIGTERM", async () => {
+    try {
+      // Flushing before shutdown is implemented on a per-exporter basis.
+      await sdk.shutdown();
+      console.log("NodeJS OpenTelemetry instrumentation shutdown successfully");
+    } catch (error) {
+      console.error(
+        "Error shutting down NodeJS OpenTelemetry instrumentation:",
+        error,
+      );
+    }
+  });
+}

package/lib/internals/pii-detection.ts

@@ -0,0 +1,145 @@
+import type { AnyValue, AnyValueMap } from "@opentelemetry/api-logs";
+import { _getPIICounterRedactionMetric } from "./shared-metrics.js";
+
+const EMAIL_REGEX = /[a-zA-Z0-9._%+-]+@([a-zA-Z0-9.-]+\.[a-z]{2,})/gi;
+
+const decoder = new TextDecoder();
+const encoder = new TextEncoder();
+
+export type PIISource = "trace" | "log" | "metric";
+
+/**
+ * Redacts all email addresses in the input string and collects metadata.
+ *
+ * @param {string} value The input string potentially containing email addresses.
+ * @returns {{
+ *   redacted: string,
+ *   count: number,
+ *   domains: Record<string, number>
+ * }}
+ *
+ * An object containing:
+ *   - `redacted`: the string with email addresses replaced by `[REDACTED EMAIL]`
+ *   - `count`: total number of email addresses redacted
+ *   - `domains`: a map of domain names to the number of times they were redacted
+ */
+function _redactEmails(value: string): {
+  redacted: string;
+  count: number;
+  domains: Record<string, number>;
+} {
+  let count = 0;
+  const domains: Record<string, number> = {};
+
+  const redacted = value.replace(EMAIL_REGEX, (_, domain) => {
+    count++;
+    domains[domain] = (domains[domain] || 0) + 1;
+    return "[REDACTED EMAIL]";
+  });
+
+  return { redacted, count, domains };
+}
+
+/**
+ * Checks whether a string contains URI-encoded components.
+ *
+ * @param {string} value - The string to inspect.
+ * @returns {boolean} `true` if the string is encoded, `false` otherwise.
+ */
+function _containsEncodedComponents(value: string) {
+  try {
+    return decodeURI(value) !== decodeURIComponent(value);
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Cleans a string by redacting email addresses and emitting metrics for PII.
+ *
+ * If the string is URL-encoded, it will be decoded before redaction.
+ * Metrics are emitted for each domain found in redacted email addresses.
+ *
+ * @param {string} value - The input string to sanitize.
+ * @param {"trace" | "log"} source - The source context of the input, used in metrics.
+ * @returns {string} The cleaned string with any email addresses replaced by `[REDACTED EMAIL]`.
+ */
+export function _cleanStringPII(value: AnyValue, source: PIISource): AnyValue {
+  if (Array.isArray(value)) {
+    return value.map((v) => _cleanStringPII(v, source));
+  }
+
+  if (typeof value !== "string") {
+    return value;
+  }
+
+  let kind: "string" | "url" = "string";
+  let decodedValue = value;
+
+  if (_containsEncodedComponents(value)) {
+    decodedValue = decodeURIComponent(value);
+    kind = "url";
+  }
+
+  const { redacted, count, domains } = _redactEmails(decodedValue);
+
+  if (count > 0) {
+    for (const [domain, domainCount] of Object.entries(domains)) {
+      _getPIICounterRedactionMetric().add(domainCount, {
+        pii_type: "email",
+        redaction_source: source,
+        pii_email_domain: domain,
+        pii_format: kind,
+      });
+    }
+  }
+  return redacted;
+}
+
+export function _cleanObjectPII(entry: object, source: PIISource) {
+  if (!entry) {
+    return entry;
+  }
+
+  return Object.fromEntries(
+    Object.entries(entry).map(([k, v]) => [k, _cleanStringPII(v, source)]),
+  );
+}
+
+export function _cleanLogBodyPII(value: AnyValue): AnyValue {
+  if (typeof value === "string") {
+    return _cleanStringPII(value, "log");
+  }
+
+  if (
+    typeof value === "number" ||
+    typeof value === "boolean" ||
+    value == null
+  ) {
+    return value;
+  }
+
+  if (value instanceof Uint8Array) {
+    try {
+      const decoded = decoder.decode(value);
+      const sanitized = _cleanStringPII(decoded, "log") as string;
+      return encoder.encode(sanitized);
+    } catch {
+      return value;
+    }
+  }
+
+  if (Array.isArray(value)) {
+    return value.map(_cleanLogBodyPII);
+  }
+
+  if (typeof value === "object") {
+    const sanitized: AnyValueMap = {};
+    for (const [key, val] of Object.entries(value)) {
+      sanitized[key] = _cleanLogBodyPII(val);
+    }
+    return sanitized;
+  }
+
+  return value;
+}

package/lib/internals/shared-metrics.ts

@@ -0,0 +1,34 @@
+import { Attributes, Counter } from "@opentelemetry/api";
+import { getMetric } from "../metrics.js";
+import { PIISource } from "./pii-detection.js";
+
+interface RedactionMetric extends Attributes {
+  /** Type of PII redacted (e.g., "email", "phone"). */
+  pii_type: string;
+  /** Domain part of the redacted PII (e.g., "gmail.com"). */
+  pii_email_domain?: string;
+  /** Source of the redaction (trace, log or metric). */
+  redaction_source: PIISource;
+  /** Format or structure of the redacted value. */
+  pii_format: "string" | "url";
+}
+
+// Cached singleton instance of the redaction counter metric
+let _redactedCounter: undefined | Counter;
+
+/**
+ * Returns a singleton OpenTelemetry counter metric used to record occurrences of PII redactions.
+ *
+ * @returns {Counter} The singleton OpenTelemetry counter metric for PII redactions.
+ */
+export function _getPIICounterRedactionMetric() {
+  if (_redactedCounter) {
+    return _redactedCounter;
+  }
+
+  _redactedCounter = getMetric<"counter", RedactionMetric>("counter", {
+    meterName: "o11y",
+    metricName: "o11y_pii_redaction",
+  });
+  return _redactedCounter;
+}

package/lib/resource.ts

@@ -19,11 +19,12 @@ export class ObservabilityResourceDetector implements ResourceDetector {
     if (this._resourceAttributes) {
       attributes = {
         ...this._resourceAttributes,
-        "o11y.sdk.name": packageJson.name,
-        "o11y.sdk.version": packageJson.version,
       };
     }
 
+    attributes["o11y.sdk.name"] = packageJson.name;
+    attributes["o11y.sdk.version"] = packageJson.version;
+
     return { attributes };
   }
 }