@sentienguard/apm 1.0.10 → 1.0.14

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sentienguard/apm",
-  "version": "1.0.10",
+  "version": "1.0.14",
   "description": "SentienGuard APM SDK - Minimal, production-safe application performance monitoring",
   "main": "src/index.js",
   "types": "src/index.d.ts",

package/src/config.js

@@ -14,6 +14,7 @@ const config = {
   service: '',
   environment: 'production',
   endpoint: 'https://sentienguard-dev.the-algo.com/api/v1/apm/ingest',
+  tracesEndpoint: '',
   flushInterval: 10,
   maxRoutes: 100,
   maxPayloadSize: 1024 * 1024,
@@ -43,7 +44,12 @@ const config = {
     /** When true, outgoing HTTP to localhost is traced (for multi-service dev). Default false. */
     traceLocalHttp: false,
     /** Port -> display name for local peers, from SENTIENGUARD_PEER_SERVICE_MAP */
-    peerServiceMap: {}
+    peerServiceMap: {},
+    /** Sampling rate for exporting raw spans (0..1). Parent-based. */
+    sampleRate: 0.05,
+    /** Span export queue + batch sizes (drop-on-pressure). */
+    maxQueueSize: 2048,
+    maxBatchSize: 256
   }
 };
 
@@ -60,6 +66,10 @@ export function loadConfig({ force = false } = {}) {
   config.service = process.env.SENTIENGUARD_SERVICE || '';
   config.environment = process.env.SENTIENGUARD_ENV || 'production';
   config.endpoint = process.env.SENTIENGUARD_ENDPOINT || 'https://sentienguard-dev.the-algo.com/api/v1/apm/ingest';
+  config.tracesEndpoint =
+    process.env.SENTIENGUARD_TRACES_ENDPOINT ||
+    deriveTracesEndpoint(config.endpoint) ||
+    'https://sentienguard-dev.the-algo.com/api/v1/apm/traces';
   config.flushInterval = parseInt(process.env.SENTIENGUARD_FLUSH_INTERVAL, 10) || 10;
   config.maxRoutes = parseInt(process.env.SENTIENGUARD_MAX_ROUTES, 10) || 100;
   config.maxPayloadSize = parseInt(process.env.SENTIENGUARD_MAX_PAYLOAD_SIZE, 10) || 1024 * 1024;
@@ -82,6 +92,13 @@ export function loadConfig({ force = false } = {}) {
   config.openai.slowCallMs = parseInt(process.env.SENTIENGUARD_OPENAI_SLOW_CALL_MS, 10) || 5000;
 
   config.tracing.enabled = process.env.SENTIENGUARD_TRACING !== 'false';
+  const sampleRaw = process.env.SENTIENGUARD_TRACE_SAMPLE_RATE;
+  const sample = sampleRaw != null ? Number(sampleRaw) : NaN;
+  if (!Number.isNaN(sample) && sample >= 0 && sample <= 1) {
+    config.tracing.sampleRate = sample;
+  }
+  config.tracing.maxQueueSize = parseInt(process.env.SENTIENGUARD_TRACE_MAX_QUEUE_SIZE, 10) || config.tracing.maxQueueSize;
+  config.tracing.maxBatchSize = parseInt(process.env.SENTIENGUARD_TRACE_MAX_BATCH_SIZE, 10) || config.tracing.maxBatchSize;
   // Default behavior:
   // - production: do NOT record localhost dependency edges (noise + self-calls)
   // - non-production: DO record localhost edges (local multi-service dev "just works")
@@ -114,6 +131,17 @@ function parsePeerServiceMap(raw) {
   return map;
 }
 
+function deriveTracesEndpoint(ingestEndpoint) {
+  try {
+    const u = new URL(String(ingestEndpoint));
+    // Common default: /api/v1/apm/ingest -> /api/v1/apm/traces
+    u.pathname = u.pathname.replace(/\/apm\/ingest\/?$/i, '/apm/traces');
+    return u.toString();
+  } catch {
+    return '';
+  }
+}
+
 /**
  * Check if SDK is properly configured and should be active.
  * Triggers lazy config load if not yet loaded.

package/src/index.d.ts

@@ -8,6 +8,7 @@ export interface ApmConfig {
   service?: string;
   environment?: string;
   endpoint?: string;
+  tracesEndpoint?: string;
   flushInterval?: number;
   maxRoutes?: number;
   maxPayloadSize?: number;
@@ -17,6 +18,12 @@ export interface ApmConfig {
     enabled?: boolean;
     traceLocalHttp?: boolean;
     peerServiceMap?: Record<string, string>;
+    /** Export sampling for raw traces only (0..1). Metrics are not sampled. */
+    sampleRate?: number;
+    /** Drop-on-pressure queue size for raw span export */
+    maxQueueSize?: number;
+    /** Batch size for raw span export */
+    maxBatchSize?: number;
   };
 }
 

package/src/index.js

@@ -14,8 +14,12 @@
  *   SENTIENGUARD_SERVICE=my-api        (required)
  *   SENTIENGUARD_ENV=production        (optional, default: production)
  *   SENTIENGUARD_ENDPOINT=https://...  (optional)
+ *   SENTIENGUARD_TRACES_ENDPOINT=https://... (optional, raw span ingest; default derived from SENTIENGUARD_ENDPOINT)
  *   SENTIENGUARD_FLUSH_INTERVAL=10     (optional, seconds)
  *   SENTIENGUARD_TRACING=false          (optional, disable OpenTelemetry / W3C propagation; use legacy HTTP patches)
+ *   SENTIENGUARD_TRACE_SAMPLE_RATE=0.05 (optional, export sampling for raw traces only; metrics are not sampled)
+ *   SENTIENGUARD_TRACE_MAX_QUEUE_SIZE=2048 (optional, drop-on-pressure queue size for raw spans)
+ *   SENTIENGUARD_TRACE_MAX_BATCH_SIZE=256  (optional, batch size for raw span export)
  *   SENTIENGUARD_TRACE_LOCAL_HTTP=true  (optional, record outgoing HTTP to localhost as dependencies; use with SENTIENGUARD_PEER_SERVICE_MAP)
  *   SENTIENGUARD_PEER_SERVICE_MAP=3001:service-b,3002:other  (optional, port -> callee name for local peers)
  *
@@ -33,6 +37,7 @@ import { instrumentMongoDB, autoInstrumentMongoDB, stopMongoDBInstrumentation }
 import { instrumentOpenAI, stopOpenAIInstrumentation } from './openai.js';
 import { createBreaker, wrapMongoOperation, getBreakerStats, shutdownBreakers } from './circuitBreaker.js';
 import { startTracing, shutdownTracing, getActiveTraceId, isTracingActive } from './tracing.js';
+import { flushTraceQueue } from './traceTransport.js';
 
 let isInitialized = false;
 
@@ -100,6 +105,8 @@ function setupGracefulShutdown() {
 
     // Final flush
     await finalFlush();
+    // Best-effort flush of queued raw spans
+    await flushTraceQueue();
 
     debug('Shutdown complete');
   };
@@ -122,6 +129,8 @@ async function shutdown() {
   debug('Shutting down SDK');
 
   await shutdownTracing();
+  // After OTel shutdown, best-effort drain any serialized spans still queued in transport.
+  await flushTraceQueue();
 
   // Stop MongoDB instrumentation
   stopMongoDBInstrumentation();

package/src/traceSpanExporter.js

@@ -0,0 +1,123 @@
+/**
+ * SpanExporter that ships sampled raw spans to SentienGuard trace ingest.
+ *
+ * This is intentionally "lossy": it enqueues serialized spans to an async transport
+ * and returns SUCCESS quickly to avoid blocking the app.
+ */
+
+import { ExportResultCode, hrTimeToMilliseconds } from '@opentelemetry/core';
+import { SpanStatusCode } from '@opentelemetry/api';
+import { enqueueSpans } from './traceTransport.js';
+import { getConfig } from './config.js';
+
+function hrTimeToUnixNanoString(hrTime) {
+  // hrTime is [seconds, nanoseconds]
+  if (!Array.isArray(hrTime) || hrTime.length !== 2) return '';
+  const sec = BigInt(hrTime[0] || 0);
+  const ns = BigInt(hrTime[1] || 0);
+  return String(sec * 1000000000n + ns);
+}
+
+function statusForSpan(span) {
+  const code = span?.status?.code;
+  if (code === SpanStatusCode.ERROR) return { code: 'ERROR', message: span.status?.message || '' };
+  if (code === SpanStatusCode.OK) return { code: 'OK', message: span.status?.message || '' };
+  return { code: 'UNSET', message: span?.status?.message || '' };
+}
+
+function safeAttrs(attrs) {
+  if (!attrs) return {};
+  if (typeof attrs.get === 'function') {
+    const out = {};
+    for (const [k, v] of attrs.entries()) out[k] = v;
+    return out;
+  }
+  if (typeof attrs === 'object') return attrs;
+  return {};
+}
+
+function shouldSampleTraceId(traceId, sampleRate) {
+  if (sampleRate == null) return true;
+  const r = Number(sampleRate);
+  if (!Number.isFinite(r)) return true;
+  if (r <= 0) return false;
+  if (r >= 1) return true;
+
+  // Deterministic sampling based on trace_id (stable across services).
+  // Use the first 8 hex chars (32 bits) -> [0,1).
+  try {
+    const prefix = String(traceId).slice(0, 8);
+    if (!/^[0-9a-f]{8}$/i.test(prefix)) return Math.random() < r;
+    const n = parseInt(prefix, 16) >>> 0;
+    const p = n / 0x100000000; // 2^32
+    return p < r;
+  } catch {
+    return Math.random() < r;
+  }
+}
+
+function serializeSpan(span) {
+  const ctx = span?.spanContext?.();
+  if (!ctx?.traceId || !ctx?.spanId) return null;
+
+  const startNano = hrTimeToUnixNanoString(span.startTime);
+  const endNano = hrTimeToUnixNanoString(span.endTime);
+  if (!startNano || !endNano) return null;
+
+  const parentSpanId = span?.parentSpanId || span?.parentSpanContext?.spanId || null;
+  const status = statusForSpan(span);
+
+  const durationMs =
+    span.endTime && span.startTime
+      ? Math.max(0, hrTimeToMilliseconds(span.endTime) - hrTimeToMilliseconds(span.startTime))
+      : 0;
+
+  return {
+    trace_id: ctx.traceId,
+    span_id: ctx.spanId,
+    parent_span_id: parentSpanId || null,
+    name: span.name || '',
+    kind: span.kind != null ? String(span.kind) : undefined,
+    start_time_unix_nano: startNano,
+    end_time_unix_nano: endNano,
+    status,
+    attributes: safeAttrs(span.attributes),
+    events: Array.isArray(span.events) ? span.events : [],
+    links: Array.isArray(span.links) ? span.links : [],
+    duration_ms: Math.round(durationMs)
+  };
+}
+
+export class SentienGuardTraceSpanExporter {
+  export(spans, resultCallback) {
+    try {
+      const cfg = getConfig();
+      const rate = cfg?.tracing?.sampleRate;
+
+      const serialized = [];
+      for (const span of spans) {
+        try {
+          const s = serializeSpan(span);
+          if (s && shouldSampleTraceId(s.trace_id, rate)) serialized.push(s);
+        } catch {
+          // ignore
+        }
+      }
+
+      if (serialized.length) {
+        enqueueSpans(serialized);
+      }
+
+      resultCallback({ code: ExportResultCode.SUCCESS });
+    } catch (err) {
+      resultCallback({ code: ExportResultCode.FAILED, error: err });
+    }
+  }
+
+  shutdown() {
+    return Promise.resolve();
+  }
+}
+
+export default SentienGuardTraceSpanExporter;
+

package/src/traceTransport.js

@@ -0,0 +1,210 @@
+/**
+ * Trace Transport (raw spans)
+ *
+ * Rules:
+ * - Never block app requests.
+ * - Best-effort delivery; data loss is acceptable.
+ * - Drop under sustained failure or memory pressure.
+ */
+
+import https from 'https';
+import http from 'http';
+import { debug, warn, getConfig, isEnabled } from './config.js';
+
+let queue = [];
+let scheduled = false;
+let consecutiveFailures = 0;
+let lastFailureAtMs = 0;
+let recoveryTimer = null;
+
+const MAX_CONSECUTIVE_FAILURES = 5;
+const RECOVERY_BACKOFF_MS = 30_000;
+
+function sendToBackend(payload) {
+  return new Promise((resolve, reject) => {
+    const cfg = getConfig();
+    const data = JSON.stringify(payload);
+
+    // Reuse the same payload size protection as metrics
+    const maxBytes = cfg.maxPayloadSize || 1024 * 1024;
+    if (Buffer.byteLength(data) > maxBytes) {
+      return reject(new Error('Payload too large'));
+    }
+
+    let url;
+    try {
+      url = new URL(cfg.tracesEndpoint);
+    } catch {
+      return reject(new Error('Invalid traces endpoint URL'));
+    }
+
+    const isHttps = url.protocol === 'https:';
+    const transport = isHttps ? https : http;
+
+    const options = {
+      hostname: url.hostname,
+      port: url.port || (isHttps ? 443 : 80),
+      path: url.pathname + url.search,
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'Content-Length': Buffer.byteLength(data),
+        'X-APM-Key': cfg.apiKey,
+        'X-Service': cfg.service,
+        'User-Agent': '@sentienguard/apm/1.0.0'
+      },
+      timeout: 5000
+    };
+
+    const req = transport.request(options, (res) => {
+      let responseData = '';
+      res.on('data', (chunk) => {
+        responseData += chunk;
+      });
+      res.on('end', () => {
+        if (res.statusCode >= 200 && res.statusCode < 300) {
+          resolve({ statusCode: res.statusCode, data: responseData });
+        } else {
+          reject(new Error(`HTTP ${res.statusCode}: ${responseData}`));
+        }
+      });
+    });
+
+    req.on('error', (error) => {
+      const reason = error instanceof Error ? error : new Error(String(error));
+      reject(reason);
+    });
+
+    req.on('timeout', () => {
+      req.destroy();
+      reject(new Error('Request timeout'));
+    });
+
+    req.write(data);
+    req.end();
+  });
+}
+
+async function flushOnce(batch) {
+  if (!isEnabled()) return;
+  if (!batch.length) return;
+
+  const cfg = getConfig();
+  const payload = {
+    service: cfg.service,
+    environment: cfg.environment,
+    spans: batch
+  };
+
+  try {
+    await sendToBackend(payload);
+    consecutiveFailures = 0;
+    lastFailureAtMs = 0;
+    debug(`Trace flush ok: spans=${batch.length}`);
+  } catch (err) {
+    consecutiveFailures++;
+    lastFailureAtMs = Date.now();
+    warn(`Trace flush failed (attempt ${consecutiveFailures}): ${err.message}`);
+    if (consecutiveFailures >= MAX_CONSECUTIVE_FAILURES) {
+      // Stop retrying aggressively; drop future spans until backend recovers.
+      warn('Trace flush: max failures reached; dropping spans under backpressure');
+      if (!recoveryTimer) {
+        recoveryTimer = setTimeout(() => {
+          recoveryTimer = null;
+          consecutiveFailures = 0;
+          lastFailureAtMs = 0;
+          debug('Trace flush: recovery window elapsed; retrying delivery');
+          if (queue.length && !scheduled) {
+            scheduled = true;
+            setImmediate(drainQueue);
+          }
+        }, RECOVERY_BACKOFF_MS);
+        // Allow process to exit naturally.
+        if (typeof recoveryTimer.unref === 'function') recoveryTimer.unref();
+      }
+    }
+  }
+}
+
+function drainQueue() {
+  scheduled = false;
+  const cfg = getConfig();
+  const maxBatch = cfg.tracing?.maxBatchSize || 256;
+
+  // If backend is unhealthy, drop to protect app memory.
+  if (consecutiveFailures >= MAX_CONSECUTIVE_FAILURES) {
+    // During the recovery backoff window, keep dropping to protect memory.
+    // After backoff, recoveryTimer resets failures and normal flushing resumes.
+    queue = [];
+    return;
+  }
+
+  // Send at most one batch per tick to keep exporter callbacks cheap.
+  const batch = queue.slice(0, maxBatch);
+  queue = queue.slice(batch.length);
+  void flushOnce(batch);
+
+  if (queue.length) {
+    scheduled = true;
+    setImmediate(drainQueue);
+  }
+}
+
+export function enqueueSpans(serializedSpans) {
+  const cfg = getConfig();
+  const maxQueue = cfg.tracing?.maxQueueSize || 2048;
+
+  if (!Array.isArray(serializedSpans) || serializedSpans.length === 0) return;
+  if (!isEnabled()) return;
+
+  // Drop-on-pressure.
+  const room = maxQueue - queue.length;
+  if (room <= 0) return;
+  if (serializedSpans.length > room) {
+    queue.push(...serializedSpans.slice(0, room));
+  } else {
+    queue.push(...serializedSpans);
+  }
+
+  if (!scheduled) {
+    scheduled = true;
+    setImmediate(drainQueue);
+  }
+}
+
+export function resetTraceQueueForTests() {
+  queue = [];
+  scheduled = false;
+  consecutiveFailures = 0;
+  lastFailureAtMs = 0;
+  if (recoveryTimer) {
+    clearTimeout(recoveryTimer);
+    recoveryTimer = null;
+  }
+}
+
+/**
+ * Best-effort flush of queued spans (e.g. during shutdown).
+ * Never throws; safe to call even if tracing is disabled.
+ */
+export async function flushTraceQueue({ timeoutMs = 1500 } = {}) {
+  try {
+    if (!isEnabled()) return;
+    const deadline = Date.now() + Math.max(0, Number(timeoutMs) || 0);
+    const cfg = getConfig();
+    const maxBatch = cfg.tracing?.maxBatchSize || 256;
+
+    while (queue.length && Date.now() < deadline) {
+      const batch = queue.slice(0, maxBatch);
+      queue = queue.slice(batch.length);
+      // Attempt flush even if failures happened earlier; this is a single best-effort drain.
+      consecutiveFailures = 0;
+      lastFailureAtMs = 0;
+      // eslint-disable-next-line no-await-in-loop
+      await flushOnce(batch);
+    }
+  } catch {
+    // ignore
+  }
+}
+

package/src/tracing.js

@@ -10,8 +10,10 @@ import { SEMRESATTRS_SERVICE_NAME, SEMRESATTRS_DEPLOYMENT_ENVIRONMENT } from '@o
 import { W3CTraceContextPropagator } from '@opentelemetry/core';
 import { HttpInstrumentation } from '@opentelemetry/instrumentation-http';
 import { ExpressInstrumentation } from '@opentelemetry/instrumentation-express';
+import { AlwaysOnSampler, BatchSpanProcessor } from '@opentelemetry/sdk-trace-base';
 import { getConfig, debug } from './config.js';
 import { SentienGuardSpanExporter } from './spanExporter.js';
+import { SentienGuardTraceSpanExporter } from './traceSpanExporter.js';
 
 let sdk = null;
 let tracingActive = false;
@@ -55,7 +57,8 @@ export function startTracing() {
       [SEMRESATTRS_DEPLOYMENT_ENVIRONMENT]: cfg.environment
     });
 
-    const traceExporter = new SentienGuardSpanExporter();
+    const metricsExporter = new SentienGuardSpanExporter();
+    const traceExporter = new SentienGuardTraceSpanExporter();
 
     const httpInstrumentation = new HttpInstrumentation({
       ignoreOutgoingRequestHook: (requestOptions) => {
@@ -68,9 +71,15 @@ export function startTracing() {
 
     sdk = new NodeSDK({
       resource,
-      traceExporter,
+      // Important: keep span recording ON so APM metrics derived from spans are not undersampled.
+      // Raw trace export sampling is handled inside SentienGuardTraceSpanExporter instead.
+      sampler: new AlwaysOnSampler(),
       textMapPropagator: new W3CTraceContextPropagator(),
       instrumentations: [httpInstrumentation, expressInstrumentation],
+      spanProcessors: [
+        new BatchSpanProcessor(metricsExporter),
+        new BatchSpanProcessor(traceExporter)
+      ],
       autoDetectResources: false
     });