@omen.foundation/node-microservice-runtime 0.1.17 → 0.1.19

package/src/collector-manager.ts

@@ -0,0 +1,299 @@
+import { spawn, ChildProcess } from 'child_process';
+import { createWriteStream, existsSync, chmodSync, mkdirSync } from 'fs';
+import { join } from 'path';
+import { pipeline } from 'stream/promises';
+import { createGunzip } from 'zlib';
+import type { Logger } from 'pino';
+import dgram from 'dgram';
+
+interface CollectorDiscoveryEntry {
+  version: string;
+  status: string;
+  pid: number;
+  otlpEndpoint: string;
+}
+
+interface CollectorStatus {
+  isRunning: boolean;
+  isReady: boolean;
+  pid: number;
+  otlpEndpoint?: string;
+  version?: string;
+}
+
+const COLLECTOR_VERSION = '0.0.123'; // Match C# collector version
+const COLLECTOR_DOWNLOAD_BASE = `https://collectors.beamable.com/version/${COLLECTOR_VERSION}`;
+const DISCOVERY_PORT = parseInt(process.env.BEAM_COLLECTOR_DISCOVERY_PORT || '8688', 10);
+const DISCOVERY_DELAY = 100; // milliseconds
+const DISCOVERY_ATTEMPTS = 10;
+
+/**
+ * Gets the collector storage directory (similar to C# LocalApplicationData/beam/collectors/version)
+ */
+function getCollectorStoragePath(): string {
+  // Use temp directory for now - in containers this should be writable
+  const tempDir = process.env.TMPDIR || process.env.TMP || '/tmp';
+  return join(tempDir, 'beam', 'collectors', COLLECTOR_VERSION);
+}
+
+/**
+ * Gets the collector binary name for the current platform
+ */
+function getCollectorBinaryName(): string {
+  const platform = process.platform;
+  const arch = process.arch;
+  
+  if (platform === 'linux' && arch === 'x64') {
+    return 'beamable-collector-linux-amd64';
+  } else if (platform === 'linux' && arch === 'arm64') {
+    return 'beamable-collector-linux-arm64';
+  } else if (platform === 'darwin' && arch === 'x64') {
+    return 'beamable-collector-darwin-amd64';
+  } else if (platform === 'darwin' && arch === 'arm64') {
+    return 'beamable-collector-darwin-arm64';
+  } else if (platform === 'win32' && arch === 'x64') {
+    return 'beamable-collector-windows-amd64.exe';
+  }
+  
+  throw new Error(`Unsupported platform: ${platform} ${arch}`);
+}
+
+/**
+ * Downloads and decompresses a gzipped file
+ */
+async function downloadAndDecompressGzip(url: string, outputPath: string, makeExecutable: boolean = false): Promise<void> {
+  const response = await fetch(url);
+  if (!response.ok) {
+    throw new Error(`Failed to download ${url}: ${response.statusText}`);
+  }
+  
+  const dir = require('path').dirname(outputPath);
+  if (!existsSync(dir)) {
+    mkdirSync(dir, { recursive: true });
+  }
+  
+  const gunzip = createGunzip();
+  const writeStream = createWriteStream(outputPath);
+  
+  await pipeline(response.body as any, gunzip, writeStream);
+  
+  if (makeExecutable && process.platform !== 'win32') {
+    try {
+      chmodSync(outputPath, 0o755);
+    } catch (error) {
+      console.error(`Failed to make ${outputPath} executable:`, error);
+    }
+  }
+}
+
+/**
+ * Resolves the collector binary and config, downloading if needed
+ */
+async function resolveCollector(allowDownload: boolean = true): Promise<{ binaryPath: string | null; configPath: string | null }> {
+  const basePath = getCollectorStoragePath();
+  const binaryName = getCollectorBinaryName();
+  const configName = 'clickhouse-config.yaml';
+  
+  const binaryPath = join(basePath, binaryName);
+  const configPath = join(basePath, configName);
+  
+  const itemsToDownload: Array<{ url: string; path: string; executable: boolean }> = [];
+  
+  if (!existsSync(binaryPath) && allowDownload) {
+    const binaryUrl = `${COLLECTOR_DOWNLOAD_BASE}/${binaryName}.gz`;
+    itemsToDownload.push({ url: binaryUrl, path: binaryPath, executable: true });
+  }
+  
+  if (!existsSync(configPath) && allowDownload) {
+    const configUrl = `${COLLECTOR_DOWNLOAD_BASE}/${configName}.gz`;
+    itemsToDownload.push({ url: configUrl, path: configPath, executable: false });
+  }
+  
+  for (const item of itemsToDownload) {
+    await downloadAndDecompressGzip(item.url, item.path, item.executable);
+  }
+  
+  return {
+    binaryPath: existsSync(binaryPath) ? binaryPath : null,
+    configPath: existsSync(configPath) ? configPath : null,
+  };
+}
+
+/**
+ * Discovers collector via UDP broadcast
+ */
+function discoverCollectorViaUDP(timeoutMs: number = 5000): Promise<CollectorDiscoveryEntry | null> {
+  return new Promise((resolve) => {
+    const socket = dgram.createSocket('udp4');
+    const discovered: CollectorDiscoveryEntry[] = [];
+    let timeout: NodeJS.Timeout;
+    
+    socket.on('message', (msg) => {
+      try {
+        const message = JSON.parse(msg.toString()) as CollectorDiscoveryEntry;
+        // Check if version matches
+        if (message.version === COLLECTOR_VERSION && message.status === 'READY') {
+          discovered.push(message);
+        }
+      } catch (error) {
+        // Ignore parse errors
+      }
+    });
+    
+    socket.on('error', () => {
+      clearTimeout(timeout);
+      socket.close();
+      resolve(null);
+    });
+    
+    socket.bind(() => {
+      socket.setBroadcast(true);
+      
+      timeout = setTimeout(() => {
+        socket.close();
+        // Return the first discovered collector
+        resolve(discovered.length > 0 ? discovered[0] : null);
+      }, timeoutMs);
+    });
+  });
+}
+
+/**
+ * Checks if collector is already running via UDP discovery
+ */
+export async function isCollectorRunning(): Promise<CollectorStatus> {
+  try {
+    const discovered = await discoverCollectorViaUDP(2000);
+    if (discovered) {
+      return {
+        isRunning: true,
+        isReady: discovered.status === 'READY',
+        pid: discovered.pid,
+        otlpEndpoint: discovered.otlpEndpoint,
+        version: discovered.version,
+      };
+    }
+  } catch (error) {
+    // Discovery failed, collector probably not running
+  }
+  
+  return {
+    isRunning: false,
+    isReady: false,
+    pid: 0,
+  };
+}
+
+/**
+ * Starts the OpenTelemetry collector process
+ */
+export async function startCollector(
+  logger: Logger,
+  otlpEndpoint?: string
+): Promise<{ process: ChildProcess; endpoint: string }> {
+  const collectorInfo = await resolveCollector(true);
+  
+  if (!collectorInfo.binaryPath) {
+    throw new Error('Collector binary not found and download failed');
+  }
+  
+  if (!collectorInfo.configPath) {
+    throw new Error('Collector config not found and download failed');
+  }
+  
+  // Determine OTLP endpoint
+  let localEndpoint = otlpEndpoint || 'localhost:4318';
+  localEndpoint = localEndpoint.replace(/^http:\/\//, '').replace(/^https:\/\//, '');
+  
+  // Set environment variables for collector
+  const env = {
+    ...process.env,
+    BEAM_OTLP_HTTP_ENDPOINT: localEndpoint,
+    BEAM_CLICKHOUSE_ENDPOINT: process.env.BEAM_CLICKHOUSE_ENDPOINT || '',
+    BEAM_CLICKHOUSE_USERNAME: process.env.BEAM_CLICKHOUSE_USERNAME || '',
+    BEAM_CLICKHOUSE_PASSWORD: process.env.BEAM_CLICKHOUSE_PASSWORD || '',
+    BEAM_COLLECTOR_DISCOVERY_PORT: String(DISCOVERY_PORT),
+    BEAM_CLICKHOUSE_PROCESSOR_TIMEOUT: process.env.BEAM_CLICKHOUSE_PROCESSOR_TIMEOUT || '5s',
+    BEAM_CLICKHOUSE_PROCESSOR_BATCH_SIZE: process.env.BEAM_CLICKHOUSE_PROCESSOR_BATCH_SIZE || '5000',
+    BEAM_CLICKHOUSE_EXPORTER_TIMEOUT: process.env.BEAM_CLICKHOUSE_EXPORTER_TIMEOUT || '5s',
+    BEAM_CLICKHOUSE_EXPORTER_QUEUE_SIZE: process.env.BEAM_CLICKHOUSE_EXPORTER_QUEUE_SIZE || '1000',
+    BEAM_CLICKHOUSE_EXPORTER_RETRY_ENABLED: process.env.BEAM_CLICKHOUSE_EXPORTER_RETRY_ENABLED || 'true',
+    BEAM_CLICKHOUSE_EXPORTER_RETRY_INITIAL_INTERVAL: process.env.BEAM_CLICKHOUSE_EXPORTER_RETRY_INITIAL_INTERVAL || '5s',
+    BEAM_CLICKHOUSE_EXPORTER_RETRY_MAX_INTERVAL: process.env.BEAM_CLICKHOUSE_EXPORTER_RETRY_MAX_INTERVAL || '30s',
+    BEAM_CLICKHOUSE_EXPORTER_RETRY_MAX_ELAPSED_TIME: process.env.BEAM_CLICKHOUSE_EXPORTER_RETRY_MAX_ELAPSED_TIME || '300s',
+  };
+  
+  // Start collector process
+  const collectorProcess = spawn(collectorInfo.binaryPath, ['--config', collectorInfo.configPath], {
+    env,
+    stdio: ['ignore', 'pipe', 'pipe'],
+    detached: false,
+  });
+  
+  collectorProcess.stdout?.on('data', (data) => {
+    logger.debug(`[Collector] ${data.toString().trim()}`);
+  });
+  
+  collectorProcess.stderr?.on('data', (data) => {
+    logger.debug(`[Collector ERR] ${data.toString().trim()}`);
+  });
+  
+  collectorProcess.on('error', (err) => {
+    logger.error(`[Collector] Failed to start: ${err.message}`);
+  });
+  
+  collectorProcess.on('exit', (code) => {
+    logger.warn(`[Collector] Process exited with code ${code}`);
+  });
+  
+  logger.info(`[Collector] Started with PID ${collectorProcess.pid}, endpoint: ${localEndpoint}`);
+  
+  return {
+    process: collectorProcess,
+    endpoint: `http://${localEndpoint}`,
+  };
+}
+
+/**
+ * Discovers or starts the collector and returns the OTLP endpoint
+ */
+export async function discoverOrStartCollector(
+  logger: Logger,
+  standardOtelEnabled: boolean
+): Promise<string | null> {
+  if (!standardOtelEnabled) {
+    return null;
+  }
+  
+  // First, check if collector is already running
+  const status = await isCollectorRunning();
+  if (status.isRunning && status.isReady && status.otlpEndpoint) {
+    logger.info(`[Collector] Found running collector at ${status.otlpEndpoint}`);
+    return `http://${status.otlpEndpoint}`;
+  }
+  
+  // Collector not running - start it
+  try {
+    logger.info('[Collector] Starting OpenTelemetry collector...');
+    const { endpoint } = await startCollector(logger);
+    
+    // Wait a bit for collector to start and become ready
+    // Try to discover it
+    for (let i = 0; i < DISCOVERY_ATTEMPTS; i++) {
+      await new Promise(resolve => setTimeout(resolve, DISCOVERY_DELAY));
+      const newStatus = await isCollectorRunning();
+      if (newStatus.isRunning && newStatus.isReady) {
+        logger.info(`[Collector] Collector is ready at ${newStatus.otlpEndpoint || endpoint}`);
+        return newStatus.otlpEndpoint ? `http://${newStatus.otlpEndpoint}` : endpoint;
+      }
+    }
+    
+    // Return the endpoint we started with, even if discovery didn't find it yet
+    logger.warn('[Collector] Collector started but not yet ready, using configured endpoint');
+    return endpoint;
+  } catch (err) {
+    logger.error(`[Collector] Failed to start collector: ${err instanceof Error ? err.message : String(err)}`);
+    return null;
+  }
+}
+

package/src/logger.ts

@@ -7,6 +7,8 @@ import { logs } from '@opentelemetry/api-logs';
 import { LoggerProvider, SimpleLogRecordProcessor } from '@opentelemetry/sdk-logs';
 import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-http';
 import { resourceFromAttributes, defaultResource } from '@opentelemetry/resources';
+import { discoverOrStartCollector } from './collector-manager.js';
+import type { Logger as PinoLogger } from 'pino';
 
 // Helper to get require function that works in both CJS and ESM
 declare const require: any;
@@ -73,22 +75,27 @@ function mapPinoLevelToBeamableLevel(level: number): string {
  * @param env - Environment configuration
  * @returns OTLP logger provider if configured, null otherwise
  */
-function initializeOtlpLogging(
+async function initializeOtlpLogging(
   serviceName?: string,
   qualifiedServiceName?: string,
-  env?: EnvironmentConfig
-): LoggerProvider | null {
+  env?: EnvironmentConfig,
+  logger?: PinoLogger
+): Promise<LoggerProvider | null> {
   // Check for explicit OTLP endpoint (same as C#: BEAM_OTEL_EXPORTER_OTLP_ENDPOINT)
-  const otlpEndpoint = process.env.BEAM_OTEL_EXPORTER_OTLP_ENDPOINT || process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
+  // Also check standard OTEL environment variables
+  const otlpEndpoint = process.env.BEAM_OTEL_EXPORTER_OTLP_ENDPOINT 
+    || process.env.OTEL_EXPORTER_OTLP_ENDPOINT
+    || (process.env.OTEL_EXPORTER_OTLP_ENDPOINT_LOGS ? process.env.OTEL_EXPORTER_OTLP_ENDPOINT_LOGS : null);
   
   // Check if standard OTLP is enabled (similar to C# OtelExporterStandardEnabled)
-  // In Docker, OTLP should be enabled unless explicitly disabled
+  // In Docker containers, OTLP should be enabled unless explicitly disabled
+  // Simple check: if IS_LOCAL is not set, we're likely in a container
   const isInDocker = process.env.IS_LOCAL !== '1' && process.env.IS_LOCAL !== 'true';
   const standardOtelEnabled = isInDocker && !process.env.BEAM_DISABLE_STANDARD_OTEL;
   
   // If no explicit endpoint and standard OTLP not enabled, skip OTLP
   if (!otlpEndpoint && !standardOtelEnabled) {
-    return null;
+    return Promise.resolve(null);
   }
   
   try {
@@ -113,23 +120,41 @@ function initializeOtlpLogging(
     
     // Determine endpoint URL
     // If explicit endpoint provided, use it (append /v1/logs if needed for HTTP)
-    // Otherwise, for standard OTLP, we'd need to discover it (similar to C# collector discovery)
-    // For now, if no explicit endpoint, we skip (collector discovery would go here)
+    // If standard OTLP enabled but no explicit endpoint, try to discover or start collector
     let endpointUrl = otlpEndpoint;
+    
+    if (!endpointUrl && standardOtelEnabled && logger) {
+      // Try to discover existing collector or start a new one (like C# does)
+      try {
+        const discoveredEndpoint = await discoverOrStartCollector(logger, standardOtelEnabled);
+        if (discoveredEndpoint) {
+          endpointUrl = discoveredEndpoint;
+        } else {
+          console.error('[OTLP] Standard OTLP is enabled but could not discover or start collector.');
+          return Promise.resolve(null);
+        }
+      } catch (error) {
+        console.error('[OTLP] Failed to discover/start collector:', error instanceof Error ? error.message : String(error));
+        return Promise.resolve(null);
+      }
+    }
+    
+    // If still no endpoint, skip OTLP
     if (!endpointUrl) {
-      // Standard OTLP enabled but no endpoint - would need collector discovery here
-      // For now, skip until we implement collector discovery
-      return null;
+      if (standardOtelEnabled) {
+        console.error('[OTLP] Standard OTLP is enabled but no endpoint available. Set BEAM_OTEL_EXPORTER_OTLP_ENDPOINT to enable OTLP logging.');
+      }
+      return Promise.resolve(null);
     }
     
     // Ensure endpoint has /v1/logs suffix for HTTP exporter (like C# does)
-    if (endpointUrl && !endpointUrl.includes('/v1/logs')) {
-      endpointUrl = `${endpointUrl.replace(/\/$/, '')}/v1/logs`;
-    }
+    const finalEndpointUrl = endpointUrl.includes('/v1/logs') 
+      ? endpointUrl 
+      : `${endpointUrl.replace(/\/$/, '')}/v1/logs`;
     
     // Create OTLP HTTP exporter (C# uses HttpProtobuf by default)
     const exporter = new OTLPLogExporter({
-      url: endpointUrl,
+      url: finalEndpointUrl,
       // Headers if provided (similar to C# OtelExporterOtlpHeaders)
       headers: process.env.BEAM_OTEL_EXPORTER_OTLP_HEADERS 
         ? JSON.parse(process.env.BEAM_OTEL_EXPORTER_OTLP_HEADERS)
@@ -153,12 +178,20 @@ function initializeOtlpLogging(
     // Set as global logger provider
     logs.setGlobalLoggerProvider(loggerProvider);
     
-    return loggerProvider;
+    // Log successful initialization (to stdout for debugging)
+    // This helps diagnose if OTLP is working in deployed environments
+    console.error(`[OTLP] OpenTelemetry logging initialized. Endpoint: ${finalEndpointUrl}, Service: ${serviceName || 'unknown'}`);
+    
+    return Promise.resolve(loggerProvider);
   } catch (error) {
     // If OTLP initialization fails, log error but continue without OTLP
     // Don't throw - we still want stdout logging to work
-    console.error('Failed to initialize OTLP logging:', error);
-    return null;
+    // Log to stderr so it's visible in container logs
+    console.error('[OTLP] Failed to initialize OTLP logging:', error instanceof Error ? error.message : String(error));
+    if (error instanceof Error && error.stack) {
+      console.error('[OTLP] Stack trace:', error.stack);
+    }
+    return Promise.resolve(null);
   }
 }
 
@@ -173,7 +206,7 @@ function initializeOtlpLogging(
 function createBeamableLogFormatter(
   serviceName?: string,
   qualifiedServiceName?: string,
-  otlpLoggerProvider?: LoggerProvider | null
+  otlpProviderRef?: { provider: LoggerProvider | null }
 ): Transform {
   return new Transform({
     objectMode: false, // Pino writes strings/Buffers, not objects
@@ -344,9 +377,10 @@ function createBeamableLogFormatter(
         Object.assign(beamableLog, otelFields);
         
         // Send log via OTLP if configured (similar to C# MicroserviceOtelLogRecordExporter)
-        if (otlpLoggerProvider) {
+        // Check if provider is available (may be null if still initializing)
+        if (otlpProviderRef?.provider) {
           try {
-            const otlpLogger = otlpLoggerProvider.getLogger(
+            const otlpLogger = otlpProviderRef.provider.getLogger(
               serviceName || 'beamable-node-runtime',
               undefined, // version
               {
@@ -404,13 +438,38 @@ export function createLogger(env: EnvironmentConfig, options: LoggerFactoryOptio
   const configuredDestination = options.destinationPath ?? process.env.LOG_PATH;
   const usePrettyLogs = shouldUsePrettyLogs();
   
-  // Initialize OTLP logging if configured (similar to C# microservices)
+  // Shared reference for OTLP logger provider (updated asynchronously)
+  const otlpProviderRef: { provider: LoggerProvider | null } = { provider: null };
+  
+  // Initialize OTLP logging in the background (similar to C# microservices)
   // This sends logs via OpenTelemetry Protocol in addition to stdout JSON
-  const otlpLoggerProvider = initializeOtlpLogging(
-    options.serviceName,
-    options.qualifiedServiceName,
-    env
-  );
+  // We start this async so it doesn't block logger creation
+  const isInDocker = process.env.IS_LOCAL !== '1' && process.env.IS_LOCAL !== 'true';
+  const standardOtelEnabled = isInDocker && !process.env.BEAM_DISABLE_STANDARD_OTEL;
+  
+  // Start collector discovery/start in background
+  if (standardOtelEnabled || process.env.BEAM_OTEL_EXPORTER_OTLP_ENDPOINT) {
+    // Create a temporary logger for OTLP initialization
+    const tempLogger = pino({
+      name: options.name ?? 'beamable-node-runtime',
+      level: env.logLevel,
+    }, process.stdout);
+    
+    // Initialize OTLP asynchronously (fire-and-forget)
+    initializeOtlpLogging(
+      options.serviceName,
+      options.qualifiedServiceName,
+      env,
+      tempLogger
+    ).then((provider) => {
+      otlpProviderRef.provider = provider;
+      if (provider) {
+        tempLogger.info('[OTLP] OpenTelemetry logging initialized successfully');
+      }
+    }).catch((error) => {
+      tempLogger.error(`[OTLP] Failed to initialize: ${error instanceof Error ? error.message : String(error)}`);
+    });
+  }
 
   const pinoOptions: LoggerOptions = {
     name: options.name ?? 'beamable-node-runtime',
@@ -442,7 +501,7 @@ export function createLogger(env: EnvironmentConfig, options: LoggerFactoryOptio
       const beamableFormatter = createBeamableLogFormatter(
         options.serviceName,
         options.qualifiedServiceName,
-        otlpLoggerProvider
+        otlpProviderRef
       );
       beamableFormatter.pipe(process.stdout);
       return pino(pinoOptions, beamableFormatter);
@@ -478,7 +537,7 @@ export function createLogger(env: EnvironmentConfig, options: LoggerFactoryOptio
     const beamableFormatter = createBeamableLogFormatter(
       options.serviceName,
       options.qualifiedServiceName,
-      otlpLoggerProvider
+      otlpProviderRef
     );
     const fileStream = destination({ dest: resolvedDestination, mkdir: true, append: true, sync: false });
     beamableFormatter.pipe(fileStream as unknown as NodeJS.WritableStream);