@omen.foundation/node-microservice-runtime 0.1.25 → 0.1.27

package/src/collector-manager.ts

@@ -5,6 +5,8 @@ import { pipeline } from 'stream/promises';
 import { createGunzip } from 'zlib';
 import type { Logger } from 'pino';
 import dgram from 'dgram';
+import type { EnvironmentConfig } from './types.js';
+import { hostToHttpUrl } from './utils/urls.js';
 
 // Protocol is httpprotobuf 
 
@@ -15,7 +17,7 @@ interface CollectorDiscoveryEntry {
   otlpEndpoint: string;
 }
 
-interface CollectorStatus {
+export interface CollectorStatus {
   isRunning: boolean;
   isReady: boolean;
   pid: number;
@@ -29,6 +31,62 @@ const DISCOVERY_PORT = parseInt(process.env.BEAM_COLLECTOR_DISCOVERY_PORT || '86
 const DISCOVERY_DELAY = 100; // milliseconds
 const DISCOVERY_ATTEMPTS = 10;
 
+/**
+ * ClickHouse credentials response from Beamable API
+ */
+interface ClickHouseCredentials {
+  endpoint: string;
+  expiresAt?: string;
+  password: string;
+  username: string;
+}
+
+/**
+ * Fetches ClickHouse credentials from Beamable API
+ * GET /api/beamo/otel/auth/writer/config
+ */
+export async function fetchClickHouseCredentials(
+  env: EnvironmentConfig,
+  logger: Logger
+): Promise<ClickHouseCredentials> {
+  const apiUrl = hostToHttpUrl(env.host);
+  const configUrl = new URL('/api/beamo/otel/auth/writer/config', apiUrl).toString();
+  
+  logger.info('[Collector] Fetching ClickHouse credentials from Beamable API...');
+  
+  // Build headers with authentication
+  const headers: Record<string, string> = {
+    'Content-Type': 'application/json',
+    Accept: 'application/json',
+    'beam-scope': `${env.cid}.${env.pid}`,
+  };
+  
+  // If we have a secret, we might need to sign the request
+  // For now, try without signature (the API might use the beam-scope header for auth)
+  const response = await fetch(configUrl, {
+    method: 'GET',
+    headers,
+  });
+  
+  if (!response.ok) {
+    const errorText = await response.text().catch(() => 'Unknown error');
+    throw new Error(`Failed to fetch ClickHouse credentials: ${response.status} ${response.statusText} - ${errorText}`);
+  }
+  
+  const credentials = await response.json() as ClickHouseCredentials;
+  
+  if (!credentials.endpoint || !credentials.username || !credentials.password) {
+    throw new Error('Invalid ClickHouse credentials response: missing required fields');
+  }
+  
+  logger.info('[Collector] Successfully fetched ClickHouse credentials');
+  if (credentials.expiresAt) {
+    logger.debug(`[Collector] Credentials expire at: ${credentials.expiresAt}`);
+  }
+  
+  return credentials;
+}
+
 /**
  * Gets the collector storage directory (similar to C# LocalApplicationData/beam/collectors/version)
  */
@@ -170,6 +228,36 @@ function discoverCollectorViaUDP(timeoutMs: number = 5000): Promise<CollectorDis
   });
 }
 
+/**
+ * Gets the current status of ClickHouse credentials (from env or API)
+ */
+export function getClickHouseCredentialsStatus(): {
+  hasEndpoint: boolean;
+  hasUsername: boolean;
+  hasPassword: boolean;
+  source: 'environment' | 'api' | 'missing';
+} {
+  const hasEndpoint = !!process.env.BEAM_CLICKHOUSE_ENDPOINT;
+  const hasUsername = !!process.env.BEAM_CLICKHOUSE_USERNAME;
+  const hasPassword = !!process.env.BEAM_CLICKHOUSE_PASSWORD;
+  
+  if (hasEndpoint && hasUsername && hasPassword) {
+    return {
+      hasEndpoint: true,
+      hasUsername: true,
+      hasPassword: true,
+      source: 'environment',
+    };
+  }
+  
+  return {
+    hasEndpoint,
+    hasUsername,
+    hasPassword,
+    source: 'missing',
+  };
+}
+
 /**
  * Checks if collector is already running via UDP discovery
  */
@@ -201,7 +289,8 @@ export async function isCollectorRunning(): Promise<CollectorStatus> {
  */
 export async function startCollector(
   logger: Logger,
-  otlpEndpoint?: string
+  otlpEndpoint?: string,
+  env?: EnvironmentConfig
 ): Promise<{ process: ChildProcess; endpoint: string }> {
   logger.info('[Collector] Resolving collector binary and config...');
   const collectorInfo = await resolveCollector(true, logger);
@@ -219,20 +308,34 @@ export async function startCollector(
   logger.info(`[Collector] Using binary: ${collectorInfo.binaryPath}`);
   logger.info(`[Collector] Using config: ${collectorInfo.configPath}`);
   
-  // Validate required environment variables (matching C# behavior)
-  // These must be set before starting the collector, otherwise it will fail
-  const requiredEnvVars = {
-    BEAM_CLICKHOUSE_ENDPOINT: process.env.BEAM_CLICKHOUSE_ENDPOINT,
-    BEAM_CLICKHOUSE_USERNAME: process.env.BEAM_CLICKHOUSE_USERNAME,
-    BEAM_CLICKHOUSE_PASSWORD: process.env.BEAM_CLICKHOUSE_PASSWORD,
-  };
+  // Fetch ClickHouse credentials from Beamable API if not in environment variables
+  // Per Gabriel: "these OTEL related ones you need to do it yourself at the beginning of the microservice startup"
+  let clickhouseEndpoint = process.env.BEAM_CLICKHOUSE_ENDPOINT;
+  let clickhouseUsername = process.env.BEAM_CLICKHOUSE_USERNAME;
+  let clickhousePassword = process.env.BEAM_CLICKHOUSE_PASSWORD;
   
-  const missingVars = Object.entries(requiredEnvVars)
-    .filter(([_, value]) => !value || value.trim() === '')
-    .map(([key]) => key);
+  if ((!clickhouseEndpoint || !clickhouseUsername || !clickhousePassword) && env) {
+    try {
+      const credentials = await fetchClickHouseCredentials(env, logger);
+      clickhouseEndpoint = credentials.endpoint;
+      clickhouseUsername = credentials.username;
+      clickhousePassword = credentials.password;
+      
+      // Set them in process.env so the collector can access them
+      process.env.BEAM_CLICKHOUSE_ENDPOINT = clickhouseEndpoint;
+      process.env.BEAM_CLICKHOUSE_USERNAME = clickhouseUsername;
+      process.env.BEAM_CLICKHOUSE_PASSWORD = clickhousePassword;
+    } catch (error) {
+      const errorMsg = `[Collector] Failed to fetch ClickHouse credentials from API: ${error instanceof Error ? error.message : String(error)}`;
+      logger.error(errorMsg);
+      throw new Error(errorMsg);
+    }
+  }
   
-  if (missingVars.length > 0) {
-    const errorMsg = `[Collector] Required environment variables are missing or empty: ${missingVars.join(', ')}. Collector will fail to start without these.`;
+  // Validate required environment variables (matching C# behavior)
+  // These must be set before starting the collector, otherwise it will fail
+  if (!clickhouseEndpoint || !clickhouseUsername || !clickhousePassword) {
+    const errorMsg = `[Collector] Required ClickHouse credentials are missing. Set BEAM_CLICKHOUSE_ENDPOINT, BEAM_CLICKHOUSE_USERNAME, and BEAM_CLICKHOUSE_PASSWORD, or ensure the API endpoint is accessible.`;
     logger.error(errorMsg);
     throw new Error(errorMsg);
   }
@@ -250,12 +353,12 @@ export async function startCollector(
   // Set environment variables for collector
   // Note: BEAM_CLICKHOUSE_ENDPOINT is for collector → ClickHouse communication
   // This is different from the OTLP endpoint (microservice → collector)
-  const env = {
+  const collectorEnv = {
     ...process.env,
     BEAM_OTLP_HTTP_ENDPOINT: localEndpoint,
-    BEAM_CLICKHOUSE_ENDPOINT: requiredEnvVars.BEAM_CLICKHOUSE_ENDPOINT!,
-    BEAM_CLICKHOUSE_USERNAME: requiredEnvVars.BEAM_CLICKHOUSE_USERNAME!,
-    BEAM_CLICKHOUSE_PASSWORD: requiredEnvVars.BEAM_CLICKHOUSE_PASSWORD!,
+    BEAM_CLICKHOUSE_ENDPOINT: clickhouseEndpoint!,
+    BEAM_CLICKHOUSE_USERNAME: clickhouseUsername!,
+    BEAM_CLICKHOUSE_PASSWORD: clickhousePassword!,
     BEAM_COLLECTOR_DISCOVERY_PORT: String(DISCOVERY_PORT),
     BEAM_CLICKHOUSE_PROCESSOR_TIMEOUT: process.env.BEAM_CLICKHOUSE_PROCESSOR_TIMEOUT || '5s',
     BEAM_CLICKHOUSE_PROCESSOR_BATCH_SIZE: process.env.BEAM_CLICKHOUSE_PROCESSOR_BATCH_SIZE || '5000',
@@ -269,7 +372,7 @@ export async function startCollector(
   
   // Start collector process
   const collectorProcess = spawn(collectorInfo.binaryPath, ['--config', collectorInfo.configPath], {
-    env,
+    env: collectorEnv,
     stdio: ['ignore', 'pipe', 'pipe'],
     detached: false,
   });
@@ -303,7 +406,8 @@ export async function startCollector(
  */
 export async function discoverOrStartCollector(
   logger: Logger,
-  standardOtelEnabled: boolean
+  standardOtelEnabled: boolean,
+  env?: EnvironmentConfig
 ): Promise<string | null> {
   if (!standardOtelEnabled) {
     return null;
@@ -319,7 +423,7 @@ export async function discoverOrStartCollector(
   // Collector not running - start it
   try {
     logger.info('[Collector] Starting OpenTelemetry collector...');
-    const { endpoint } = await startCollector(logger);
+    const { endpoint } = await startCollector(logger, undefined, env);
     
     // Wait a bit for collector to start and become ready
     // Try to discover it

package/src/index.ts

@@ -49,3 +49,8 @@ export {
   type FederatedItemProperty,
   type FederatedItemProxy,
 } from './federation.js';
+export {
+  isCollectorRunning,
+  getClickHouseCredentialsStatus,
+  type CollectorStatus,
+} from './collector-manager.js';

package/src/logger.ts

@@ -135,7 +135,7 @@ async function initializeOtlpLogging(
     if (!endpointUrl && standardOtelEnabled && logger) {
       // Try to discover existing collector or start a new one (like C# does)
       try {
-        const discoveredEndpoint = await discoverOrStartCollector(logger, standardOtelEnabled);
+        const discoveredEndpoint = await discoverOrStartCollector(logger, standardOtelEnabled, env);
         if (discoveredEndpoint) {
           endpointUrl = discoveredEndpoint;
         } else {
@@ -252,6 +252,31 @@ async function initializeOtlpLogging(
     // This helps diagnose if OTLP is working in deployed environments
     console.error(`[OTLP] OpenTelemetry logging initialized. Endpoint: ${finalEndpointUrl}, Service: ${serviceName || 'unknown'}`);
     
+    // CRITICAL: Verify collector is actually ready before returning
+    // If we started a collector, wait for it to be ready (up to 5 seconds)
+    if (standardOtelEnabled && logger) {
+      try {
+        const { isCollectorRunning } = await import('./collector-manager.js');
+        // Wait up to 5 seconds for collector to be ready
+        for (let i = 0; i < 50; i++) {
+          const status = await isCollectorRunning();
+          if (status.isRunning && status.isReady) {
+            if (logger) {
+              logger.info(`[OTLP] Collector confirmed ready at ${status.otlpEndpoint || finalEndpointUrl}`);
+            }
+            break;
+          }
+          // Wait 100ms between checks
+          await new Promise(resolve => setTimeout(resolve, 100));
+        }
+      } catch (error) {
+        // If we can't verify, log but continue
+        if (logger) {
+          logger.warn(`[OTLP] Could not verify collector readiness: ${error instanceof Error ? error.message : String(error)}`);
+        }
+      }
+    }
+    
     return Promise.resolve(loggerProvider);
   } catch (error) {
     // If OTLP initialization fails, log error but continue without OTLP
@@ -517,7 +542,7 @@ function initializeOtlpSync(
   serviceName?: string,
   qualifiedServiceName?: string,
   env?: EnvironmentConfig,
-  timeoutMs: number = 30000  // Increased to 30 seconds to allow collector download
+  timeoutMs: number = 60000  // 60 seconds to allow collector download, startup, and readiness check
 ): LoggerProvider | null {
   // Match C# logic: (this.InDocker() || UseLocalOtel) && !BEAM_DISABLE_STANDARD_OTEL
   const isInDocker = process.env.IS_LOCAL !== '1' && process.env.IS_LOCAL !== 'true';
@@ -575,14 +600,20 @@ function initializeOtlpSync(
   // Wait synchronously for the promise to resolve using deasync
   // deasync.loopWhile() allows event loop to process while we wait
   // This enables async operations (like collector download) to run and complete
+  // CRITICAL: This must complete before the logger is created to capture all startup logs
   try {
     // Use deasync to wait for completion, with timeout check
+    // loopWhile returns false to stop waiting, true to continue
     deasync.loopWhile(() => {
       // Check if we've exceeded timeout
-      if (Date.now() - startTime >= timeoutMs) {
+      const elapsed = Date.now() - startTime;
+      if (elapsed >= timeoutMs) {
+        initLogger.warn(`[OTLP] Synchronous wait timeout after ${elapsed}ms`);
         return false; // Stop waiting
       }
-      return !completed; // Continue waiting if not completed
+      // Continue waiting if not completed
+      // This allows the event loop to process async operations
+      return !completed;
     });
   } catch (error) {
     // If deasync fails, log and continue
@@ -591,9 +622,14 @@ function initializeOtlpSync(
   
   clearTimeout(timeoutId);
   
-  // If we timed out, log a warning but continue
-  if (!completed) {
+  // Verify we got a provider (collector is ready)
+  if (completed && provider) {
+    initLogger.info('[OTLP] Initialization completed successfully, collector is ready');
+  } else if (!completed) {
     initLogger.warn('[OTLP] Initialization did not complete in time, logs may not be sent via OTLP initially');
+    initLogger.warn('[OTLP] Service will continue without OTLP telemetry - requests will still be serviced');
+  } else {
+    initLogger.warn('[OTLP] Initialization completed but no provider returned, OTLP telemetry disabled');
   }
   
   return provider;
@@ -606,11 +642,12 @@ export function createLogger(env: EnvironmentConfig, options: LoggerFactoryOptio
   // Initialize OTLP synchronously BEFORE creating the logger
   // This ensures all logs from this point forward are captured via OTLP
   // Similar to C# microservices which configure logging early in startup
+  // CRITICAL: We wait for collector to be fully initialized and ready before proceeding
   const otlpProvider = initializeOtlpSync(
     options.serviceName,
     options.qualifiedServiceName,
     env,
-    30000 // 30 second timeout to allow collector download and startup
+    60000 // 60 second timeout to allow collector download, startup, and readiness verification
   );
   
   // Shared reference for OTLP logger provider