@omen.foundation/node-microservice-runtime 0.1.76 → 0.1.77

package/dist/collector-manager.cjs

@@ -0,0 +1,957 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.fetchClickHouseCredentials = fetchClickHouseCredentials;
+exports.getClickHouseCredentialsStatus = getClickHouseCredentialsStatus;
+exports.isCollectorRunning = isCollectorRunning;
+exports.addAuthEnvironmentVars = addAuthEnvironmentVars;
+exports.startCollectorAndWaitForReady = startCollectorAndWaitForReady;
+exports.startCollectorAsync = startCollectorAsync;
+exports.setupCollectorBeforeLogging = setupCollectorBeforeLogging;
+exports.startCollector = startCollector;
+exports.getCollectorProcessStatus = getCollectorProcessStatus;
+exports.discoverOrStartCollector = discoverOrStartCollector;
+const child_process_1 = require("child_process");
+const fs_1 = require("fs");
+const path_1 = require("path");
+const promises_1 = require("stream/promises");
+const zlib_1 = require("zlib");
+const node_crypto_1 = require("node:crypto");
+const net_1 = require("net");
+const pino_1 = __importDefault(require("pino"));
+const dgram_1 = __importDefault(require("dgram"));
+const deasync_1 = __importDefault(require("deasync"));
+const urls_js_1 = require("./utils/urls.js");
+const COLLECTOR_VERSION = '1.0.1';
+const COLLECTOR_DOWNLOAD_BASE = `https://collectors.beamable.com/version/${COLLECTOR_VERSION}`;
+const DISCOVERY_PORT = parseInt(process.env.BEAM_COLLECTOR_DISCOVERY_PORT || '8688', 10);
+function findFreePort() {
+    return new Promise((resolve, reject) => {
+        const server = (0, net_1.createServer)();
+        server.listen(0, () => {
+            const address = server.address();
+            const port = typeof address === 'object' && address !== null ? address.port : 0;
+            server.close(() => {
+                if (port > 0) {
+                    resolve(port);
+                }
+                else {
+                    resolve(9090);
+                }
+            });
+        });
+        server.on('error', (err) => {
+            reject(err);
+        });
+    });
+}
+let globalCollectorProcess = null;
+let globalCollectorStartError = null;
+let globalCollectorExitCode = null;
+let globalCollectorStderr = [];
+let globalCollectorStartupPromise = null;
+let globalCollectorInitError = null;
+function calculateSignature(pid, secret, uriPathAndQuery, body = null, version = '1') {
+    let dataToSign = `${secret}${pid}${version}${uriPathAndQuery}`;
+    if (body) {
+        dataToSign += body;
+    }
+    const hash = (0, node_crypto_1.createHash)('md5').update(dataToSign, 'utf8').digest('base64');
+    return hash;
+}
+function getPathAndQuery(url) {
+    try {
+        const urlObj = new URL(url);
+        return urlObj.pathname + urlObj.search;
+    }
+    catch {
+        return url;
+    }
+}
+async function fetchClickHouseCredentials(env) {
+    const apiUrl = (0, urls_js_1.hostToHttpUrl)(env.host);
+    const uriPath = '/api/beamo/otel/auth/writer/config';
+    const configUrl = new URL(uriPath, apiUrl).toString();
+    const verboseLogger = (0, pino_1.default)({ name: 'fetch-credentials', level: 'info' }, process.stdout);
+    verboseLogger.info(`[Credentials] Starting credential fetch from ${configUrl}...`);
+    const secret = process.env.SECRET;
+    if (!secret) {
+        verboseLogger.error('[Credentials] SECRET environment variable is missing');
+        throw new Error('SECRET environment variable is required to fetch ClickHouse credentials');
+    }
+    verboseLogger.info('[Credentials] SECRET found, calculating signature...');
+    const pathAndQuery = getPathAndQuery(uriPath);
+    const signature = calculateSignature(env.pid, secret, pathAndQuery, null, '1');
+    verboseLogger.info('[Credentials] Signature calculated, making API request...');
+    const headers = {
+        'Content-Type': 'application/json',
+        Accept: 'application/json',
+        'X-BEAM-SCOPE': `${env.cid}.${env.pid}`,
+        'X-BEAM-SIGNATURE': signature,
+    };
+    const fetchStartTime = Date.now();
+    const response = await fetch(configUrl, {
+        method: 'GET',
+        headers,
+    });
+    const fetchElapsed = Date.now() - fetchStartTime;
+    verboseLogger.info(`[Credentials] API request completed in ${fetchElapsed}ms, status: ${response.status}`);
+    if (!response.ok) {
+        const errorText = await response.text().catch(() => 'Unknown error');
+        verboseLogger.error(`[Credentials] API request failed: ${response.status} ${response.statusText}`);
+        throw new Error(`Failed to fetch ClickHouse credentials from ${configUrl}: ${response.status} ${response.statusText} - ${errorText.substring(0, 200)}`);
+    }
+    verboseLogger.info('[Credentials] Parsing response JSON...');
+    const credentials = await response.json();
+    if (!credentials.endpoint || !credentials.username || !credentials.password) {
+        verboseLogger.error('[Credentials] Invalid response: missing required fields');
+        throw new Error('Invalid ClickHouse credentials response: missing required fields');
+    }
+    verboseLogger.info(`[Credentials] Successfully fetched credentials (endpoint: ${credentials.endpoint}, username: ${credentials.username})`);
+    return credentials;
+}
+function getCollectorStoragePath() {
+    const productionPath = '/opt/beam/collectors';
+    const productionVersionPath = (0, path_1.join)(productionPath, COLLECTOR_VERSION);
+    if ((0, fs_1.existsSync)(productionPath)) {
+        return productionVersionPath;
+    }
+    const tempDir = process.env.TMPDIR || process.env.TMP || '/tmp';
+    return (0, path_1.join)(tempDir, 'beam', 'collectors', COLLECTOR_VERSION);
+}
+function getCollectorBinaryName() {
+    const platform = process.platform;
+    const arch = process.arch;
+    if (platform === 'linux' && arch === 'x64') {
+        return 'collector-linux-amd64';
+    }
+    else if (platform === 'linux' && arch === 'arm64') {
+        return 'collector-linux-arm64';
+    }
+    else if (platform === 'darwin' && arch === 'x64') {
+        return 'collector-darwin-amd64';
+    }
+    else if (platform === 'darwin' && arch === 'arm64') {
+        return 'collector-darwin-arm64';
+    }
+    else if (platform === 'win32' && arch === 'x64') {
+        return 'collector-windows-amd64.exe';
+    }
+    throw new Error(`Unsupported platform: ${platform} ${arch}`);
+}
+async function downloadAndDecompressGzip(url, outputPath, makeExecutable = false) {
+    var _a;
+    let response;
+    try {
+        const controller = new AbortController();
+        const timeoutId = setTimeout(() => controller.abort(), 30000);
+        try {
+            response = await fetch(url, {
+                signal: controller.signal,
+                headers: {
+                    'User-Agent': 'Beamable-Node-Microservice-Runtime/1.0',
+                },
+            });
+            clearTimeout(timeoutId);
+        }
+        catch (fetchError) {
+            clearTimeout(timeoutId);
+            if (fetchError instanceof Error) {
+                let underlyingError = '';
+                if (fetchError.cause) {
+                    const cause = fetchError.cause;
+                    if (cause instanceof Error) {
+                        const causeCode = cause.code;
+                        underlyingError = ` (cause: ${cause.name} - ${cause.message}${causeCode ? ` [code: ${causeCode}]` : ''})`;
+                    }
+                    else if (typeof cause === 'object' && cause.code) {
+                        underlyingError = ` (cause code: ${cause.code})`;
+                    }
+                }
+                if (fetchError.name === 'AbortError') {
+                    throw new Error(`Timeout downloading ${url} (30s limit exceeded)${underlyingError}`);
+                }
+                const errorCode = fetchError.code || ((_a = fetchError.cause) === null || _a === void 0 ? void 0 : _a.code);
+                if (errorCode) {
+                    if (errorCode === 'ENOTFOUND' || errorCode === 'EAI_AGAIN') {
+                        throw new Error(`DNS resolution failed for ${url} (code: ${errorCode}). Check network connectivity and DNS settings.`);
+                    }
+                    if (errorCode === 'ECONNREFUSED') {
+                        throw new Error(`Connection refused to ${url} (code: ${errorCode}). Server may be down or firewall blocking.`);
+                    }
+                    if (errorCode === 'ETIMEDOUT' || errorCode === 'ECONNRESET') {
+                        throw new Error(`Connection timeout/reset to ${url} (code: ${errorCode}). Network may be slow or unreachable.`);
+                    }
+                    if (errorCode === 'CERT_HAS_EXPIRED' || errorCode === 'UNABLE_TO_VERIFY_LEAF_SIGNATURE') {
+                        throw new Error(`SSL/TLS certificate error connecting to ${url} (code: ${errorCode}): ${fetchError.message}`);
+                    }
+                }
+                const errorMsg = fetchError.message.toLowerCase();
+                if (errorMsg.includes('enotfound') || errorMsg.includes('dns')) {
+                    throw new Error(`DNS resolution failed for ${url}. Check network connectivity and DNS settings.${underlyingError}`);
+                }
+                if (errorMsg.includes('econnrefused') || errorMsg.includes('connection refused')) {
+                    throw new Error(`Connection refused to ${url}. Server may be down or firewall blocking.${underlyingError}`);
+                }
+                if (errorMsg.includes('etimedout') || errorMsg.includes('timeout')) {
+                    throw new Error(`Connection timeout to ${url}. Network may be slow or unreachable.${underlyingError}`);
+                }
+                if (errorMsg.includes('certificate') || errorMsg.includes('ssl') || errorMsg.includes('tls')) {
+                    throw new Error(`SSL/TLS error connecting to ${url}: ${fetchError.message}${underlyingError}`);
+                }
+                throw new Error(`Network error downloading ${url}: ${fetchError.name} - ${fetchError.message}${underlyingError}`);
+            }
+            throw new Error(`Network error downloading ${url}: ${String(fetchError)}`);
+        }
+    }
+    catch (error) {
+        if (error instanceof Error) {
+            if (error.message.includes('Timeout') || error.message.includes('DNS') || error.message.includes('Connection')) {
+                throw error;
+            }
+            throw new Error(`Network error downloading ${url}: ${error.message}`);
+        }
+        throw new Error(`Network error downloading ${url}: ${String(error)}`);
+    }
+    if (!response.ok) {
+        const errorText = await response.text().catch(() => '');
+        throw new Error(`HTTP ${response.status} downloading ${url}: ${response.statusText}${errorText ? ` - ${errorText.substring(0, 200)}` : ''}`);
+    }
+    const dir = (0, path_1.dirname)(outputPath);
+    if (!(0, fs_1.existsSync)(dir)) {
+        (0, fs_1.mkdirSync)(dir, { recursive: true });
+    }
+    const gunzip = (0, zlib_1.createGunzip)();
+    const writeStream = (0, fs_1.createWriteStream)(outputPath);
+    await (0, promises_1.pipeline)(response.body, gunzip, writeStream);
+    if (makeExecutable && process.platform !== 'win32') {
+        try {
+            (0, fs_1.chmodSync)(outputPath, 0o755);
+        }
+        catch (error) {
+            console.error(`Failed to make ${outputPath} executable:`, error);
+        }
+    }
+}
+async function resolveCollector(allowDownload = true, logger) {
+    const basePath = getCollectorStoragePath();
+    const binaryName = getCollectorBinaryName();
+    const configName = 'clickhouse-config.yaml';
+    const binaryPath = (0, path_1.join)(basePath, binaryName);
+    const configPath = (0, path_1.join)(basePath, configName);
+    logger === null || logger === void 0 ? void 0 : logger.info(`[Collector] Resolving collector - basePath: ${basePath}, binaryName: ${binaryName}`);
+    logger === null || logger === void 0 ? void 0 : logger.info(`[Collector] Checking binary at: ${binaryPath}`);
+    logger === null || logger === void 0 ? void 0 : logger.info(`[Collector] Checking config at: ${configPath}`);
+    const baseDirExists = (0, fs_1.existsSync)(basePath);
+    logger === null || logger === void 0 ? void 0 : logger.info(`[Collector] Base directory exists: ${baseDirExists}`);
+    if (baseDirExists) {
+        try {
+            const fsPromises = await Promise.resolve().then(() => __importStar(require('fs/promises')));
+            const dirContents = await fsPromises.readdir(basePath).catch(() => []);
+            logger === null || logger === void 0 ? void 0 : logger.info(`[Collector] Base directory contents: ${dirContents.join(', ') || '(empty)'}`);
+        }
+        catch (e) {
+            logger === null || logger === void 0 ? void 0 : logger.warn(`[Collector] Could not read base directory: ${e instanceof Error ? e.message : String(e)}`);
+        }
+    }
+    const binaryExists = (0, fs_1.existsSync)(binaryPath);
+    const configExists = (0, fs_1.existsSync)(configPath);
+    logger === null || logger === void 0 ? void 0 : logger.info(`[Collector] Binary exists: ${binaryExists}, Config exists: ${configExists}`);
+    const itemsToDownload = [];
+    if (!binaryExists && allowDownload) {
+        const binaryUrl = `${COLLECTOR_DOWNLOAD_BASE}/${binaryName}.gz`;
+        itemsToDownload.push({ url: binaryUrl, path: binaryPath, executable: true });
+        logger === null || logger === void 0 ? void 0 : logger.warn(`[Collector] Binary not found at ${binaryPath}, will download from: ${binaryUrl}`);
+    }
+    else if (binaryExists) {
+        logger === null || logger === void 0 ? void 0 : logger.info(`[Collector] Binary found at: ${binaryPath}`);
+    }
+    else if (!allowDownload) {
+        logger === null || logger === void 0 ? void 0 : logger.warn(`[Collector] Binary not found at ${binaryPath} and downloads are disabled`);
+    }
+    if (!configExists && allowDownload) {
+        const configUrl = `${COLLECTOR_DOWNLOAD_BASE}/${configName}.gz`;
+        itemsToDownload.push({ url: configUrl, path: configPath, executable: false });
+        logger === null || logger === void 0 ? void 0 : logger.warn(`[Collector] Config not found at ${configPath}, will download from: ${configUrl}`);
+    }
+    else if (configExists) {
+        logger === null || logger === void 0 ? void 0 : logger.info(`[Collector] Config found at: ${configPath}`);
+    }
+    else if (!allowDownload) {
+        logger === null || logger === void 0 ? void 0 : logger.warn(`[Collector] Config not found at ${configPath} and downloads are disabled`);
+    }
+    if (itemsToDownload.length > 0) {
+        logger === null || logger === void 0 ? void 0 : logger.info(`[Collector] Downloading ${itemsToDownload.length} item(s) in parallel...`);
+        const downloadPromises = itemsToDownload.map(async (item) => {
+            try {
+                logger === null || logger === void 0 ? void 0 : logger.info(`[Collector] Downloading ${item.url}...`);
+                await downloadAndDecompressGzip(item.url, item.path, item.executable);
+                logger === null || logger === void 0 ? void 0 : logger.info(`[Collector] Downloaded to ${item.path}`);
+            }
+            catch (error) {
+                const errorMsg = error instanceof Error ? error.message : String(error);
+                logger === null || logger === void 0 ? void 0 : logger.error(`[Collector] Failed to download ${item.url}: ${errorMsg}`);
+                if (error instanceof Error && errorMsg.includes(item.url)) {
+                    throw error;
+                }
+                throw new Error(`Failed to download collector ${item.executable ? 'binary' : 'config'} from ${item.url}: ${errorMsg}`);
+            }
+        });
+        await Promise.all(downloadPromises);
+    }
+    return {
+        binaryPath: (0, fs_1.existsSync)(binaryPath) ? binaryPath : null,
+        configPath: (0, fs_1.existsSync)(configPath) ? configPath : null,
+    };
+}
+function discoverCollectorViaUDP(timeoutMs = 1000) {
+    return new Promise((resolve) => {
+        const socket = dgram_1.default.createSocket('udp4');
+        const discovered = [];
+        let timeout;
+        let resolved = false;
+        const startTime = Date.now();
+        const safetyTimeout = setTimeout(() => {
+            if (!resolved) {
+                console.log(`[UDP Discovery] Safety timeout triggered - forcing resolution`);
+                resolved = true;
+                try {
+                    socket.removeAllListeners();
+                    socket.close();
+                }
+                catch (e) {
+                }
+                clearTimeout(timeout);
+                resolve(discovered.length > 0 ? discovered[0] : null);
+            }
+        }, timeoutMs + 100);
+        const doResolve = (result) => {
+            if (!resolved) {
+                resolved = true;
+                clearTimeout(timeout);
+                clearTimeout(safetyTimeout);
+                try {
+                    socket.removeAllListeners();
+                    socket.close();
+                }
+                catch (e) {
+                }
+                console.log(`[UDP Discovery] Resolving promise with result: ${result ? 'found' : 'null'}`);
+                resolve(result);
+            }
+        };
+        socket.on('message', (msg) => {
+            try {
+                const message = JSON.parse(msg.toString());
+                if (message.version === COLLECTOR_VERSION && message.status === 'READY') {
+                    discovered.push(message);
+                    const elapsed = Date.now() - startTime;
+                    console.log(`[UDP Discovery] Found collector after ${elapsed}ms: ${message.otlpEndpoint}`);
+                    doResolve(discovered[0]);
+                }
+            }
+            catch (error) {
+            }
+        });
+        socket.on('error', (err) => {
+            const elapsed = Date.now() - startTime;
+            console.log(`[UDP Discovery] Socket error after ${elapsed}ms: ${err.message}`);
+            doResolve(null);
+        });
+        socket.bind(() => {
+            socket.setBroadcast(true);
+            console.log(`[UDP Discovery] Listening for collector broadcasts (timeout: ${timeoutMs}ms)...`);
+            timeout = setTimeout(() => {
+                const elapsed = Date.now() - startTime;
+                if (discovered.length > 0) {
+                    console.log(`[UDP Discovery] Timeout after ${elapsed}ms, found ${discovered.length} collector(s)`);
+                }
+                else {
+                    console.log(`[UDP Discovery] Timeout after ${elapsed}ms, no collector found`);
+                }
+                doResolve(discovered.length > 0 ? discovered[0] : null);
+            }, timeoutMs);
+        });
+    });
+}
+function getClickHouseCredentialsStatus() {
+    const hasEndpoint = !!process.env.BEAM_CLICKHOUSE_ENDPOINT;
+    const hasUsername = !!process.env.BEAM_CLICKHOUSE_USERNAME;
+    const hasPassword = !!process.env.BEAM_CLICKHOUSE_PASSWORD;
+    if (hasEndpoint && hasUsername && hasPassword) {
+        return {
+            hasEndpoint: true,
+            hasUsername: true,
+            hasPassword: true,
+            source: 'environment',
+        };
+    }
+    return {
+        hasEndpoint,
+        hasUsername,
+        hasPassword,
+        source: 'missing',
+    };
+}
+async function isCollectorRunning() {
+    if (globalCollectorProcess) {
+        try {
+            const processAlive = globalCollectorProcess.exitCode === null &&
+                globalCollectorProcess.killed === false;
+            if (processAlive) {
+                const stderrText = globalCollectorStderr.join('\n');
+                const isReady = stderrText.includes('Everything is ready') ||
+                    stderrText.includes('Begin running and processing data');
+                if (isReady) {
+                    try {
+                        const discovered = await discoverCollectorViaUDP(1000);
+                        if (discovered && discovered.status === 'READY') {
+                            return {
+                                isRunning: true,
+                                isReady: true,
+                                pid: discovered.pid || globalCollectorProcess.pid || 0,
+                                otlpEndpoint: discovered.otlpEndpoint,
+                                version: discovered.version,
+                            };
+                        }
+                    }
+                    catch {
+                    }
+                    const configuredEndpoint = process.env.BEAM_OTLP_HTTP_ENDPOINT || '0.0.0.0:4318';
+                    return {
+                        isRunning: true,
+                        isReady: true,
+                        pid: globalCollectorProcess.pid || 0,
+                        otlpEndpoint: configuredEndpoint,
+                        version: COLLECTOR_VERSION,
+                    };
+                }
+            }
+        }
+        catch {
+        }
+    }
+    try {
+        const udpStartTime = Date.now();
+        console.log('[UDP Discovery] Starting UDP discovery in isCollectorRunning()...');
+        console.log('[UDP Discovery] Calling discoverCollectorViaUDP(1000)...');
+        const discovered = await discoverCollectorViaUDP(1000);
+        const udpElapsed = Date.now() - udpStartTime;
+        console.log(`[UDP Discovery] UDP discovery promise resolved after ${udpElapsed}ms, result: ${discovered ? 'found' : 'null'}`);
+        if (discovered) {
+            console.log(`[UDP Discovery] Found collector in ${udpElapsed}ms: ${discovered.otlpEndpoint}`);
+            return {
+                isRunning: true,
+                isReady: discovered.status === 'READY',
+                pid: discovered.pid,
+                otlpEndpoint: discovered.otlpEndpoint,
+                version: discovered.version,
+            };
+        }
+        else {
+            console.log(`[UDP Discovery] No collector found after ${udpElapsed}ms, returning not running status`);
+        }
+    }
+    catch (error) {
+        const errorMsg = error instanceof Error ? error.message : String(error);
+        console.log(`[UDP Discovery] Error in isCollectorRunning(): ${errorMsg}`);
+        if (error instanceof Error && error.stack) {
+            console.log(`[UDP Discovery] Error stack: ${error.stack}`);
+        }
+    }
+    console.log('[UDP Discovery] Returning default not-running status from isCollectorRunning()');
+    return {
+        isRunning: false,
+        isReady: false,
+        pid: 0,
+    };
+}
+function addCollectorConfigurationToEnvironment() {
+    const defaults = {
+        BEAM_CLICKHOUSE_PROCESSOR_TIMEOUT: '5s',
+        BEAM_CLICKHOUSE_PROCESSOR_BATCH_SIZE: '5000',
+        BEAM_CLICKHOUSE_EXPORTER_TIMEOUT: '5s',
+        BEAM_CLICKHOUSE_EXPORTER_QUEUE_SIZE: '1000',
+        BEAM_CLICKHOUSE_EXPORTER_RETRY_ENABLED: 'true',
+        BEAM_CLICKHOUSE_EXPORTER_RETRY_INITIAL_INTERVAL: '5s',
+        BEAM_CLICKHOUSE_EXPORTER_RETRY_MAX_INTERVAL: '30s',
+        BEAM_CLICKHOUSE_EXPORTER_RETRY_MAX_ELAPSED_TIME: '300s',
+    };
+    for (const [key, defaultValue] of Object.entries(defaults)) {
+        if (!process.env[key]) {
+            process.env[key] = defaultValue;
+        }
+    }
+}
+function addAuthEnvironmentVars(endpoint, username, password) {
+    process.env.BEAM_CLICKHOUSE_ENDPOINT = endpoint;
+    process.env.BEAM_CLICKHOUSE_USERNAME = username;
+    process.env.BEAM_CLICKHOUSE_PASSWORD = password;
+    if (!process.env.BEAM_CLICKHOUSE_ENDPOINT || !process.env.BEAM_CLICKHOUSE_USERNAME || !process.env.BEAM_CLICKHOUSE_PASSWORD) {
+        throw new Error(`Failed to set ClickHouse credentials in process.env - this should never happen in Node.js`);
+    }
+}
+async function startCollectorAndWaitForReady(env, _timeoutMs) {
+    const isInDocker = process.env.IS_LOCAL !== '1' && process.env.IS_LOCAL !== 'true';
+    const useLocalOtel = !!process.env.BEAM_LOCAL_OTEL;
+    const standardOtelEnabled = (isInDocker || useLocalOtel) && !process.env.BEAM_DISABLE_STANDARD_OTEL;
+    const hasExplicitEndpoint = !!process.env.BEAM_OTEL_EXPORTER_OTLP_ENDPOINT;
+    if (!standardOtelEnabled && !hasExplicitEndpoint) {
+        return null;
+    }
+    const initLogger = (0, pino_1.default)({
+        name: 'beamable-otlp-init',
+        level: 'info',
+    }, process.stdout);
+    initLogger.info('[OTLP] Setting up collector (waiting for readiness before enabling Portal logs)...');
+    const setupStartTime = Date.now();
+    try {
+        initLogger.info('[OTLP] Step 1: Discovering or starting collector...');
+        const endpoint = await discoverOrStartCollector(initLogger, standardOtelEnabled, env);
+        const elapsed = Date.now() - setupStartTime;
+        if (endpoint) {
+            initLogger.info(`[OTLP] Collector ready at ${endpoint}. Portal logs now enabled. (took ${elapsed}ms)`);
+            return endpoint;
+        }
+        else {
+            initLogger.warn(`[OTLP] Collector setup failed, continuing without Portal logs. (took ${elapsed}ms)`);
+            return null;
+        }
+    }
+    catch (error) {
+        const elapsed = Date.now() - setupStartTime;
+        const errorMsg = error instanceof Error ? error.message : String(error);
+        initLogger.error(`[OTLP] Collector setup failed after ${elapsed}ms: ${errorMsg}`);
+        return null;
+    }
+}
+function startCollectorAsync(env) {
+    const isInDocker = process.env.IS_LOCAL !== '1' && process.env.IS_LOCAL !== 'true';
+    const useLocalOtel = !!process.env.BEAM_LOCAL_OTEL;
+    const standardOtelEnabled = (isInDocker || useLocalOtel) && !process.env.BEAM_DISABLE_STANDARD_OTEL;
+    const hasExplicitEndpoint = !!process.env.BEAM_OTEL_EXPORTER_OTLP_ENDPOINT;
+    if (!standardOtelEnabled && !hasExplicitEndpoint) {
+        return;
+    }
+    const initLogger = (0, pino_1.default)({
+        name: 'beamable-otlp-init',
+        level: 'info',
+    }, process.stdout);
+    initLogger.info('[OTLP] Starting collector setup in background (non-blocking)...');
+    discoverOrStartCollector(initLogger, standardOtelEnabled, env)
+        .then((result) => {
+        if (result) {
+            initLogger.info(`[OTLP] Collector setup complete in background, endpoint: ${result}`);
+        }
+        else {
+            initLogger.warn('[OTLP] Collector setup returned null, OTLP logging will be disabled');
+        }
+    })
+        .catch((error) => {
+        const errorMsg = error instanceof Error ? error.message : String(error);
+        initLogger.error(`[OTLP] Failed to setup collector in background: ${errorMsg}`);
+    });
+}
+function setupCollectorBeforeLogging(env, timeoutMs = 60000) {
+    const isInDocker = process.env.IS_LOCAL !== '1' && process.env.IS_LOCAL !== 'true';
+    const useLocalOtel = !!process.env.BEAM_LOCAL_OTEL;
+    const standardOtelEnabled = (isInDocker || useLocalOtel) && !process.env.BEAM_DISABLE_STANDARD_OTEL;
+    const hasExplicitEndpoint = !!process.env.BEAM_OTEL_EXPORTER_OTLP_ENDPOINT;
+    if (!standardOtelEnabled && !hasExplicitEndpoint) {
+        return null;
+    }
+    const initLogger = (0, pino_1.default)({
+        name: 'beamable-otlp-init',
+        level: 'info',
+    }, process.stdout);
+    initLogger.info('[OTLP] Setting up collector before logging initialization...');
+    let endpoint = null;
+    let completed = false;
+    let setupError = null;
+    discoverOrStartCollector(initLogger, standardOtelEnabled, env)
+        .then((result) => {
+        endpoint = result;
+        completed = true;
+        if (result) {
+            initLogger.info(`[OTLP] Collector setup complete, endpoint: ${result}`);
+        }
+        else {
+            initLogger.warn('[OTLP] Collector setup returned null, OTLP logging will be disabled');
+        }
+        return result;
+    })
+        .catch((error) => {
+        setupError = error instanceof Error ? error.message : String(error);
+        completed = true;
+        initLogger.error(`[OTLP] Failed to setup collector: ${setupError}`);
+        endpoint = null;
+    });
+    const startTime = Date.now();
+    const timeoutId = setTimeout(() => {
+        if (!completed) {
+            initLogger.warn(`[OTLP] Collector setup timeout after ${timeoutMs}ms, continuing without OTLP`);
+            completed = true;
+        }
+    }, timeoutMs);
+    try {
+        deasync_1.default.loopWhile(() => {
+            const elapsed = Date.now() - startTime;
+            if (elapsed >= timeoutMs) {
+                return false;
+            }
+            return !completed;
+        });
+    }
+    catch (error) {
+        initLogger.error(`[OTLP] Error during collector setup wait: ${error instanceof Error ? error.message : String(error)}`);
+    }
+    clearTimeout(timeoutId);
+    if (completed && endpoint) {
+        initLogger.info('[OTLP] Collector is ready, proceeding with logger creation');
+        return endpoint;
+    }
+    else if (setupError) {
+        initLogger.error(`[OTLP] Collector setup failed: ${setupError}, continuing without OTLP`);
+        return null;
+    }
+    else {
+        initLogger.warn('[OTLP] Collector setup did not complete in time, continuing without OTLP');
+        return null;
+    }
+}
+async function startCollector(logger, otlpEndpoint, env) {
+    var _a, _b;
+    logger.info('[Collector] === startCollector() ENTERED ===');
+    globalCollectorStartError = null;
+    globalCollectorInitError = null;
+    let clickhouseEndpoint = process.env.BEAM_CLICKHOUSE_ENDPOINT;
+    let clickhouseUsername = process.env.BEAM_CLICKHOUSE_USERNAME;
+    let clickhousePassword = process.env.BEAM_CLICKHOUSE_PASSWORD;
+    if ((!clickhouseEndpoint || !clickhouseUsername || !clickhousePassword) && env) {
+        try {
+            logger.info('[Collector] Fetching ClickHouse credentials from Beamable API...');
+            const credStartTime = Date.now();
+            const credentials = await fetchClickHouseCredentials(env);
+            const credElapsed = Date.now() - credStartTime;
+            logger.info(`[Collector] ClickHouse credentials fetch completed in ${credElapsed}ms`);
+            clickhouseEndpoint = credentials.endpoint;
+            clickhouseUsername = credentials.username;
+            clickhousePassword = credentials.password;
+            addAuthEnvironmentVars(clickhouseEndpoint, clickhouseUsername, clickhousePassword);
+            const verifyEndpoint = process.env.BEAM_CLICKHOUSE_ENDPOINT;
+            const verifyUsername = process.env.BEAM_CLICKHOUSE_USERNAME;
+            const verifyPassword = process.env.BEAM_CLICKHOUSE_PASSWORD;
+            if (!verifyEndpoint || !verifyUsername || !verifyPassword) {
+                logger.error(`[Collector] CRITICAL: Credentials were set but are missing from process.env! This should never happen.`);
+                throw new Error('Failed to persist ClickHouse credentials in process.env');
+            }
+            logger.info('[Collector] ClickHouse credentials fetched from API and verified in process.env');
+        }
+        catch (error) {
+            const errorMsg = `[Collector] Failed to fetch ClickHouse credentials from API: ${error instanceof Error ? error.message : String(error)}`;
+            logger.error(errorMsg);
+            throw new Error(errorMsg);
+        }
+    }
+    if (!clickhouseEndpoint || !clickhouseUsername || !clickhousePassword) {
+        const errorMsg = `[Collector] Required ClickHouse credentials are missing. Set BEAM_CLICKHOUSE_ENDPOINT, BEAM_CLICKHOUSE_USERNAME, and BEAM_CLICKHOUSE_PASSWORD, or ensure the API endpoint is accessible.`;
+        logger.error(errorMsg);
+        throw new Error(errorMsg);
+    }
+    logger.info('[Collector] Step 2: Resolving collector binary and config...');
+    const resolveStartTime = Date.now();
+    const collectorInfo = await resolveCollector(true, logger);
+    const resolveElapsed = Date.now() - resolveStartTime;
+    logger.info(`[Collector] Collector binary/config resolution completed in ${resolveElapsed}ms`);
+    if (!collectorInfo.binaryPath) {
+        logger.error('[Collector] Binary not found and download failed');
+        throw new Error('Collector binary not found and download failed');
+    }
+    if (!collectorInfo.configPath) {
+        logger.error('[Collector] Config not found and download failed');
+        throw new Error('Collector config not found and download failed');
+    }
+    logger.info(`[Collector] Using binary: ${collectorInfo.binaryPath}`);
+    logger.info(`[Collector] Using config: ${collectorInfo.configPath}`);
+    if (!collectorInfo.binaryPath || !(0, fs_1.existsSync)(collectorInfo.binaryPath)) {
+        throw new Error(`Collector binary not found at ${collectorInfo.binaryPath}`);
+    }
+    if (process.platform !== 'win32') {
+        try {
+            const fsPromises = await Promise.resolve().then(() => __importStar(require('fs/promises')));
+            const stats = await fsPromises.stat(collectorInfo.binaryPath);
+            const mode = Number(stats.mode);
+            const isExecutable = !!(mode & parseInt('111', 8));
+            if (!isExecutable) {
+                logger.warn(`[Collector] Binary may not be executable. Mode: ${mode.toString(8)}`);
+                try {
+                    (0, fs_1.chmodSync)(collectorInfo.binaryPath, 0o755);
+                    logger.info(`[Collector] Made binary executable`);
+                }
+                catch (chmodError) {
+                    logger.error(`[Collector] Failed to make binary executable: ${chmodError instanceof Error ? chmodError.message : String(chmodError)}`);
+                }
+            }
+        }
+        catch (statError) {
+            logger.warn(`[Collector] Could not check binary permissions: ${statError instanceof Error ? statError.message : String(statError)}`);
+        }
+    }
+    addCollectorConfigurationToEnvironment();
+    if (!process.env.BEAM_COLLECTOR_DISCOVERY_PORT) {
+        process.env.BEAM_COLLECTOR_DISCOVERY_PORT = String(DISCOVERY_PORT);
+        logger.info(`[Collector] Set BEAM_COLLECTOR_DISCOVERY_PORT=${DISCOVERY_PORT} (default)`);
+    }
+    let localEndpoint = otlpEndpoint;
+    if (!localEndpoint) {
+        localEndpoint = '0.0.0.0:4318';
+    }
+    localEndpoint = localEndpoint.replace(/^http:\/\//, '').replace(/^https:\/\//, '');
+    const prometheusPort = await findFreePort();
+    const collectorEnv = {
+        ...process.env,
+        BEAM_OTLP_HTTP_ENDPOINT: localEndpoint,
+        BEAM_COLLECTOR_DISCOVERY_PORT: String(DISCOVERY_PORT),
+        BEAM_COLLECTOR_PROMETHEUS_PORT: String(prometheusPort),
+    };
+    const collectorProcess = (0, child_process_1.spawn)(collectorInfo.binaryPath, ['--config', collectorInfo.configPath], {
+        env: collectorEnv,
+        stdio: ['ignore', 'pipe', 'pipe'],
+        detached: false,
+    });
+    globalCollectorProcess = collectorProcess;
+    globalCollectorStartError = null;
+    globalCollectorExitCode = null;
+    globalCollectorStderr = [];
+    (_a = collectorProcess.stdout) === null || _a === void 0 ? void 0 : _a.on('data', (data) => {
+        const outputLine = data.toString().trim();
+        logger.info(`[Collector OUT] ${outputLine}`);
+    });
+    (_b = collectorProcess.stderr) === null || _b === void 0 ? void 0 : _b.on('data', (data) => {
+        const errorLine = data.toString().trim();
+        globalCollectorStderr.push(errorLine);
+        if (globalCollectorStderr.length > 50) {
+            globalCollectorStderr.shift();
+        }
+        logger.info(`[Collector ERR] ${errorLine}`);
+    });
+    collectorProcess.on('error', (err) => {
+        globalCollectorStartError = err.message;
+        logger.error(`[Collector] Failed to start: ${err.message}`);
+    });
+    collectorProcess.on('exit', (code, signal) => {
+        globalCollectorExitCode = code;
+        logger.warn(`[Collector] Process exited with code ${code}${signal ? `, signal ${signal}` : ''}`);
+        if (code !== 0 && globalCollectorStderr.length > 0) {
+            logger.error(`[Collector] Exit error - Last ${Math.min(10, globalCollectorStderr.length)} stderr lines:`);
+            globalCollectorStderr.slice(-10).forEach((line) => {
+                logger.error(`[Collector ERR] ${line}`);
+            });
+        }
+    });
+    logger.info(`[Collector] Started with PID ${collectorProcess.pid}, endpoint: ${localEndpoint}`);
+    await new Promise(resolve => setTimeout(resolve, 500));
+    if (globalCollectorExitCode !== null && globalCollectorExitCode !== 0) {
+        const stderrMsg = globalCollectorStderr.length > 0
+            ? ` Stderr: ${globalCollectorStderr.join('; ')}`
+            : ' No stderr output captured.';
+        throw new Error(`Collector process exited immediately after startup with code ${globalCollectorExitCode}.${stderrMsg}`);
+    }
+    return {
+        process: collectorProcess,
+        endpoint: `http://${localEndpoint}`,
+    };
+}
+function getCollectorProcessStatus() {
+    var _a;
+    const processAlive = globalCollectorProcess !== null &&
+        globalCollectorProcess.exitCode === null &&
+        globalCollectorProcess.killed === false;
+    const stderrText = globalCollectorStderr.join('\n');
+    const isReady = stderrText.includes('Everything is ready') ||
+        stderrText.includes('Begin running and processing data');
+    const actualInitError = (processAlive && isReady) ? null : globalCollectorInitError;
+    const basePath = getCollectorStoragePath();
+    const binaryName = getCollectorBinaryName();
+    const configName = 'clickhouse-config.yaml';
+    const binaryPath = (0, path_1.join)(basePath, binaryName);
+    const configPath = (0, path_1.join)(basePath, configName);
+    const binaryExists = (0, fs_1.existsSync)(binaryPath);
+    const configExists = (0, fs_1.existsSync)(configPath);
+    const isPreInstalled = binaryExists && configExists;
+    let diagnosticInitError = actualInitError;
+    if (!isPreInstalled && actualInitError && actualInitError.includes('DNS resolution failed')) {
+        const baseDirExists = (0, fs_1.existsSync)(basePath);
+        diagnosticInitError = `${actualInitError} (Pre-installed collector not found: basePath=${basePath}, binaryExists=${binaryExists}, configExists=${configExists}, baseDirExists=${baseDirExists}. This suggests the Docker image was not built with the pre-installed collector, or the files are in a different location.)`;
+    }
+    return {
+        hasProcess: globalCollectorProcess !== null,
+        pid: (_a = globalCollectorProcess === null || globalCollectorProcess === void 0 ? void 0 : globalCollectorProcess.pid) !== null && _a !== void 0 ? _a : null,
+        exitCode: globalCollectorExitCode,
+        startError: globalCollectorStartError,
+        initError: diagnosticInitError,
+        stderr: [...globalCollectorStderr],
+        isPreInstalled,
+        binaryPath: isPreInstalled ? binaryPath : null,
+        configPath: isPreInstalled ? configPath : null,
+    };
+}
+async function discoverOrStartCollector(logger, standardOtelEnabled, env) {
+    if (!standardOtelEnabled) {
+        return null;
+    }
+    if (globalCollectorStartupPromise) {
+        logger.info('[Collector] Collector startup already in progress, waiting for existing startup to complete...');
+        try {
+            const result = await globalCollectorStartupPromise;
+            globalCollectorStartupPromise = null;
+            return result;
+        }
+        catch (error) {
+            logger.error(`[Collector] Existing startup promise failed: ${error instanceof Error ? error.message : String(error)}`);
+            globalCollectorStartupPromise = null;
+        }
+    }
+    let existingEndpoint;
+    if (globalCollectorProcess) {
+        const processAlive = globalCollectorProcess.exitCode === null &&
+            globalCollectorProcess.killed === false;
+        if (processAlive) {
+            logger.info(`[Collector] Collector process already exists (PID ${globalCollectorProcess.pid}), waiting for it to be ready...`);
+            existingEndpoint = process.env.BEAM_OTLP_HTTP_ENDPOINT ?
+                `http://${process.env.BEAM_OTLP_HTTP_ENDPOINT}` :
+                'http://0.0.0.0:4318';
+        }
+        else {
+            logger.warn(`[Collector] Previous collector process (PID ${globalCollectorProcess.pid}) is dead, starting new one...`);
+            globalCollectorProcess = null;
+            globalCollectorExitCode = null;
+            globalCollectorStderr = [];
+        }
+    }
+    if (!globalCollectorProcess) {
+        logger.info('[Collector] Quick check for existing collector via UDP discovery (timeout: 1s, matching C#)...');
+        const udpStartTime = Date.now();
+        logger.info('[Collector] Calling isCollectorRunning()...');
+        const status = await isCollectorRunning();
+        const udpElapsed = Date.now() - udpStartTime;
+        logger.info(`[Collector] isCollectorRunning() returned after ${udpElapsed}ms: isRunning=${status.isRunning}, isReady=${status.isReady}, endpoint=${status.otlpEndpoint || 'none'}`);
+        if (status.isRunning && status.isReady && status.otlpEndpoint) {
+            logger.info(`[Collector] Found running collector at ${status.otlpEndpoint}, will reuse it`);
+            return `http://${status.otlpEndpoint}`;
+        }
+        logger.info('[Collector] No existing collector found, will start new one...');
+    }
+    else {
+        logger.info('[Collector] Collector process already exists, skipping UDP discovery...');
+    }
+    logger.info('[Collector] Creating startup promise...');
+    const startupPromise = (async () => {
+        logger.info('[Collector] Startup promise executing...');
+        try {
+            globalCollectorInitError = null;
+            logger.info(`[Collector] Checking existingEndpoint: ${existingEndpoint || 'none'}`);
+            let endpoint;
+            if (existingEndpoint) {
+                endpoint = existingEndpoint;
+                logger.info(`[Collector] Waiting for existing collector to become ready at ${endpoint}...`);
+            }
+            else {
+                logger.info('[Collector] Starting new OpenTelemetry collector...');
+                const startCollectorTime = Date.now();
+                const startResult = await startCollector(logger, undefined, env);
+                const startCollectorElapsed = Date.now() - startCollectorTime;
+                logger.info(`[Collector] Collector process started in ${startCollectorElapsed}ms, endpoint: ${startResult.endpoint}`);
+                endpoint = startResult.endpoint;
+                logger.info('[Collector] Checking if collector process is still running...');
+                await new Promise(resolve => setTimeout(resolve, 200));
+                if (globalCollectorExitCode !== null && globalCollectorExitCode !== 0) {
+                    const errorMsg = `Collector process exited immediately with code ${globalCollectorExitCode}. ${globalCollectorStderr.length > 0 ? `Stderr: ${globalCollectorStderr.join('; ')}` : 'No stderr output.'}`;
+                    globalCollectorInitError = errorMsg;
+                    logger.error(`[Collector] ${errorMsg}`);
+                    return null;
+                }
+                logger.info('[Collector] Collector process is still running, proceeding to readiness check...');
+            }
+            const maxWaitTime = 60000;
+            const checkInterval = 500;
+            const maxChecks = Math.floor(maxWaitTime / checkInterval);
+            logger.info(`[Collector] Waiting for collector to become ready (checking every ${checkInterval}ms, max ${maxWaitTime / 1000}s)...`);
+            const readinessStartTime = Date.now();
+            for (let i = 0; i < maxChecks; i++) {
+                await new Promise(resolve => setTimeout(resolve, checkInterval));
+                const elapsed = Date.now() - readinessStartTime;
+                if (globalCollectorExitCode !== null && globalCollectorExitCode !== 0) {
+                    const errorMsg = `Collector process exited during startup with code ${globalCollectorExitCode}. ${globalCollectorStderr.length > 0 ? `Stderr: ${globalCollectorStderr.join('; ')}` : 'No stderr output.'}`;
+                    globalCollectorInitError = errorMsg;
+                    logger.error(`[Collector] ${errorMsg} (after ${elapsed}ms)`);
+                    return null;
+                }
+                logger.info(`[Collector] Checking collector readiness... (${(elapsed / 1000).toFixed(1)}s elapsed, attempt ${i + 1}/${maxChecks})`);
+                const newStatus = await isCollectorRunning();
+                logger.info(`[Collector] Collector status: isRunning=${newStatus.isRunning}, isReady=${newStatus.isReady}, pid=${newStatus.pid}, endpoint=${newStatus.otlpEndpoint || 'none'}`);
+                if (newStatus.isRunning && newStatus.isReady) {
+                    logger.info(`[Collector] Collector is ready at ${newStatus.otlpEndpoint || endpoint} (ready after ${elapsed}ms)`);
+                    return newStatus.otlpEndpoint ? `http://${newStatus.otlpEndpoint}` : endpoint;
+                }
+                if (i > 0 && i % 2 === 0) {
+                    logger.info(`[Collector] Still waiting for collector to become ready... (${(elapsed / 1000).toFixed(1)}s elapsed)`);
+                }
+            }
+            if (globalCollectorExitCode !== null && globalCollectorExitCode !== 0) {
+                const errorMsg = `Collector process exited with code ${globalCollectorExitCode}. ${globalCollectorStderr.length > 0 ? `Stderr: ${globalCollectorStderr.join('; ')}` : 'No stderr output.'}`;
+                globalCollectorInitError = errorMsg;
+                logger.error(`[Collector] ${errorMsg}`);
+                return null;
+            }
+            logger.error(`[Collector] Collector did not become ready within ${maxWaitTime / 1000} seconds`);
+            return null;
+        }
+        catch (err) {
+            const errorMsg = err instanceof Error ? err.message : String(err);
+            globalCollectorInitError = errorMsg;
+            logger.error(`[Collector] Failed to start collector: ${errorMsg}`);
+            return null;
+        }
+    })();
+    globalCollectorStartupPromise = startupPromise;
+    startupPromise.finally(() => {
+        globalCollectorStartupPromise = null;
+    });
+    return await startupPromise;
+}