@omen.foundation/node-microservice-runtime 0.1.56 → 0.1.57
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -107,30 +107,42 @@ async function fetchClickHouseCredentials(env) {
|
|
|
107
107
|
const apiUrl = (0, urls_js_1.hostToHttpUrl)(env.host);
|
|
108
108
|
const uriPath = '/api/beamo/otel/auth/writer/config';
|
|
109
109
|
const configUrl = new URL(uriPath, apiUrl).toString();
|
|
110
|
+
const verboseLogger = (0, pino_1.default)({ name: 'fetch-credentials', level: 'info' }, process.stdout);
|
|
111
|
+
verboseLogger.info(`[Credentials] Starting credential fetch from ${configUrl}...`);
|
|
110
112
|
const secret = process.env.SECRET;
|
|
111
113
|
if (!secret) {
|
|
114
|
+
verboseLogger.error('[Credentials] SECRET environment variable is missing');
|
|
112
115
|
throw new Error('SECRET environment variable is required to fetch ClickHouse credentials');
|
|
113
116
|
}
|
|
117
|
+
verboseLogger.info('[Credentials] SECRET found, calculating signature...');
|
|
114
118
|
const pathAndQuery = getPathAndQuery(uriPath);
|
|
115
119
|
const signature = calculateSignature(env.pid, secret, pathAndQuery, null, '1');
|
|
120
|
+
verboseLogger.info('[Credentials] Signature calculated, making API request...');
|
|
116
121
|
const headers = {
|
|
117
122
|
'Content-Type': 'application/json',
|
|
118
123
|
Accept: 'application/json',
|
|
119
124
|
'X-BEAM-SCOPE': `${env.cid}.${env.pid}`,
|
|
120
125
|
'X-BEAM-SIGNATURE': signature,
|
|
121
126
|
};
|
|
127
|
+
const fetchStartTime = Date.now();
|
|
122
128
|
const response = await fetch(configUrl, {
|
|
123
129
|
method: 'GET',
|
|
124
130
|
headers,
|
|
125
131
|
});
|
|
132
|
+
const fetchElapsed = Date.now() - fetchStartTime;
|
|
133
|
+
verboseLogger.info(`[Credentials] API request completed in ${fetchElapsed}ms, status: ${response.status}`);
|
|
126
134
|
if (!response.ok) {
|
|
127
135
|
const errorText = await response.text().catch(() => 'Unknown error');
|
|
136
|
+
verboseLogger.error(`[Credentials] API request failed: ${response.status} ${response.statusText}`);
|
|
128
137
|
throw new Error(`Failed to fetch ClickHouse credentials from ${configUrl}: ${response.status} ${response.statusText} - ${errorText.substring(0, 200)}`);
|
|
129
138
|
}
|
|
139
|
+
verboseLogger.info('[Credentials] Parsing response JSON...');
|
|
130
140
|
const credentials = await response.json();
|
|
131
141
|
if (!credentials.endpoint || !credentials.username || !credentials.password) {
|
|
142
|
+
verboseLogger.error('[Credentials] Invalid response: missing required fields');
|
|
132
143
|
throw new Error('Invalid ClickHouse credentials response: missing required fields');
|
|
133
144
|
}
|
|
145
|
+
verboseLogger.info(`[Credentials] Successfully fetched credentials (endpoint: ${credentials.endpoint}, username: ${credentials.username})`);
|
|
134
146
|
return credentials;
|
|
135
147
|
}
|
|
136
148
|
function getCollectorStoragePath() {
|
|
@@ -442,20 +454,24 @@ async function startCollectorAndWaitForReady(env, _timeoutMs) {
|
|
|
442
454
|
level: 'info',
|
|
443
455
|
}, process.stdout);
|
|
444
456
|
initLogger.info('[OTLP] Setting up collector (waiting for readiness before enabling Portal logs)...');
|
|
457
|
+
const setupStartTime = Date.now();
|
|
445
458
|
try {
|
|
459
|
+
initLogger.info('[OTLP] Step 1: Discovering or starting collector...');
|
|
446
460
|
const endpoint = await discoverOrStartCollector(initLogger, standardOtelEnabled, env);
|
|
461
|
+
const elapsed = Date.now() - setupStartTime;
|
|
447
462
|
if (endpoint) {
|
|
448
|
-
initLogger.info(`[OTLP] Collector ready at ${endpoint}. Portal logs now enabled
|
|
463
|
+
initLogger.info(`[OTLP] Collector ready at ${endpoint}. Portal logs now enabled. (took ${elapsed}ms)`);
|
|
449
464
|
return endpoint;
|
|
450
465
|
}
|
|
451
466
|
else {
|
|
452
|
-
initLogger.warn(
|
|
467
|
+
initLogger.warn(`[OTLP] Collector setup failed, continuing without Portal logs. (took ${elapsed}ms)`);
|
|
453
468
|
return null;
|
|
454
469
|
}
|
|
455
470
|
}
|
|
456
471
|
catch (error) {
|
|
472
|
+
const elapsed = Date.now() - setupStartTime;
|
|
457
473
|
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
458
|
-
initLogger.error(`[OTLP] Collector setup failed: ${errorMsg}`);
|
|
474
|
+
initLogger.error(`[OTLP] Collector setup failed after ${elapsed}ms: ${errorMsg}`);
|
|
459
475
|
return null;
|
|
460
476
|
}
|
|
461
477
|
}
|
|
@@ -563,7 +579,10 @@ async function startCollector(logger, otlpEndpoint, env) {
|
|
|
563
579
|
if ((!clickhouseEndpoint || !clickhouseUsername || !clickhousePassword) && env) {
|
|
564
580
|
try {
|
|
565
581
|
logger.info('[Collector] Fetching ClickHouse credentials from Beamable API...');
|
|
582
|
+
const credStartTime = Date.now();
|
|
566
583
|
const credentials = await fetchClickHouseCredentials(env);
|
|
584
|
+
const credElapsed = Date.now() - credStartTime;
|
|
585
|
+
logger.info(`[Collector] ClickHouse credentials fetch completed in ${credElapsed}ms`);
|
|
567
586
|
clickhouseEndpoint = credentials.endpoint;
|
|
568
587
|
clickhouseUsername = credentials.username;
|
|
569
588
|
clickhousePassword = credentials.password;
|
|
@@ -588,8 +607,11 @@ async function startCollector(logger, otlpEndpoint, env) {
|
|
|
588
607
|
logger.error(errorMsg);
|
|
589
608
|
throw new Error(errorMsg);
|
|
590
609
|
}
|
|
591
|
-
logger.info('[Collector] Resolving collector binary and config...');
|
|
610
|
+
logger.info('[Collector] Step 2: Resolving collector binary and config...');
|
|
611
|
+
const resolveStartTime = Date.now();
|
|
592
612
|
const collectorInfo = await resolveCollector(true, logger);
|
|
613
|
+
const resolveElapsed = Date.now() - resolveStartTime;
|
|
614
|
+
logger.info(`[Collector] Collector binary/config resolution completed in ${resolveElapsed}ms`);
|
|
593
615
|
if (!collectorInfo.binaryPath) {
|
|
594
616
|
logger.error('[Collector] Binary not found and download failed');
|
|
595
617
|
throw new Error('Collector binary not found and download failed');
|
|
@@ -734,11 +756,14 @@ async function discoverOrStartCollector(logger, standardOtelEnabled, env) {
|
|
|
734
756
|
}
|
|
735
757
|
}
|
|
736
758
|
if (!globalCollectorProcess) {
|
|
759
|
+
logger.info('[Collector] Checking for existing collector via UDP discovery...');
|
|
737
760
|
const status = await isCollectorRunning();
|
|
761
|
+
logger.info(`[Collector] UDP discovery result: isRunning=${status.isRunning}, isReady=${status.isReady}, endpoint=${status.otlpEndpoint || 'none'}`);
|
|
738
762
|
if (status.isRunning && status.isReady && status.otlpEndpoint) {
|
|
739
763
|
logger.info(`[Collector] Found running collector at ${status.otlpEndpoint}`);
|
|
740
764
|
return `http://${status.otlpEndpoint}`;
|
|
741
765
|
}
|
|
766
|
+
logger.info('[Collector] No existing collector found, will start new one...');
|
|
742
767
|
}
|
|
743
768
|
const startupPromise = (async () => {
|
|
744
769
|
try {
|
|
@@ -749,9 +774,13 @@ async function discoverOrStartCollector(logger, standardOtelEnabled, env) {
|
|
|
749
774
|
logger.info(`[Collector] Waiting for existing collector to become ready at ${endpoint}...`);
|
|
750
775
|
}
|
|
751
776
|
else {
|
|
752
|
-
logger.info('[Collector] Starting OpenTelemetry collector...');
|
|
777
|
+
logger.info('[Collector] Starting new OpenTelemetry collector...');
|
|
778
|
+
const startCollectorTime = Date.now();
|
|
753
779
|
const startResult = await startCollector(logger, undefined, env);
|
|
780
|
+
const startCollectorElapsed = Date.now() - startCollectorTime;
|
|
781
|
+
logger.info(`[Collector] Collector process started in ${startCollectorElapsed}ms, endpoint: ${startResult.endpoint}`);
|
|
754
782
|
endpoint = startResult.endpoint;
|
|
783
|
+
logger.info('[Collector] Checking if collector process is still running...');
|
|
755
784
|
await new Promise(resolve => setTimeout(resolve, 200));
|
|
756
785
|
if (globalCollectorExitCode !== null && globalCollectorExitCode !== 0) {
|
|
757
786
|
const errorMsg = `Collector process exited immediately with code ${globalCollectorExitCode}. ${globalCollectorStderr.length > 0 ? `Stderr: ${globalCollectorStderr.join('; ')}` : 'No stderr output.'}`;
|
|
@@ -759,26 +788,31 @@ async function discoverOrStartCollector(logger, standardOtelEnabled, env) {
|
|
|
759
788
|
logger.error(`[Collector] ${errorMsg}`);
|
|
760
789
|
return null;
|
|
761
790
|
}
|
|
791
|
+
logger.info('[Collector] Collector process is still running, proceeding to readiness check...');
|
|
762
792
|
}
|
|
763
793
|
const maxWaitTime = 60000;
|
|
764
794
|
const checkInterval = 500;
|
|
765
795
|
const maxChecks = Math.floor(maxWaitTime / checkInterval);
|
|
766
|
-
logger.info(
|
|
796
|
+
logger.info(`[Collector] Waiting for collector to become ready (checking every ${checkInterval}ms, max ${maxWaitTime / 1000}s)...`);
|
|
797
|
+
const readinessStartTime = Date.now();
|
|
767
798
|
for (let i = 0; i < maxChecks; i++) {
|
|
768
799
|
await new Promise(resolve => setTimeout(resolve, checkInterval));
|
|
800
|
+
const elapsed = Date.now() - readinessStartTime;
|
|
769
801
|
if (globalCollectorExitCode !== null && globalCollectorExitCode !== 0) {
|
|
770
802
|
const errorMsg = `Collector process exited during startup with code ${globalCollectorExitCode}. ${globalCollectorStderr.length > 0 ? `Stderr: ${globalCollectorStderr.join('; ')}` : 'No stderr output.'}`;
|
|
771
803
|
globalCollectorInitError = errorMsg;
|
|
772
|
-
logger.error(`[Collector] ${errorMsg}`);
|
|
804
|
+
logger.error(`[Collector] ${errorMsg} (after ${elapsed}ms)`);
|
|
773
805
|
return null;
|
|
774
806
|
}
|
|
807
|
+
logger.info(`[Collector] Checking collector readiness... (${(elapsed / 1000).toFixed(1)}s elapsed, attempt ${i + 1}/${maxChecks})`);
|
|
775
808
|
const newStatus = await isCollectorRunning();
|
|
809
|
+
logger.info(`[Collector] Collector status: isRunning=${newStatus.isRunning}, isReady=${newStatus.isReady}, pid=${newStatus.pid}, endpoint=${newStatus.otlpEndpoint || 'none'}`);
|
|
776
810
|
if (newStatus.isRunning && newStatus.isReady) {
|
|
777
|
-
logger.info(`[Collector] Collector is ready at ${newStatus.otlpEndpoint || endpoint}`);
|
|
811
|
+
logger.info(`[Collector] Collector is ready at ${newStatus.otlpEndpoint || endpoint} (ready after ${elapsed}ms)`);
|
|
778
812
|
return newStatus.otlpEndpoint ? `http://${newStatus.otlpEndpoint}` : endpoint;
|
|
779
813
|
}
|
|
780
|
-
if (i > 0 && i %
|
|
781
|
-
logger.info(`[Collector] Still waiting for collector to become ready... (${(
|
|
814
|
+
if (i > 0 && i % 2 === 0) {
|
|
815
|
+
logger.info(`[Collector] Still waiting for collector to become ready... (${(elapsed / 1000).toFixed(1)}s elapsed)`);
|
|
782
816
|
}
|
|
783
817
|
}
|
|
784
818
|
if (globalCollectorExitCode !== null && globalCollectorExitCode !== 0) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"collector-manager.d.ts","sourceRoot":"","sources":["../src/collector-manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAS,YAAY,EAAE,MAAM,eAAe,CAAC;AAOpD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAInC,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAYpD,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAuCD;;GAEG;AACH,UAAU,qBAAqB;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAqCD;;;;GAIG;AACH,wBAAsB,0BAA0B,CAC9C,GAAG,EAAE,iBAAiB,GACrB,OAAO,CAAC,qBAAqB,CAAC,
|
|
1
|
+
{"version":3,"file":"collector-manager.d.ts","sourceRoot":"","sources":["../src/collector-manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAS,YAAY,EAAE,MAAM,eAAe,CAAC;AAOpD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAInC,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAYpD,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAuCD;;GAEG;AACH,UAAU,qBAAqB;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAqCD;;;;GAIG;AACH,wBAAsB,0BAA0B,CAC9C,GAAG,EAAE,iBAAiB,GACrB,OAAO,CAAC,qBAAqB,CAAC,CAsDhC;AAiPD;;GAEG;AACH,wBAAgB,8BAA8B,IAAI;IAChD,WAAW,EAAE,OAAO,CAAC;IACrB,WAAW,EAAE,OAAO,CAAC;IACrB,WAAW,EAAE,OAAO,CAAC;IACrB,MAAM,EAAE,aAAa,GAAG,KAAK,GAAG,SAAS,CAAC;CAC3C,CAoBA;AAED;;GAEG;AACH,wBAAsB,kBAAkB,IAAI,OAAO,CAAC,eAAe,CAAC,CAyEnE;AA0BD;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI,CAYjG;AAED;;;;GAIG;AACH,wBAAsB,6BAA6B,CACjD,GAAG,EAAE,iBAAiB,EACtB,UAAU,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAwCxB;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,iBAAiB,GAAG,IAAI,CAsChE;AAED;;;;;GAKG;AACH,wBAAgB,2BAA2B,CACzC,GAAG,EAAE,iBAAiB,EACtB,SAAS,GAAE,MAAc,GACxB,MAAM,GAAG,IAAI,CA8Ef;AAED;;GAEG;AACH,wBAAsB,cAAc,CAClC,MAAM,EAAE,MAAM,EACd,YAAY,CAAC,EAAE,MAAM,EACrB,GAAG,CAAC,EAAE,iBAAiB,GACtB,OAAO,CAAC;IAAE,OAAO,EAAE,YAAY,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC,CAsNtD;AAED;;GAEG;AACH,wBAAgB,yBAAyB,IAAI;IAC3C,UAAU,EAAE,OAAO,CAAC;IACpB,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACnB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,CASA;AAED;;GAEG;AACH,wBAAsB,wBAAwB,CAC5C,MAAM,EAAE,MAAM,EACd,mBAAmB,EAAE,OAAO,EAC5B,GAAG,CAAC,EAAE,iBAAiB,GACtB,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CA+JxB"}
|
|
@@ -80,14 +80,20 @@ export async function fetchClickHouseCredentials(env) {
|
|
|
80
80
|
const apiUrl = hostToHttpUrl(env.host);
|
|
81
81
|
const uriPath = '/api/beamo/otel/auth/writer/config';
|
|
82
82
|
const configUrl = new URL(uriPath, apiUrl).toString();
|
|
83
|
+
// Create a minimal logger for verbose output
|
|
84
|
+
const verboseLogger = pino({ name: 'fetch-credentials', level: 'info' }, process.stdout);
|
|
85
|
+
verboseLogger.info(`[Credentials] Starting credential fetch from ${configUrl}...`);
|
|
83
86
|
// Get secret from environment (required for signed requests)
|
|
84
87
|
const secret = process.env.SECRET;
|
|
85
88
|
if (!secret) {
|
|
89
|
+
verboseLogger.error('[Credentials] SECRET environment variable is missing');
|
|
86
90
|
throw new Error('SECRET environment variable is required to fetch ClickHouse credentials');
|
|
87
91
|
}
|
|
92
|
+
verboseLogger.info('[Credentials] SECRET found, calculating signature...');
|
|
88
93
|
// Calculate signature for signed request (matching C# HttpSignedRequester)
|
|
89
94
|
const pathAndQuery = getPathAndQuery(uriPath);
|
|
90
95
|
const signature = calculateSignature(env.pid, secret, pathAndQuery, null, '1');
|
|
96
|
+
verboseLogger.info('[Credentials] Signature calculated, making API request...');
|
|
91
97
|
// Build headers with signed authentication (matching C# HttpSignedRequester)
|
|
92
98
|
const headers = {
|
|
93
99
|
'Content-Type': 'application/json',
|
|
@@ -95,18 +101,25 @@ export async function fetchClickHouseCredentials(env) {
|
|
|
95
101
|
'X-BEAM-SCOPE': `${env.cid}.${env.pid}`,
|
|
96
102
|
'X-BEAM-SIGNATURE': signature,
|
|
97
103
|
};
|
|
104
|
+
const fetchStartTime = Date.now();
|
|
98
105
|
const response = await fetch(configUrl, {
|
|
99
106
|
method: 'GET',
|
|
100
107
|
headers,
|
|
101
108
|
});
|
|
109
|
+
const fetchElapsed = Date.now() - fetchStartTime;
|
|
110
|
+
verboseLogger.info(`[Credentials] API request completed in ${fetchElapsed}ms, status: ${response.status}`);
|
|
102
111
|
if (!response.ok) {
|
|
103
112
|
const errorText = await response.text().catch(() => 'Unknown error');
|
|
113
|
+
verboseLogger.error(`[Credentials] API request failed: ${response.status} ${response.statusText}`);
|
|
104
114
|
throw new Error(`Failed to fetch ClickHouse credentials from ${configUrl}: ${response.status} ${response.statusText} - ${errorText.substring(0, 200)}`);
|
|
105
115
|
}
|
|
116
|
+
verboseLogger.info('[Credentials] Parsing response JSON...');
|
|
106
117
|
const credentials = await response.json();
|
|
107
118
|
if (!credentials.endpoint || !credentials.username || !credentials.password) {
|
|
119
|
+
verboseLogger.error('[Credentials] Invalid response: missing required fields');
|
|
108
120
|
throw new Error('Invalid ClickHouse credentials response: missing required fields');
|
|
109
121
|
}
|
|
122
|
+
verboseLogger.info(`[Credentials] Successfully fetched credentials (endpoint: ${credentials.endpoint}, username: ${credentials.username})`);
|
|
110
123
|
return credentials;
|
|
111
124
|
}
|
|
112
125
|
/**
|
|
@@ -489,21 +502,25 @@ export async function startCollectorAndWaitForReady(env, _timeoutMs // Not used
|
|
|
489
502
|
level: 'info',
|
|
490
503
|
}, process.stdout);
|
|
491
504
|
initLogger.info('[OTLP] Setting up collector (waiting for readiness before enabling Portal logs)...');
|
|
505
|
+
const setupStartTime = Date.now();
|
|
492
506
|
// Simple linear async/await - no timeouts, no Promise.race complexity
|
|
493
507
|
try {
|
|
508
|
+
initLogger.info('[OTLP] Step 1: Discovering or starting collector...');
|
|
494
509
|
const endpoint = await discoverOrStartCollector(initLogger, standardOtelEnabled, env);
|
|
510
|
+
const elapsed = Date.now() - setupStartTime;
|
|
495
511
|
if (endpoint) {
|
|
496
|
-
initLogger.info(`[OTLP] Collector ready at ${endpoint}. Portal logs now enabled
|
|
512
|
+
initLogger.info(`[OTLP] Collector ready at ${endpoint}. Portal logs now enabled. (took ${elapsed}ms)`);
|
|
497
513
|
return endpoint;
|
|
498
514
|
}
|
|
499
515
|
else {
|
|
500
|
-
initLogger.warn(
|
|
516
|
+
initLogger.warn(`[OTLP] Collector setup failed, continuing without Portal logs. (took ${elapsed}ms)`);
|
|
501
517
|
return null;
|
|
502
518
|
}
|
|
503
519
|
}
|
|
504
520
|
catch (error) {
|
|
521
|
+
const elapsed = Date.now() - setupStartTime;
|
|
505
522
|
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
506
|
-
initLogger.error(`[OTLP] Collector setup failed: ${errorMsg}`);
|
|
523
|
+
initLogger.error(`[OTLP] Collector setup failed after ${elapsed}ms: ${errorMsg}`);
|
|
507
524
|
return null;
|
|
508
525
|
}
|
|
509
526
|
}
|
|
@@ -645,7 +662,10 @@ export async function startCollector(logger, otlpEndpoint, env) {
|
|
|
645
662
|
if ((!clickhouseEndpoint || !clickhouseUsername || !clickhousePassword) && env) {
|
|
646
663
|
try {
|
|
647
664
|
logger.info('[Collector] Fetching ClickHouse credentials from Beamable API...');
|
|
665
|
+
const credStartTime = Date.now();
|
|
648
666
|
const credentials = await fetchClickHouseCredentials(env);
|
|
667
|
+
const credElapsed = Date.now() - credStartTime;
|
|
668
|
+
logger.info(`[Collector] ClickHouse credentials fetch completed in ${credElapsed}ms`);
|
|
649
669
|
clickhouseEndpoint = credentials.endpoint;
|
|
650
670
|
clickhouseUsername = credentials.username;
|
|
651
671
|
clickhousePassword = credentials.password;
|
|
@@ -677,8 +697,11 @@ export async function startCollector(logger, otlpEndpoint, env) {
|
|
|
677
697
|
throw new Error(errorMsg);
|
|
678
698
|
}
|
|
679
699
|
// Now resolve collector binary and config (after credentials are fetched and set)
|
|
680
|
-
logger.info('[Collector] Resolving collector binary and config...');
|
|
700
|
+
logger.info('[Collector] Step 2: Resolving collector binary and config...');
|
|
701
|
+
const resolveStartTime = Date.now();
|
|
681
702
|
const collectorInfo = await resolveCollector(true, logger);
|
|
703
|
+
const resolveElapsed = Date.now() - resolveStartTime;
|
|
704
|
+
logger.info(`[Collector] Collector binary/config resolution completed in ${resolveElapsed}ms`);
|
|
682
705
|
if (!collectorInfo.binaryPath) {
|
|
683
706
|
logger.error('[Collector] Binary not found and download failed');
|
|
684
707
|
throw new Error('Collector binary not found and download failed');
|
|
@@ -876,11 +899,14 @@ export async function discoverOrStartCollector(logger, standardOtelEnabled, env)
|
|
|
876
899
|
}
|
|
877
900
|
// First, check if collector is already running (via UDP discovery)
|
|
878
901
|
if (!globalCollectorProcess) {
|
|
902
|
+
logger.info('[Collector] Checking for existing collector via UDP discovery...');
|
|
879
903
|
const status = await isCollectorRunning();
|
|
904
|
+
logger.info(`[Collector] UDP discovery result: isRunning=${status.isRunning}, isReady=${status.isReady}, endpoint=${status.otlpEndpoint || 'none'}`);
|
|
880
905
|
if (status.isRunning && status.isReady && status.otlpEndpoint) {
|
|
881
906
|
logger.info(`[Collector] Found running collector at ${status.otlpEndpoint}`);
|
|
882
907
|
return `http://${status.otlpEndpoint}`;
|
|
883
908
|
}
|
|
909
|
+
logger.info('[Collector] No existing collector found, will start new one...');
|
|
884
910
|
}
|
|
885
911
|
// Collector not running - start it (or wait for existing one to be ready)
|
|
886
912
|
// Wrap the entire startup logic in a promise that we track globally
|
|
@@ -897,11 +923,14 @@ export async function discoverOrStartCollector(logger, standardOtelEnabled, env)
|
|
|
897
923
|
}
|
|
898
924
|
else {
|
|
899
925
|
// Start a new collector
|
|
900
|
-
logger.info('[Collector] Starting OpenTelemetry collector...');
|
|
926
|
+
logger.info('[Collector] Starting new OpenTelemetry collector...');
|
|
927
|
+
const startCollectorTime = Date.now();
|
|
901
928
|
const startResult = await startCollector(logger, undefined, env);
|
|
929
|
+
const startCollectorElapsed = Date.now() - startCollectorTime;
|
|
930
|
+
logger.info(`[Collector] Collector process started in ${startCollectorElapsed}ms, endpoint: ${startResult.endpoint}`);
|
|
902
931
|
endpoint = startResult.endpoint;
|
|
903
932
|
// Check if collector process exited immediately (crashed)
|
|
904
|
-
|
|
933
|
+
logger.info('[Collector] Checking if collector process is still running...');
|
|
905
934
|
await new Promise(resolve => setTimeout(resolve, 200));
|
|
906
935
|
if (globalCollectorExitCode !== null && globalCollectorExitCode !== 0) {
|
|
907
936
|
const errorMsg = `Collector process exited immediately with code ${globalCollectorExitCode}. ${globalCollectorStderr.length > 0 ? `Stderr: ${globalCollectorStderr.join('; ')}` : 'No stderr output.'}`;
|
|
@@ -909,6 +938,7 @@ export async function discoverOrStartCollector(logger, standardOtelEnabled, env)
|
|
|
909
938
|
logger.error(`[Collector] ${errorMsg}`);
|
|
910
939
|
return null;
|
|
911
940
|
}
|
|
941
|
+
logger.info('[Collector] Collector process is still running, proceeding to readiness check...');
|
|
912
942
|
}
|
|
913
943
|
// CRITICAL: Wait for collector to be fully ready before returning
|
|
914
944
|
// We'll wait up to 60 seconds, checking every 500ms
|
|
@@ -916,24 +946,28 @@ export async function discoverOrStartCollector(logger, standardOtelEnabled, env)
|
|
|
916
946
|
const maxWaitTime = 60000; // 60 seconds
|
|
917
947
|
const checkInterval = 500; // Check every 500ms
|
|
918
948
|
const maxChecks = Math.floor(maxWaitTime / checkInterval);
|
|
919
|
-
logger.info(
|
|
949
|
+
logger.info(`[Collector] Waiting for collector to become ready (checking every ${checkInterval}ms, max ${maxWaitTime / 1000}s)...`);
|
|
950
|
+
const readinessStartTime = Date.now();
|
|
920
951
|
for (let i = 0; i < maxChecks; i++) {
|
|
921
952
|
await new Promise(resolve => setTimeout(resolve, checkInterval));
|
|
953
|
+
const elapsed = Date.now() - readinessStartTime;
|
|
922
954
|
// Check if process exited during wait
|
|
923
955
|
if (globalCollectorExitCode !== null && globalCollectorExitCode !== 0) {
|
|
924
956
|
const errorMsg = `Collector process exited during startup with code ${globalCollectorExitCode}. ${globalCollectorStderr.length > 0 ? `Stderr: ${globalCollectorStderr.join('; ')}` : 'No stderr output.'}`;
|
|
925
957
|
globalCollectorInitError = errorMsg;
|
|
926
|
-
logger.error(`[Collector] ${errorMsg}`);
|
|
958
|
+
logger.error(`[Collector] ${errorMsg} (after ${elapsed}ms)`);
|
|
927
959
|
return null;
|
|
928
960
|
}
|
|
961
|
+
logger.info(`[Collector] Checking collector readiness... (${(elapsed / 1000).toFixed(1)}s elapsed, attempt ${i + 1}/${maxChecks})`);
|
|
929
962
|
const newStatus = await isCollectorRunning();
|
|
963
|
+
logger.info(`[Collector] Collector status: isRunning=${newStatus.isRunning}, isReady=${newStatus.isReady}, pid=${newStatus.pid}, endpoint=${newStatus.otlpEndpoint || 'none'}`);
|
|
930
964
|
if (newStatus.isRunning && newStatus.isReady) {
|
|
931
|
-
logger.info(`[Collector] Collector is ready at ${newStatus.otlpEndpoint || endpoint}`);
|
|
965
|
+
logger.info(`[Collector] Collector is ready at ${newStatus.otlpEndpoint || endpoint} (ready after ${elapsed}ms)`);
|
|
932
966
|
return newStatus.otlpEndpoint ? `http://${newStatus.otlpEndpoint}` : endpoint;
|
|
933
967
|
}
|
|
934
|
-
// Log progress every 2
|
|
935
|
-
if (i > 0 && i %
|
|
936
|
-
logger.info(`[Collector] Still waiting for collector to become ready... (${(
|
|
968
|
+
// Log progress every second (every 2 checks)
|
|
969
|
+
if (i > 0 && i % 2 === 0) {
|
|
970
|
+
logger.info(`[Collector] Still waiting for collector to become ready... (${(elapsed / 1000).toFixed(1)}s elapsed)`);
|
|
937
971
|
}
|
|
938
972
|
}
|
|
939
973
|
// Check one more time if process exited
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"collector-manager.js","sourceRoot":"","sources":["../src/collector-manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAgB,MAAM,eAAe,CAAC;AACpD,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AACzE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AACrC,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,MAAM,MAAM,CAAC;AACpC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,KAAK,CAAC;AAEnC,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAmBhD,MAAM,iBAAiB,GAAG,OAAO,CAAC,CAAC,2DAA2D;AAC9F,MAAM,uBAAuB,GAAG,2CAA2C,iBAAiB,EAAE,CAAC;AAC/F,MAAM,cAAc,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,6BAA6B,IAAI,MAAM,EAAE,EAAE,CAAC,CAAC;AAEzF;;GAEG;AACH,SAAS,YAAY;IACnB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,MAAM,GAAG,YAAY,EAAE,CAAC;QAC9B,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,GAAG,EAAE;YACpB,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YAChF,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE;gBAChB,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;oBACb,OAAO,CAAC,IAAI,CAAC,CAAC;gBAChB,CAAC;qBAAM,CAAC;oBACN,qDAAqD;oBACrD,OAAO,CAAC,IAAI,CAAC,CAAC;gBAChB,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QACH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACzB,MAAM,CAAC,GAAG,CAAC,CAAC;QACd,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,mFAAmF;AACnF,IAAI,sBAAsB,GAAwB,IAAI,CAAC;AACvD,IAAI,yBAAyB,GAAkB,IAAI,CAAC;AACpD,IAAI,uBAAuB,GAAkB,IAAI,CAAC;AAClD,IAAI,qBAAqB,GAAa,EAAE,CAAC;AACzC,wEAAwE;AACxE,IAAI,6BAA6B,GAAkC,IAAI,CAAC;AACxE,IAAI,wBAAwB,GAAkB,IAAI,CAAC,CAAC,8CAA8C;AAYlG;;;;GAIG;AACH,SAAS,kBAAkB,CACzB,GAAW,EACX,MAAc,EACd,eAAuB,EACvB,OAAsB,IAAI,EAC1B,UAAkB,GAAG;IAErB,IAAI,UAAU,GAAG,GAAG,MAAM,GAAG,GAAG,GAAG,OAAO,GAAG,eAAe,EAAE,CAAC;IAC/D,IAAI,IAAI,EAAE,CAAC;QACT,UAAU,IAAI,IAAI,CAAC;IACrB,CAAC;IAED,kDAAkD;IAClD,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC3E,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,GAAW;IAClC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,OAAO,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC;IACzC,CAAC;IAAC,MAAM,CAAC;QACP,qCAAqC;QACrC,OAAO,GAAG,CAAC;IACb,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC9C,GAAsB;IAEtB,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,OAAO,GAAG,oCAAoC,CAAC;IACrD,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC;IAEtD,6DAA6D;IAC7D,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC;IAClC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CAAC,yEAAyE,CAAC,CAAC;IAC7F,CAAC;IAED,2EAA2E;IAC3E,MAAM,YAAY,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IAC9C,MAAM,SAAS,GAAG,kBAAkB,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,YAAY,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;IAE/E,6EAA6E;IAC7E,MAAM,OAAO,GAA2B;QACtC,cAAc,EAAE,kBAAkB;QAClC,MAAM,EAAE,kBAAkB;QAC1B,cAAc,EAAE,GAAG,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,EAAE;QACvC,kBAAkB,EAAE,SAAS;KAC9B,CAAC;IAEF,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE;QACtC,MAAM,EAAE,KAAK;QACb,OAAO;KACR,CAAC,CAAC;IAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,eAAe,CAAC,CAAC;QACrE,MAAM,IAAI,KAAK,CAAC,+CAA+C,SAAS,KAAK,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,MAAM,SAAS,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IAC1J,CAAC;IAED,MAAM,WAAW,GAAG,MAAM,QAAQ,CAAC,IAAI,EAA2B,CAAC;IAEnE,IAAI,CAAC,WAAW,CAAC,QAAQ,IAAI,CAAC,WAAW,CAAC,QAAQ,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC;QAC5E,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;IACtF,CAAC;IAED,OAAO,WAAW,CAAC;AACrB,CAAC;AAED;;GAEG;AACH,SAAS,uBAAuB;IAC9B,qEAAqE;IACrE,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,MAAM,CAAC;IAChE,OAAO,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,iBAAiB,CAAC,CAAC;AAChE,CAAC;AAED;;GAEG;AACH,SAAS,sBAAsB;IAC7B,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;IAClC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAE1B,4CAA4C;IAC5C,2EAA2E;IAC3E,IAAI,QAAQ,KAAK,OAAO,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;QAC3C,OAAO,uBAAuB,CAAC;IACjC,CAAC;SAAM,IAAI,QAAQ,KAAK,OAAO,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;QACpD,OAAO,uBAAuB,CAAC;IACjC,CAAC;SAAM,IAAI,QAAQ,KAAK,QAAQ,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;QACnD,OAAO,wBAAwB,CAAC;IAClC,CAAC;SAAM,IAAI,QAAQ,KAAK,QAAQ,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;QACrD,OAAO,wBAAwB,CAAC;IAClC,CAAC;SAAM,IAAI,QAAQ,KAAK,OAAO,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;QAClD,OAAO,6BAA6B,CAAC;IACvC,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,yBAAyB,QAAQ,IAAI,IAAI,EAAE,CAAC,CAAC;AAC/D,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,yBAAyB,CAAC,GAAW,EAAE,UAAkB,EAAE,iBAA0B,KAAK;IACvG,IAAI,QAAkB,CAAC;IACvB,IAAI,CAAC;QACH,wCAAwC;QACxC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC,oBAAoB;QAEnF,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAC1B,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,OAAO,EAAE;oBACP,YAAY,EAAE,wCAAwC;iBACvD;aACF,CAAC,CAAC;YACH,YAAY,CAAC,SAAS,CAAC,CAAC;QAC1B,CAAC;QAAC,OAAO,UAAU,EAAE,CAAC;YACpB,YAAY,CAAC,SAAS,CAAC,CAAC;YACxB,IAAI,UAAU,YAAY,KAAK,EAAE,CAAC;gBAChC,8CAA8C;gBAC9C,IAAI,eAAe,GAAG,EAAE,CAAC;gBACzB,IAAK,UAAkB,CAAC,KAAK,EAAE,CAAC;oBAC9B,MAAM,KAAK,GAAI,UAAkB,CAAC,KAAK,CAAC;oBACxC,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;wBAC3B,MAAM,SAAS,GAAI,KAAa,CAAC,IAAI,CAAC;wBACtC,eAAe,GAAG,YAAY,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,WAAW,SAAS,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC;oBAC5G,CAAC;yBAAM,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;wBACnD,eAAe,GAAG,iBAAiB,KAAK,CAAC,IAAI,GAAG,CAAC;oBACnD,CAAC;gBACH,CAAC;gBAED,uCAAuC;gBACvC,IAAI,UAAU,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;oBACrC,MAAM,IAAI,KAAK,CAAC,uBAAuB,GAAG,wBAAwB,eAAe,EAAE,CAAC,CAAC;gBACvF,CAAC;gBAED,wDAAwD;gBACxD,MAAM,SAAS,GAAI,UAAkB,CAAC,IAAI,IAAI,CAAE,UAAkB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;gBAChF,IAAI,SAAS,EAAE,CAAC;oBACd,IAAI,SAAS,KAAK,WAAW,IAAI,SAAS,KAAK,WAAW,EAAE,CAAC;wBAC3D,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,WAAW,SAAS,iDAAiD,CAAC,CAAC;oBACzH,CAAC;oBACD,IAAI,SAAS,KAAK,cAAc,EAAE,CAAC;wBACjC,MAAM,IAAI,KAAK,CAAC,yBAAyB,GAAG,WAAW,SAAS,6CAA6C,CAAC,CAAC;oBACjH,CAAC;oBACD,IAAI,SAAS,KAAK,WAAW,IAAI,SAAS,KAAK,YAAY,EAAE,CAAC;wBAC5D,MAAM,IAAI,KAAK,CAAC,+BAA+B,GAAG,WAAW,SAAS,wCAAwC,CAAC,CAAC;oBAClH,CAAC;oBACD,IAAI,SAAS,KAAK,kBAAkB,IAAI,SAAS,KAAK,iCAAiC,EAAE,CAAC;wBACxF,MAAM,IAAI,KAAK,CAAC,2CAA2C,GAAG,WAAW,SAAS,MAAM,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC;oBAChH,CAAC;gBACH,CAAC;gBAED,6CAA6C;gBAC7C,MAAM,QAAQ,GAAG,UAAU,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;gBAClD,IAAI,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC/D,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,iDAAiD,eAAe,EAAE,CAAC,CAAC;gBACtH,CAAC;gBACD,IAAI,QAAQ,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,oBAAoB,CAAC,EAAE,CAAC;oBACjF,MAAM,IAAI,KAAK,CAAC,yBAAyB,GAAG,6CAA6C,eAAe,EAAE,CAAC,CAAC;gBAC9G,CAAC;gBACD,IAAI,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;oBACnE,MAAM,IAAI,KAAK,CAAC,yBAAyB,GAAG,wCAAwC,eAAe,EAAE,CAAC,CAAC;gBACzG,CAAC;gBACD,IAAI,QAAQ,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC7F,MAAM,IAAI,KAAK,CAAC,+BAA+B,GAAG,KAAK,UAAU,CAAC,OAAO,GAAG,eAAe,EAAE,CAAC,CAAC;gBACjG,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,KAAK,UAAU,CAAC,IAAI,MAAM,UAAU,CAAC,OAAO,GAAG,eAAe,EAAE,CAAC,CAAC;YACpH,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,KAAK,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC7E,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,6BAA6B;QAC7B,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;gBAC/G,MAAM,KAAK,CAAC,CAAC,2BAA2B;YAC1C,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACxE,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,KAAK,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACxE,CAAC;IAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QACxD,MAAM,IAAI,KAAK,CAAC,QAAQ,QAAQ,CAAC,MAAM,gBAAgB,GAAG,KAAK,QAAQ,CAAC,UAAU,GAAG,SAAS,CAAC,CAAC,CAAC,MAAM,SAAS,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC/I,CAAC;IAED,MAAM,GAAG,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IAChC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACrB,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtC,CAAC;IAED,MAAM,MAAM,GAAG,YAAY,EAAE,CAAC;IAC9B,MAAM,WAAW,GAAG,iBAAiB,CAAC,UAAU,CAAC,CAAC;IAElD,MAAM,QAAQ,CAAC,QAAQ,CAAC,IAAW,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;IAE1D,IAAI,cAAc,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;QACnD,IAAI,CAAC;YACH,SAAS,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QAC/B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,kBAAkB,UAAU,cAAc,EAAE,KAAK,CAAC,CAAC;QACnE,CAAC;IACH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,gBAAgB,CAAC,gBAAyB,IAAI,EAAE,MAAe;IAC5E,MAAM,QAAQ,GAAG,uBAAuB,EAAE,CAAC;IAC3C,MAAM,UAAU,GAAG,sBAAsB,EAAE,CAAC;IAC5C,MAAM,UAAU,GAAG,wBAAwB,CAAC;IAE5C,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAC9C,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAE9C,MAAM,eAAe,GAA8D,EAAE,CAAC;IAEtF,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,aAAa,EAAE,CAAC;QAC7C,MAAM,SAAS,GAAG,GAAG,uBAAuB,IAAI,UAAU,KAAK,CAAC;QAChE,eAAe,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7E,MAAM,EAAE,IAAI,CAAC,0CAA0C,SAAS,EAAE,CAAC,CAAC;IACtE,CAAC;SAAM,IAAI,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAClC,MAAM,EAAE,IAAI,CAAC,gCAAgC,UAAU,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,aAAa,EAAE,CAAC;QAC7C,MAAM,SAAS,GAAG,GAAG,uBAAuB,IAAI,UAAU,KAAK,CAAC;QAChE,eAAe,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC;QAC9E,MAAM,EAAE,IAAI,CAAC,0CAA0C,SAAS,EAAE,CAAC,CAAC;IACtE,CAAC;SAAM,IAAI,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAClC,MAAM,EAAE,IAAI,CAAC,gCAAgC,UAAU,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,mFAAmF;IACnF,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/B,MAAM,EAAE,IAAI,CAAC,2BAA2B,eAAe,CAAC,MAAM,yBAAyB,CAAC,CAAC;QACzF,MAAM,gBAAgB,GAAG,eAAe,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;YAC1D,IAAI,CAAC;gBACH,MAAM,EAAE,IAAI,CAAC,2BAA2B,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC;gBACvD,MAAM,yBAAyB,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;gBACtE,MAAM,EAAE,IAAI,CAAC,6BAA6B,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YACzD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,wEAAwE;gBACxE,MAAM,QAAQ,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACxE,MAAM,EAAE,KAAK,CAAC,kCAAkC,IAAI,CAAC,GAAG,KAAK,QAAQ,EAAE,CAAC,CAAC;gBACzE,yDAAyD;gBACzD,IAAI,KAAK,YAAY,KAAK,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC1D,MAAM,KAAK,CAAC,CAAC,oCAAoC;gBACnD,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,gCAAgC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,SAAS,IAAI,CAAC,GAAG,KAAK,QAAQ,EAAE,CAAC,CAAC;YACzH,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,+CAA+C;QAC/C,MAAM,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IACtC,CAAC;IAED,OAAO;QACL,UAAU,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI;QACtD,UAAU,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI;KACvD,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,uBAAuB,CAAC,YAAoB,IAAI;IACvD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,MAAM,MAAM,GAAG,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,UAAU,GAA8B,EAAE,CAAC;QACjD,IAAI,OAAuB,CAAC;QAE5B,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,EAAE;YAC3B,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,CAA4B,CAAC;gBACtE,2BAA2B;gBAC3B,IAAI,OAAO,CAAC,OAAO,KAAK,iBAAiB,IAAI,OAAO,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;oBACxE,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAC3B,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,sBAAsB;YACxB,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YACtB,YAAY,CAAC,OAAO,CAAC,CAAC;YACtB,MAAM,CAAC,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,CAAC;QAChB,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE;YACf,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAE1B,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;gBACxB,MAAM,CAAC,KAAK,EAAE,CAAC;gBACf,wCAAwC;gBACxC,OAAO,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACxD,CAAC,EAAE,SAAS,CAAC,CAAC;QAChB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,8BAA8B;IAM5C,MAAM,WAAW,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;IAC3D,MAAM,WAAW,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;IAC3D,MAAM,WAAW,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;IAE3D,IAAI,WAAW,IAAI,WAAW,IAAI,WAAW,EAAE,CAAC;QAC9C,OAAO;YACL,WAAW,EAAE,IAAI;YACjB,WAAW,EAAE,IAAI;YACjB,WAAW,EAAE,IAAI;YACjB,MAAM,EAAE,aAAa;SACtB,CAAC;IACJ,CAAC;IAED,OAAO;QACL,WAAW;QACX,WAAW;QACX,WAAW;QACX,MAAM,EAAE,SAAS;KAClB,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB;IACtC,+DAA+D;IAC/D,uFAAuF;IACvF,IAAI,sBAAsB,EAAE,CAAC;QAC3B,kCAAkC;QAClC,IAAI,CAAC;YACH,4DAA4D;YAC5D,MAAM,YAAY,GAAG,sBAAsB,CAAC,QAAQ,KAAK,IAAI;gBACxC,sBAAsB,CAAC,MAAM,KAAK,KAAK,CAAC;YAE7D,IAAI,YAAY,EAAE,CAAC;gBACjB,kEAAkE;gBAClE,2DAA2D;gBAC3D,MAAM,UAAU,GAAG,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACpD,MAAM,OAAO,GAAG,UAAU,CAAC,QAAQ,CAAC,qBAAqB,CAAC;oBAC3C,UAAU,CAAC,QAAQ,CAAC,mCAAmC,CAAC,CAAC;gBAExE,IAAI,OAAO,EAAE,CAAC;oBACZ,sFAAsF;oBACtF,IAAI,CAAC;wBACH,MAAM,UAAU,GAAG,MAAM,uBAAuB,CAAC,IAAI,CAAC,CAAC,CAAC,6CAA6C;wBACrG,IAAI,UAAU,IAAI,UAAU,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;4BAChD,OAAO;gCACL,SAAS,EAAE,IAAI;gCACf,OAAO,EAAE,IAAI;gCACb,GAAG,EAAE,UAAU,CAAC,GAAG,IAAI,sBAAsB,CAAC,GAAG,IAAI,CAAC;gCACtD,YAAY,EAAE,UAAU,CAAC,YAAY;gCACrC,OAAO,EAAE,UAAU,CAAC,OAAO;6BAC5B,CAAC;wBACJ,CAAC;oBACH,CAAC;oBAAC,MAAM,CAAC;wBACP,mFAAmF;wBACnF,0CAA0C;oBAC5C,CAAC;oBAED,yEAAyE;oBACzE,mDAAmD;oBACnD,MAAM,kBAAkB,GAAG,OAAO,CAAC,GAAG,CAAC,uBAAuB,IAAI,cAAc,CAAC;oBACjF,OAAO;wBACL,SAAS,EAAE,IAAI;wBACf,OAAO,EAAE,IAAI;wBACb,GAAG,EAAE,sBAAsB,CAAC,GAAG,IAAI,CAAC;wBACpC,YAAY,EAAE,kBAAkB;wBAChC,OAAO,EAAE,iBAAiB;qBAC3B,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,0CAA0C;QAC5C,CAAC;IACH,CAAC;IAED,gCAAgC;IAChC,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,MAAM,uBAAuB,CAAC,IAAI,CAAC,CAAC;QACvD,IAAI,UAAU,EAAE,CAAC;YACf,OAAO;gBACL,SAAS,EAAE,IAAI;gBACf,OAAO,EAAE,UAAU,CAAC,MAAM,KAAK,OAAO;gBACtC,GAAG,EAAE,UAAU,CAAC,GAAG;gBACnB,YAAY,EAAE,UAAU,CAAC,YAAY;gBACrC,OAAO,EAAE,UAAU,CAAC,OAAO;aAC5B,CAAC;QACJ,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,mDAAmD;IACrD,CAAC;IAED,OAAO;QACL,SAAS,EAAE,KAAK;QAChB,OAAO,EAAE,KAAK;QACd,GAAG,EAAE,CAAC;KACP,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,SAAS,sCAAsC;IAC7C,MAAM,QAAQ,GAA2B;QACvC,iCAAiC,EAAE,IAAI;QACvC,oCAAoC,EAAE,MAAM;QAC5C,gCAAgC,EAAE,IAAI;QACtC,mCAAmC,EAAE,MAAM;QAC3C,sCAAsC,EAAE,MAAM;QAC9C,+CAA+C,EAAE,IAAI;QACrD,2CAA2C,EAAE,KAAK;QAClD,+CAA+C,EAAE,MAAM;KACxD,CAAC;IAEF,qDAAqD;IACrD,KAAK,MAAM,CAAC,GAAG,EAAE,YAAY,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC3D,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,YAAY,CAAC;QAClC,CAAC;IACH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,sBAAsB,CAAC,QAAgB,EAAE,QAAgB,EAAE,QAAgB;IACzF,wEAAwE;IACxE,6FAA6F;IAC7F,kHAAkH;IAClH,OAAO,CAAC,GAAG,CAAC,wBAAwB,GAAG,QAAQ,CAAC;IAChD,OAAO,CAAC,GAAG,CAAC,wBAAwB,GAAG,QAAQ,CAAC;IAChD,OAAO,CAAC,GAAG,CAAC,wBAAwB,GAAG,QAAQ,CAAC;IAEhD,gDAAgD;IAChD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,EAAE,CAAC;QAC5H,MAAM,IAAI,KAAK,CAAC,2FAA2F,CAAC,CAAC;IAC/G,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,6BAA6B,CACjD,GAAsB,EACtB,UAAmB,CAAC,wCAAwC;;IAE5D,mFAAmF;IACnF,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,MAAM,CAAC;IACnF,MAAM,YAAY,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC;IACnD,MAAM,mBAAmB,GAAG,CAAC,UAAU,IAAI,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC;IACpG,MAAM,mBAAmB,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC;IAE3E,4CAA4C;IAC5C,IAAI,CAAC,mBAAmB,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACjD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,4EAA4E;IAC5E,MAAM,UAAU,GAAG,IAAI,CAAC;QACtB,IAAI,EAAE,oBAAoB;QAC1B,KAAK,EAAE,MAAM;KACd,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAEnB,UAAU,CAAC,IAAI,CAAC,oFAAoF,CAAC,CAAC;IAEtG,sEAAsE;IACtE,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,wBAAwB,CAAC,UAAU,EAAE,mBAAmB,EAAE,GAAG,CAAC,CAAC;QAEtF,IAAI,QAAQ,EAAE,CAAC;YACb,UAAU,CAAC,IAAI,CAAC,6BAA6B,QAAQ,4BAA4B,CAAC,CAAC;YACnF,OAAO,QAAQ,CAAC;QAClB,CAAC;aAAM,CAAC;YACN,UAAU,CAAC,IAAI,CAAC,+DAA+D,CAAC,CAAC;YACjF,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,QAAQ,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACxE,UAAU,CAAC,KAAK,CAAC,kCAAkC,QAAQ,EAAE,CAAC,CAAC;QAC/D,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB,CAAC,GAAsB;IACxD,mFAAmF;IACnF,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,MAAM,CAAC;IACnF,MAAM,YAAY,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC;IACnD,MAAM,mBAAmB,GAAG,CAAC,UAAU,IAAI,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC;IACpG,MAAM,mBAAmB,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC;IAE3E,4CAA4C;IAC5C,IAAI,CAAC,mBAAmB,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACjD,OAAO;IACT,CAAC;IAED,4EAA4E;IAC5E,MAAM,UAAU,GAAG,IAAI,CAAC;QACtB,IAAI,EAAE,oBAAoB;QAC1B,KAAK,EAAE,MAAM;KACd,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAEnB,UAAU,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;IAEnF,2DAA2D;IAC3D,uEAAuE;IACvE,wBAAwB,CAAC,UAAU,EAAE,mBAAmB,EAAE,GAAG,CAAC;SAC3D,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;QACf,IAAI,MAAM,EAAE,CAAC;YACX,UAAU,CAAC,IAAI,CAAC,4DAA4D,MAAM,EAAE,CAAC,CAAC;YACtF,gEAAgE;YAChE,0EAA0E;QAC5E,CAAC;aAAM,CAAC;YACN,UAAU,CAAC,IAAI,CAAC,qEAAqE,CAAC,CAAC;QACzF,CAAC;IACH,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;QACf,MAAM,QAAQ,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACxE,UAAU,CAAC,KAAK,CAAC,mDAAmD,QAAQ,EAAE,CAAC,CAAC;IAClF,CAAC,CAAC,CAAC;IAEL,0DAA0D;AAC5D,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,2BAA2B,CACzC,GAAsB,EACtB,YAAoB,KAAK;IAEzB,mFAAmF;IACnF,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,MAAM,CAAC;IACnF,MAAM,YAAY,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC;IACnD,MAAM,mBAAmB,GAAG,CAAC,UAAU,IAAI,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC;IACpG,MAAM,mBAAmB,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC;IAE3E,4CAA4C;IAC5C,IAAI,CAAC,mBAAmB,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACjD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,4EAA4E;IAC5E,MAAM,UAAU,GAAG,IAAI,CAAC;QACtB,IAAI,EAAE,oBAAoB;QAC1B,KAAK,EAAE,MAAM;KACd,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAEnB,UAAU,CAAC,IAAI,CAAC,8DAA8D,CAAC,CAAC;IAEhF,wDAAwD;IACxD,IAAI,QAAQ,GAAkB,IAAI,CAAC;IACnC,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,UAAU,GAAkB,IAAI,CAAC;IAErC,gCAAgC;IAChC,wBAAwB,CAAC,UAAU,EAAE,mBAAmB,EAAE,GAAG,CAAC;SAC3D,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;QACf,QAAQ,GAAG,MAAM,CAAC;QAClB,SAAS,GAAG,IAAI,CAAC;QACjB,IAAI,MAAM,EAAE,CAAC;YACX,UAAU,CAAC,IAAI,CAAC,8CAA8C,MAAM,EAAE,CAAC,CAAC;QAC1E,CAAC;aAAM,CAAC;YACN,UAAU,CAAC,IAAI,CAAC,qEAAqE,CAAC,CAAC;QACzF,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;QACf,UAAU,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACpE,SAAS,GAAG,IAAI,CAAC;QACjB,UAAU,CAAC,KAAK,CAAC,qCAAqC,UAAU,EAAE,CAAC,CAAC;QACpE,QAAQ,GAAG,IAAI,CAAC;IAClB,CAAC,CAAC,CAAC;IAEL,qDAAqD;IACrD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE;QAChC,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,UAAU,CAAC,IAAI,CAAC,wCAAwC,SAAS,6BAA6B,CAAC,CAAC;YAChG,SAAS,GAAG,IAAI,CAAC;QACnB,CAAC;IACH,CAAC,EAAE,SAAS,CAAC,CAAC;IAEd,IAAI,CAAC;QACH,qFAAqF;QACrF,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE;YACrB,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YACvC,IAAI,OAAO,IAAI,SAAS,EAAE,CAAC;gBACzB,OAAO,KAAK,CAAC,CAAC,eAAe;YAC/B,CAAC;YACD,OAAO,CAAC,SAAS,CAAC,CAAC,oCAAoC;QACzD,CAAC,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,UAAU,CAAC,KAAK,CAAC,6CAA6C,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC1H,CAAC;IAED,YAAY,CAAC,SAAS,CAAC,CAAC;IAExB,IAAI,SAAS,IAAI,QAAQ,EAAE,CAAC;QAC1B,UAAU,CAAC,IAAI,CAAC,4DAA4D,CAAC,CAAC;QAC9E,OAAO,QAAQ,CAAC;IAClB,CAAC;SAAM,IAAI,UAAU,EAAE,CAAC;QACtB,UAAU,CAAC,KAAK,CAAC,kCAAkC,UAAU,2BAA2B,CAAC,CAAC;QAC1F,OAAO,IAAI,CAAC;IACd,CAAC;SAAM,CAAC;QACN,UAAU,CAAC,IAAI,CAAC,0EAA0E,CAAC,CAAC;QAC5F,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAAc,EACd,YAAqB,EACrB,GAAuB;IAEvB,4BAA4B;IAC5B,yBAAyB,GAAG,IAAI,CAAC;IACjC,wBAAwB,GAAG,IAAI,CAAC;IAEhC,8EAA8E;IAC9E,+EAA+E;IAC/E,+EAA+E;IAC/E,iHAAiH;IACjH,IAAI,kBAAkB,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;IAC9D,IAAI,kBAAkB,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;IAC9D,IAAI,kBAAkB,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;IAE9D,IAAI,CAAC,CAAC,kBAAkB,IAAI,CAAC,kBAAkB,IAAI,CAAC,kBAAkB,CAAC,IAAI,GAAG,EAAE,CAAC;QAC/E,IAAI,CAAC;YACH,MAAM,CAAC,IAAI,CAAC,kEAAkE,CAAC,CAAC;YAChF,MAAM,WAAW,GAAG,MAAM,0BAA0B,CAAC,GAAG,CAAC,CAAC;YAC1D,kBAAkB,GAAG,WAAW,CAAC,QAAQ,CAAC;YAC1C,kBAAkB,GAAG,WAAW,CAAC,QAAQ,CAAC;YAC1C,kBAAkB,GAAG,WAAW,CAAC,QAAQ,CAAC;YAE1C,qHAAqH;YACrH,kHAAkH;YAClH,0GAA0G;YAC1G,sBAAsB,CAAC,kBAAkB,EAAE,kBAAkB,EAAE,kBAAkB,CAAC,CAAC;YAEnF,wEAAwE;YACxE,MAAM,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;YAC5D,MAAM,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;YAC5D,MAAM,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;YAE5D,IAAI,CAAC,cAAc,IAAI,CAAC,cAAc,IAAI,CAAC,cAAc,EAAE,CAAC;gBAC1D,MAAM,CAAC,KAAK,CAAC,wGAAwG,CAAC,CAAC;gBACvH,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;YAC7E,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,iFAAiF,CAAC,CAAC;QACjG,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,QAAQ,GAAG,gEAAgE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YAC1I,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,iEAAiE;IACjE,0EAA0E;IAC1E,IAAI,CAAC,kBAAkB,IAAI,CAAC,kBAAkB,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACtE,MAAM,QAAQ,GAAG,0LAA0L,CAAC;QAC5M,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC5B,CAAC;IAED,kFAAkF;IAClF,MAAM,CAAC,IAAI,CAAC,sDAAsD,CAAC,CAAC;IACpE,MAAM,aAAa,GAAG,MAAM,gBAAgB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAE3D,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,CAAC;QAC9B,MAAM,CAAC,KAAK,CAAC,kDAAkD,CAAC,CAAC;QACjE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;IACpE,CAAC;IAED,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,CAAC;QAC9B,MAAM,CAAC,KAAK,CAAC,kDAAkD,CAAC,CAAC;QACjE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;IACpE,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,6BAA6B,aAAa,CAAC,UAAU,EAAE,CAAC,CAAC;IACrE,MAAM,CAAC,IAAI,CAAC,6BAA6B,aAAa,CAAC,UAAU,EAAE,CAAC,CAAC;IAErE,uBAAuB;IACvB,IAAI,CAAC,aAAa,CAAC,UAAU,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC;QACvE,MAAM,IAAI,KAAK,CAAC,iCAAiC,aAAa,CAAC,UAAU,EAAE,CAAC,CAAC;IAC/E,CAAC;IAED,sFAAsF;IACtF,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;QACjC,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;YAC/C,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;YAC9D,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAChC,MAAM,YAAY,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,oBAAoB;YACxE,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,MAAM,CAAC,IAAI,CAAC,mDAAmD,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACnF,kCAAkC;gBAClC,IAAI,CAAC;oBACH,SAAS,CAAC,aAAa,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;oBAC3C,MAAM,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;gBACpD,CAAC;gBAAC,OAAO,UAAU,EAAE,CAAC;oBACpB,MAAM,CAAC,KAAK,CAAC,iDAAiD,UAAU,YAAY,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;gBACzI,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,SAAS,EAAE,CAAC;YACnB,MAAM,CAAC,IAAI,CAAC,mDAAmD,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACvI,CAAC;IACH,CAAC;IAED,2EAA2E;IAC3E,6FAA6F;IAC7F,uEAAuE;IACvE,sCAAsC,EAAE,CAAC;IAEzC,sHAAsH;IACtH,iDAAiD;IACjD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,6BAA6B,EAAE,CAAC;QAC/C,OAAO,CAAC,GAAG,CAAC,6BAA6B,GAAG,MAAM,CAAC,cAAc,CAAC,CAAC;QACnE,MAAM,CAAC,IAAI,CAAC,iDAAiD,cAAc,YAAY,CAAC,CAAC;IAC3F,CAAC;IAED,0BAA0B;IAC1B,+EAA+E;IAC/E,IAAI,aAAa,GAAG,YAAY,CAAC;IACjC,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,sCAAsC;QACtC,6DAA6D;QAC7D,aAAa,GAAG,cAAc,CAAC;IACjC,CAAC;IACD,aAAa,GAAG,aAAa,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;IAEnF,uEAAuE;IACvE,2EAA2E;IAC3E,iFAAiF;IACjF,mEAAmE;IACnE,gDAAgD;IAChD,MAAM,cAAc,GAAG,MAAM,YAAY,EAAE,CAAC;IAE5C,0CAA0C;IAC1C,6EAA6E;IAC7E,sEAAsE;IACtE,8FAA8F;IAC9F,6FAA6F;IAC7F,MAAM,YAAY,GAAsB;QACtC,GAAG,OAAO,CAAC,GAAG,EAAE,yEAAyE;QACzF,uBAAuB,EAAE,aAAa;QACtC,6BAA6B,EAAE,MAAM,CAAC,cAAc,CAAC;QACrD,8BAA8B,EAAE,MAAM,CAAC,cAAc,CAAC;KACvD,CAAC;IAEF,0BAA0B;IAC1B,MAAM,gBAAgB,GAAG,KAAK,CAAC,aAAa,CAAC,UAAU,EAAE,CAAC,UAAU,EAAE,aAAa,CAAC,UAAU,CAAC,EAAE;QAC/F,GAAG,EAAE,YAAY;QACjB,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;QACjC,QAAQ,EAAE,KAAK;KAChB,CAAC,CAAC;IAEH,gDAAgD;IAChD,sBAAsB,GAAG,gBAAgB,CAAC;IAC1C,yBAAyB,GAAG,IAAI,CAAC;IACjC,uBAAuB,GAAG,IAAI,CAAC;IAC/B,qBAAqB,GAAG,EAAE,CAAC;IAE3B,+EAA+E;IAC/E,gBAAgB,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;QAC3C,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC;QAC1C,mDAAmD;QACnD,MAAM,CAAC,IAAI,CAAC,mBAAmB,UAAU,EAAE,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,gBAAgB,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;QAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC;QACzC,qBAAqB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtC,mDAAmD;QACnD,IAAI,qBAAqB,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YACtC,qBAAqB,CAAC,KAAK,EAAE,CAAC;QAChC,CAAC;QACD,+DAA+D;QAC/D,MAAM,CAAC,IAAI,CAAC,mBAAmB,SAAS,EAAE,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;IAEH,gBAAgB,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;QACnC,yBAAyB,GAAG,GAAG,CAAC,OAAO,CAAC;QACxC,MAAM,CAAC,KAAK,CAAC,gCAAgC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;IAEH,gBAAgB,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE;QAC3C,uBAAuB,GAAG,IAAI,CAAC;QAC/B,MAAM,CAAC,IAAI,CAAC,wCAAwC,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,YAAY,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAEjG,+DAA+D;QAC/D,IAAI,IAAI,KAAK,CAAC,IAAI,qBAAqB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnD,MAAM,CAAC,KAAK,CAAC,iCAAiC,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,qBAAqB,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;YAC1G,qBAAqB,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;gBAChD,MAAM,CAAC,KAAK,CAAC,mBAAmB,IAAI,EAAE,CAAC,CAAC;YAC1C,CAAC,CAAC,CAAC;QACL,CAAC;QAED,gEAAgE;QAChE,mFAAmF;IACrF,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,IAAI,CAAC,gCAAgC,gBAAgB,CAAC,GAAG,eAAe,aAAa,EAAE,CAAC,CAAC;IAEhG,qEAAqE;IACrE,8CAA8C;IAC9C,yDAAyD;IACzD,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;IAEvD,oDAAoD;IACpD,IAAI,uBAAuB,KAAK,IAAI,IAAI,uBAAuB,KAAK,CAAC,EAAE,CAAC;QACtE,MAAM,SAAS,GAAG,qBAAqB,CAAC,MAAM,GAAG,CAAC;YAChD,CAAC,CAAC,YAAY,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YAChD,CAAC,CAAC,6BAA6B,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,gEAAgE,uBAAuB,IAAI,SAAS,EAAE,CAAC,CAAC;IAC1H,CAAC;IAED,OAAO;QACL,OAAO,EAAE,gBAAgB;QACzB,QAAQ,EAAE,UAAU,aAAa,EAAE;KACpC,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,yBAAyB;IAQvC,OAAO;QACL,UAAU,EAAE,sBAAsB,KAAK,IAAI;QAC3C,GAAG,EAAE,sBAAsB,EAAE,GAAG,IAAI,IAAI;QACxC,QAAQ,EAAE,uBAAuB;QACjC,UAAU,EAAE,yBAAyB;QACrC,SAAS,EAAE,wBAAwB;QACnC,MAAM,EAAE,CAAC,GAAG,qBAAqB,CAAC,EAAE,gBAAgB;KACrD,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,MAAc,EACd,mBAA4B,EAC5B,GAAuB;IAEvB,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,8DAA8D;IAC9D,qFAAqF;IACrF,oFAAoF;IACpF,IAAI,6BAA6B,EAAE,CAAC;QAClC,MAAM,CAAC,IAAI,CAAC,gGAAgG,CAAC,CAAC;QAC9G,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,6BAA6B,CAAC;YACnD,4DAA4D;YAC5D,6BAA6B,GAAG,IAAI,CAAC;YACrC,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,gDAAgD,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACvH,oCAAoC;YACpC,6BAA6B,GAAG,IAAI,CAAC;YACrC,kCAAkC;QACpC,CAAC;IACH,CAAC;IAED,0EAA0E;IAC1E,qFAAqF;IACrF,IAAI,gBAAoC,CAAC;IACzC,IAAI,sBAAsB,EAAE,CAAC;QAC3B,kCAAkC;QAClC,MAAM,YAAY,GAAG,sBAAsB,CAAC,QAAQ,KAAK,IAAI;YACxC,sBAAsB,CAAC,MAAM,KAAK,KAAK,CAAC;QAE7D,IAAI,YAAY,EAAE,CAAC;YACjB,MAAM,CAAC,IAAI,CAAC,qDAAqD,sBAAsB,CAAC,GAAG,kCAAkC,CAAC,CAAC;YAC/H,uEAAuE;YACvE,gBAAgB,GAAG,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;gBACtD,UAAU,OAAO,CAAC,GAAG,CAAC,uBAAuB,EAAE,CAAC,CAAC;gBACjD,qBAAqB,CAAC;YACxB,qEAAqE;QACvE,CAAC;aAAM,CAAC;YACN,4CAA4C;YAC5C,MAAM,CAAC,IAAI,CAAC,+CAA+C,sBAAsB,CAAC,GAAG,gCAAgC,CAAC,CAAC;YACvH,sBAAsB,GAAG,IAAI,CAAC;YAC9B,uBAAuB,GAAG,IAAI,CAAC;YAC/B,qBAAqB,GAAG,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,mEAAmE;IACnE,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,MAAM,kBAAkB,EAAE,CAAC;QAC1C,IAAI,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;YAC9D,MAAM,CAAC,IAAI,CAAC,0CAA0C,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC;YAC7E,OAAO,UAAU,MAAM,CAAC,YAAY,EAAE,CAAC;QACzC,CAAC;IACH,CAAC;IAED,0EAA0E;IAC1E,oEAAoE;IACpE,2EAA2E;IAC3E,MAAM,cAAc,GAAG,CAAC,KAAK,IAA4B,EAAE;QACzD,IAAI,CAAC;YACH,gCAAgC;YAChC,wBAAwB,GAAG,IAAI,CAAC;YAEhC,IAAI,QAAgB,CAAC;YACrB,IAAI,gBAAgB,EAAE,CAAC;gBACrB,2DAA2D;gBAC3D,QAAQ,GAAG,gBAAgB,CAAC;gBAC5B,MAAM,CAAC,IAAI,CAAC,iEAAiE,QAAQ,KAAK,CAAC,CAAC;YAChG,CAAC;iBAAM,CAAC;gBACN,wBAAwB;gBACxB,MAAM,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;gBAC/D,MAAM,WAAW,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC;gBACjE,QAAQ,GAAG,WAAW,CAAC,QAAQ,CAAC;gBAEhC,0DAA0D;gBAC1D,8DAA8D;gBAC9D,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;gBAEvD,IAAI,uBAAuB,KAAK,IAAI,IAAI,uBAAuB,KAAK,CAAC,EAAE,CAAC;oBACtE,MAAM,QAAQ,GAAG,kDAAkD,uBAAuB,KAAK,qBAAqB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,mBAAmB,EAAE,CAAC;oBACxM,wBAAwB,GAAG,QAAQ,CAAC;oBACpC,MAAM,CAAC,KAAK,CAAC,eAAe,QAAQ,EAAE,CAAC,CAAC;oBACxC,OAAO,IAAI,CAAC;gBACd,CAAC;YACH,CAAC;YAED,kEAAkE;YAClE,oDAAoD;YACpD,kFAAkF;YAClF,MAAM,WAAW,GAAG,KAAK,CAAC,CAAC,aAAa;YACxC,MAAM,aAAa,GAAG,GAAG,CAAC,CAAC,oBAAoB;YAC/C,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,aAAa,CAAC,CAAC;YAE1D,MAAM,CAAC,IAAI,CAAC,sDAAsD,CAAC,CAAC;YAEpE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;gBACnC,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC,CAAC;gBAEjE,sCAAsC;gBACtC,IAAI,uBAAuB,KAAK,IAAI,IAAI,uBAAuB,KAAK,CAAC,EAAE,CAAC;oBACtE,MAAM,QAAQ,GAAG,qDAAqD,uBAAuB,KAAK,qBAAqB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,mBAAmB,EAAE,CAAC;oBAC3M,wBAAwB,GAAG,QAAQ,CAAC;oBACpC,MAAM,CAAC,KAAK,CAAC,eAAe,QAAQ,EAAE,CAAC,CAAC;oBACxC,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,MAAM,SAAS,GAAG,MAAM,kBAAkB,EAAE,CAAC;gBAC7C,IAAI,SAAS,CAAC,SAAS,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;oBAC7C,MAAM,CAAC,IAAI,CAAC,qCAAqC,SAAS,CAAC,YAAY,IAAI,QAAQ,EAAE,CAAC,CAAC;oBACvF,OAAO,SAAS,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,SAAS,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC;gBAChF,CAAC;gBAED,+BAA+B;gBAC/B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;oBACzB,MAAM,CAAC,IAAI,CAAC,+DAA+D,CAAC,CAAC,GAAG,aAAa,CAAC,GAAG,IAAI,YAAY,CAAC,CAAC;gBACrH,CAAC;YACH,CAAC;YAED,wCAAwC;YACxC,IAAI,uBAAuB,KAAK,IAAI,IAAI,uBAAuB,KAAK,CAAC,EAAE,CAAC;gBACtE,MAAM,QAAQ,GAAG,sCAAsC,uBAAuB,KAAK,qBAAqB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,mBAAmB,EAAE,CAAC;gBAC5L,wBAAwB,GAAG,QAAQ,CAAC;gBACpC,MAAM,CAAC,KAAK,CAAC,eAAe,QAAQ,EAAE,CAAC,CAAC;gBACxC,OAAO,IAAI,CAAC;YACd,CAAC;YAEC,gDAAgD;YAChD,MAAM,CAAC,KAAK,CAAC,qDAAqD,WAAW,GAAG,IAAI,UAAU,CAAC,CAAC;YAChG,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,QAAQ,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAClE,wBAAwB,GAAG,QAAQ,CAAC;YACpC,MAAM,CAAC,KAAK,CAAC,0CAA0C,QAAQ,EAAE,CAAC,CAAC;YACnE,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC,CAAC,EAAE,CAAC;IAEL,6EAA6E;IAC7E,6BAA6B,GAAG,cAAc,CAAC;IAE/C,iFAAiF;IACjF,cAAc,CAAC,OAAO,CAAC,GAAG,EAAE;QAC1B,6BAA6B,GAAG,IAAI,CAAC;IACvC,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,cAAc,CAAC;AAC9B,CAAC","sourcesContent":["import { spawn, ChildProcess } from 'child_process';\r\nimport { createWriteStream, existsSync, chmodSync, mkdirSync } from 'fs';\r\nimport { join, dirname } from 'path';\r\nimport { pipeline } from 'stream/promises';\r\nimport { createGunzip } from 'zlib';\r\nimport { createHash } from 'node:crypto';\r\nimport { createServer } from 'net';\r\nimport type { Logger } from 'pino';\r\nimport pino from 'pino';\r\nimport dgram from 'dgram';\r\nimport deasync from 'deasync';\r\nimport type { EnvironmentConfig } from './types.js';\r\nimport { hostToHttpUrl } from './utils/urls.js';\r\n\r\n// Protocol is httpprotobuf \r\n\r\ninterface CollectorDiscoveryEntry {\r\n version: string;\r\n status: string;\r\n pid: number;\r\n otlpEndpoint: string;\r\n}\r\n\r\nexport interface CollectorStatus {\r\n isRunning: boolean;\r\n isReady: boolean;\r\n pid: number;\r\n otlpEndpoint?: string;\r\n version?: string;\r\n}\r\n\r\nconst COLLECTOR_VERSION = '1.0.1'; // Match C# collector version (from collector-version.json)\r\nconst COLLECTOR_DOWNLOAD_BASE = `https://collectors.beamable.com/version/${COLLECTOR_VERSION}`;\r\nconst DISCOVERY_PORT = parseInt(process.env.BEAM_COLLECTOR_DISCOVERY_PORT || '8688', 10);\r\n\r\n/**\r\n * Finds a free TCP port (matching C# PortUtil.FreeTcpPort())\r\n */\r\nfunction findFreePort(): Promise<number> {\r\n return new Promise((resolve, reject) => {\r\n const server = createServer();\r\n server.listen(0, () => {\r\n const address = server.address();\r\n const port = typeof address === 'object' && address !== null ? address.port : 0;\r\n server.close(() => {\r\n if (port > 0) {\r\n resolve(port);\r\n } else {\r\n // Fallback to a default port if something went wrong\r\n resolve(9090);\r\n }\r\n });\r\n });\r\n server.on('error', (err) => {\r\n reject(err);\r\n });\r\n });\r\n}\r\n\r\n// Global storage for collector process and errors (to prevent GC and track status)\r\nlet globalCollectorProcess: ChildProcess | null = null;\r\nlet globalCollectorStartError: string | null = null;\r\nlet globalCollectorExitCode: number | null = null;\r\nlet globalCollectorStderr: string[] = [];\r\n// Track if collector startup is in progress to prevent duplicate starts\r\nlet globalCollectorStartupPromise: Promise<string | null> | null = null;\r\nlet globalCollectorInitError: string | null = null; // Tracks errors from discoverOrStartCollector\r\n\r\n/**\r\n * ClickHouse credentials response from Beamable API\r\n */\r\ninterface ClickHouseCredentials {\r\n endpoint: string;\r\n expiresAt?: string;\r\n password: string;\r\n username: string;\r\n}\r\n\r\n/**\r\n * Calculates Beamable signature for signed requests\r\n * Signature format: MD5(secret + pid + version + uriPathAndQuery + body) as Base64\r\n * Matches C# SignedRequesterHelper.CalculateSignature\r\n */\r\nfunction calculateSignature(\r\n pid: string,\r\n secret: string,\r\n uriPathAndQuery: string,\r\n body: string | null = null,\r\n version: string = '1'\r\n): string {\r\n let dataToSign = `${secret}${pid}${version}${uriPathAndQuery}`;\r\n if (body) {\r\n dataToSign += body;\r\n }\r\n \r\n // MD5 hash to Base64 (matching C# implementation)\r\n const hash = createHash('md5').update(dataToSign, 'utf8').digest('base64');\r\n return hash;\r\n}\r\n\r\n/**\r\n * Extracts path and query from a URL\r\n */\r\nfunction getPathAndQuery(url: string): string {\r\n try {\r\n const urlObj = new URL(url);\r\n return urlObj.pathname + urlObj.search;\r\n } catch {\r\n // If URL parsing fails, return as-is\r\n return url;\r\n }\r\n}\r\n\r\n/**\r\n * Fetches ClickHouse credentials from Beamable API\r\n * GET /api/beamo/otel/auth/writer/config\r\n * Requires signed request authentication (X-BEAM-SIGNATURE header)\r\n */\r\nexport async function fetchClickHouseCredentials(\r\n env: EnvironmentConfig\r\n): Promise<ClickHouseCredentials> {\r\n const apiUrl = hostToHttpUrl(env.host);\r\n const uriPath = '/api/beamo/otel/auth/writer/config';\r\n const configUrl = new URL(uriPath, apiUrl).toString();\r\n \r\n // Get secret from environment (required for signed requests)\r\n const secret = process.env.SECRET;\r\n if (!secret) {\r\n throw new Error('SECRET environment variable is required to fetch ClickHouse credentials');\r\n }\r\n \r\n // Calculate signature for signed request (matching C# HttpSignedRequester)\r\n const pathAndQuery = getPathAndQuery(uriPath);\r\n const signature = calculateSignature(env.pid, secret, pathAndQuery, null, '1');\r\n \r\n // Build headers with signed authentication (matching C# HttpSignedRequester)\r\n const headers: Record<string, string> = {\r\n 'Content-Type': 'application/json',\r\n Accept: 'application/json',\r\n 'X-BEAM-SCOPE': `${env.cid}.${env.pid}`,\r\n 'X-BEAM-SIGNATURE': signature,\r\n };\r\n \r\n const response = await fetch(configUrl, {\r\n method: 'GET',\r\n headers,\r\n });\r\n \r\n if (!response.ok) {\r\n const errorText = await response.text().catch(() => 'Unknown error');\r\n throw new Error(`Failed to fetch ClickHouse credentials from ${configUrl}: ${response.status} ${response.statusText} - ${errorText.substring(0, 200)}`);\r\n }\r\n \r\n const credentials = await response.json() as ClickHouseCredentials;\r\n \r\n if (!credentials.endpoint || !credentials.username || !credentials.password) {\r\n throw new Error('Invalid ClickHouse credentials response: missing required fields');\r\n }\r\n \r\n return credentials;\r\n}\r\n\r\n/**\r\n * Gets the collector storage directory (similar to C# LocalApplicationData/beam/collectors/version)\r\n */\r\nfunction getCollectorStoragePath(): string {\r\n // Use temp directory for now - in containers this should be writable\r\n const tempDir = process.env.TMPDIR || process.env.TMP || '/tmp';\r\n return join(tempDir, 'beam', 'collectors', COLLECTOR_VERSION);\r\n}\r\n\r\n/**\r\n * Gets the collector binary name for the current platform\r\n */\r\nfunction getCollectorBinaryName(): string {\r\n const platform = process.platform;\r\n const arch = process.arch;\r\n \r\n // Match C# naming: collector-{osArchSuffix}\r\n // C# returns \"collector-linux-amd64\", not \"beamable-collector-linux-amd64\"\r\n if (platform === 'linux' && arch === 'x64') {\r\n return 'collector-linux-amd64';\r\n } else if (platform === 'linux' && arch === 'arm64') {\r\n return 'collector-linux-arm64';\r\n } else if (platform === 'darwin' && arch === 'x64') {\r\n return 'collector-darwin-amd64';\r\n } else if (platform === 'darwin' && arch === 'arm64') {\r\n return 'collector-darwin-arm64';\r\n } else if (platform === 'win32' && arch === 'x64') {\r\n return 'collector-windows-amd64.exe';\r\n }\r\n \r\n throw new Error(`Unsupported platform: ${platform} ${arch}`);\r\n}\r\n\r\n/**\r\n * Downloads and decompresses a gzipped file\r\n */\r\nasync function downloadAndDecompressGzip(url: string, outputPath: string, makeExecutable: boolean = false): Promise<void> {\r\n let response: Response;\r\n try {\r\n // Add timeout and better error handling\r\n const controller = new AbortController();\r\n const timeoutId = setTimeout(() => controller.abort(), 30000); // 30 second timeout\r\n \r\n try {\r\n response = await fetch(url, {\r\n signal: controller.signal,\r\n headers: {\r\n 'User-Agent': 'Beamable-Node-Microservice-Runtime/1.0',\r\n },\r\n });\r\n clearTimeout(timeoutId);\r\n } catch (fetchError) {\r\n clearTimeout(timeoutId);\r\n if (fetchError instanceof Error) {\r\n // Extract underlying error details from cause\r\n let underlyingError = '';\r\n if ((fetchError as any).cause) {\r\n const cause = (fetchError as any).cause;\r\n if (cause instanceof Error) {\r\n const causeCode = (cause as any).code;\r\n underlyingError = ` (cause: ${cause.name} - ${cause.message}${causeCode ? ` [code: ${causeCode}]` : ''})`;\r\n } else if (typeof cause === 'object' && cause.code) {\r\n underlyingError = ` (cause code: ${cause.code})`;\r\n }\r\n }\r\n \r\n // Provide more context about the error\r\n if (fetchError.name === 'AbortError') {\r\n throw new Error(`Timeout downloading ${url} (30s limit exceeded)${underlyingError}`);\r\n }\r\n \r\n // Check error code if available (Node.js system errors)\r\n const errorCode = (fetchError as any).code || ((fetchError as any).cause?.code);\r\n if (errorCode) {\r\n if (errorCode === 'ENOTFOUND' || errorCode === 'EAI_AGAIN') {\r\n throw new Error(`DNS resolution failed for ${url} (code: ${errorCode}). Check network connectivity and DNS settings.`);\r\n }\r\n if (errorCode === 'ECONNREFUSED') {\r\n throw new Error(`Connection refused to ${url} (code: ${errorCode}). Server may be down or firewall blocking.`);\r\n }\r\n if (errorCode === 'ETIMEDOUT' || errorCode === 'ECONNRESET') {\r\n throw new Error(`Connection timeout/reset to ${url} (code: ${errorCode}). Network may be slow or unreachable.`);\r\n }\r\n if (errorCode === 'CERT_HAS_EXPIRED' || errorCode === 'UNABLE_TO_VERIFY_LEAF_SIGNATURE') {\r\n throw new Error(`SSL/TLS certificate error connecting to ${url} (code: ${errorCode}): ${fetchError.message}`);\r\n }\r\n }\r\n \r\n // Check for common network errors in message\r\n const errorMsg = fetchError.message.toLowerCase();\r\n if (errorMsg.includes('enotfound') || errorMsg.includes('dns')) {\r\n throw new Error(`DNS resolution failed for ${url}. Check network connectivity and DNS settings.${underlyingError}`);\r\n }\r\n if (errorMsg.includes('econnrefused') || errorMsg.includes('connection refused')) {\r\n throw new Error(`Connection refused to ${url}. Server may be down or firewall blocking.${underlyingError}`);\r\n }\r\n if (errorMsg.includes('etimedout') || errorMsg.includes('timeout')) {\r\n throw new Error(`Connection timeout to ${url}. Network may be slow or unreachable.${underlyingError}`);\r\n }\r\n if (errorMsg.includes('certificate') || errorMsg.includes('ssl') || errorMsg.includes('tls')) {\r\n throw new Error(`SSL/TLS error connecting to ${url}: ${fetchError.message}${underlyingError}`);\r\n }\r\n throw new Error(`Network error downloading ${url}: ${fetchError.name} - ${fetchError.message}${underlyingError}`);\r\n }\r\n throw new Error(`Network error downloading ${url}: ${String(fetchError)}`);\r\n }\r\n } catch (error) {\r\n // Re-throw with more context\r\n if (error instanceof Error) {\r\n if (error.message.includes('Timeout') || error.message.includes('DNS') || error.message.includes('Connection')) {\r\n throw error; // Already has good context\r\n }\r\n throw new Error(`Network error downloading ${url}: ${error.message}`);\r\n }\r\n throw new Error(`Network error downloading ${url}: ${String(error)}`);\r\n }\r\n \r\n if (!response.ok) {\r\n const errorText = await response.text().catch(() => '');\r\n throw new Error(`HTTP ${response.status} downloading ${url}: ${response.statusText}${errorText ? ` - ${errorText.substring(0, 200)}` : ''}`);\r\n }\r\n \r\n const dir = dirname(outputPath);\r\n if (!existsSync(dir)) {\r\n mkdirSync(dir, { recursive: true });\r\n }\r\n \r\n const gunzip = createGunzip();\r\n const writeStream = createWriteStream(outputPath);\r\n \r\n await pipeline(response.body as any, gunzip, writeStream);\r\n \r\n if (makeExecutable && process.platform !== 'win32') {\r\n try {\r\n chmodSync(outputPath, 0o755);\r\n } catch (error) {\r\n console.error(`Failed to make ${outputPath} executable:`, error);\r\n }\r\n }\r\n}\r\n\r\n/**\r\n * Resolves the collector binary and config, downloading if needed\r\n */\r\nasync function resolveCollector(allowDownload: boolean = true, logger?: Logger): Promise<{ binaryPath: string | null; configPath: string | null }> {\r\n const basePath = getCollectorStoragePath();\r\n const binaryName = getCollectorBinaryName();\r\n const configName = 'clickhouse-config.yaml';\r\n \r\n const binaryPath = join(basePath, binaryName);\r\n const configPath = join(basePath, configName);\r\n \r\n const itemsToDownload: Array<{ url: string; path: string; executable: boolean }> = [];\r\n \r\n if (!existsSync(binaryPath) && allowDownload) {\r\n const binaryUrl = `${COLLECTOR_DOWNLOAD_BASE}/${binaryName}.gz`;\r\n itemsToDownload.push({ url: binaryUrl, path: binaryPath, executable: true });\r\n logger?.info(`[Collector] Will download binary from: ${binaryUrl}`);\r\n } else if (existsSync(binaryPath)) {\r\n logger?.info(`[Collector] Binary found at: ${binaryPath}`);\r\n }\r\n \r\n if (!existsSync(configPath) && allowDownload) {\r\n const configUrl = `${COLLECTOR_DOWNLOAD_BASE}/${configName}.gz`;\r\n itemsToDownload.push({ url: configUrl, path: configPath, executable: false });\r\n logger?.info(`[Collector] Will download config from: ${configUrl}`);\r\n } else if (existsSync(configPath)) {\r\n logger?.info(`[Collector] Config found at: ${configPath}`);\r\n }\r\n \r\n // Download items in parallel for faster startup (binary and config simultaneously)\r\n if (itemsToDownload.length > 0) {\r\n logger?.info(`[Collector] Downloading ${itemsToDownload.length} item(s) in parallel...`);\r\n const downloadPromises = itemsToDownload.map(async (item) => {\r\n try {\r\n logger?.info(`[Collector] Downloading ${item.url}...`);\r\n await downloadAndDecompressGzip(item.url, item.path, item.executable);\r\n logger?.info(`[Collector] Downloaded to ${item.path}`);\r\n } catch (error) {\r\n // Don't wrap the error again if it already contains the URL and details\r\n const errorMsg = error instanceof Error ? error.message : String(error);\r\n logger?.error(`[Collector] Failed to download ${item.url}: ${errorMsg}`);\r\n // Only wrap if the error doesn't already contain the URL\r\n if (error instanceof Error && errorMsg.includes(item.url)) {\r\n throw error; // Error already has URL and context\r\n }\r\n throw new Error(`Failed to download collector ${item.executable ? 'binary' : 'config'} from ${item.url}: ${errorMsg}`);\r\n }\r\n });\r\n \r\n // Wait for all downloads to complete (or fail)\r\n await Promise.all(downloadPromises);\r\n }\r\n \r\n return {\r\n binaryPath: existsSync(binaryPath) ? binaryPath : null,\r\n configPath: existsSync(configPath) ? configPath : null,\r\n };\r\n}\r\n\r\n/**\r\n * Discovers collector via UDP broadcast\r\n */\r\nfunction discoverCollectorViaUDP(timeoutMs: number = 5000): Promise<CollectorDiscoveryEntry | null> {\r\n return new Promise((resolve) => {\r\n const socket = dgram.createSocket('udp4');\r\n const discovered: CollectorDiscoveryEntry[] = [];\r\n let timeout: NodeJS.Timeout;\r\n \r\n socket.on('message', (msg) => {\r\n try {\r\n const message = JSON.parse(msg.toString()) as CollectorDiscoveryEntry;\r\n // Check if version matches\r\n if (message.version === COLLECTOR_VERSION && message.status === 'READY') {\r\n discovered.push(message);\r\n }\r\n } catch (error) {\r\n // Ignore parse errors\r\n }\r\n });\r\n \r\n socket.on('error', () => {\r\n clearTimeout(timeout);\r\n socket.close();\r\n resolve(null);\r\n });\r\n \r\n socket.bind(() => {\r\n socket.setBroadcast(true);\r\n \r\n timeout = setTimeout(() => {\r\n socket.close();\r\n // Return the first discovered collector\r\n resolve(discovered.length > 0 ? discovered[0] : null);\r\n }, timeoutMs);\r\n });\r\n });\r\n}\r\n\r\n/**\r\n * Gets the current status of ClickHouse credentials (from env or API)\r\n */\r\nexport function getClickHouseCredentialsStatus(): {\r\n hasEndpoint: boolean;\r\n hasUsername: boolean;\r\n hasPassword: boolean;\r\n source: 'environment' | 'api' | 'missing';\r\n} {\r\n const hasEndpoint = !!process.env.BEAM_CLICKHOUSE_ENDPOINT;\r\n const hasUsername = !!process.env.BEAM_CLICKHOUSE_USERNAME;\r\n const hasPassword = !!process.env.BEAM_CLICKHOUSE_PASSWORD;\r\n \r\n if (hasEndpoint && hasUsername && hasPassword) {\r\n return {\r\n hasEndpoint: true,\r\n hasUsername: true,\r\n hasPassword: true,\r\n source: 'environment',\r\n };\r\n }\r\n \r\n return {\r\n hasEndpoint,\r\n hasUsername,\r\n hasPassword,\r\n source: 'missing',\r\n };\r\n}\r\n\r\n/**\r\n * Checks if collector is already running via UDP discovery\r\n */\r\nexport async function isCollectorRunning(): Promise<CollectorStatus> {\r\n // First, check if we have a global collector process reference\r\n // If the process is running and we've seen \"Everything is ready\" in stderr, it's ready\r\n if (globalCollectorProcess) {\r\n // Check if process is still alive\r\n try {\r\n // Process is alive if we can access it and it hasn't exited\r\n const processAlive = globalCollectorProcess.exitCode === null && \r\n globalCollectorProcess.killed === false;\r\n \r\n if (processAlive) {\r\n // Check if we've seen the \"Everything is ready\" message in stderr\r\n // This is a reliable indicator that the collector is ready\r\n const stderrText = globalCollectorStderr.join('\\n');\r\n const isReady = stderrText.includes('Everything is ready') || \r\n stderrText.includes('Begin running and processing data');\r\n \r\n if (isReady) {\r\n // Try UDP discovery to get the endpoint, but if it fails, use the configured endpoint\r\n try {\r\n const discovered = await discoverCollectorViaUDP(1000); // Shorter timeout since we know it's running\r\n if (discovered && discovered.status === 'READY') {\r\n return {\r\n isRunning: true,\r\n isReady: true,\r\n pid: discovered.pid || globalCollectorProcess.pid || 0,\r\n otlpEndpoint: discovered.otlpEndpoint,\r\n version: discovered.version,\r\n };\r\n }\r\n } catch {\r\n // UDP discovery failed, but process is alive and has \"Everything is ready\" message\r\n // Use the configured endpoint as fallback\r\n }\r\n \r\n // Process is alive and we've seen \"Everything is ready\" - treat as ready\r\n // Extract endpoint from environment or use default\r\n const configuredEndpoint = process.env.BEAM_OTLP_HTTP_ENDPOINT || '0.0.0.0:4318';\r\n return {\r\n isRunning: true,\r\n isReady: true,\r\n pid: globalCollectorProcess.pid || 0,\r\n otlpEndpoint: configuredEndpoint,\r\n version: COLLECTOR_VERSION,\r\n };\r\n }\r\n }\r\n } catch {\r\n // Process might have exited or is invalid\r\n }\r\n }\r\n \r\n // Try UDP discovery as fallback\r\n try {\r\n const discovered = await discoverCollectorViaUDP(2000);\r\n if (discovered) {\r\n return {\r\n isRunning: true,\r\n isReady: discovered.status === 'READY',\r\n pid: discovered.pid,\r\n otlpEndpoint: discovered.otlpEndpoint,\r\n version: discovered.version,\r\n };\r\n }\r\n } catch (error) {\r\n // Discovery failed, collector probably not running\r\n }\r\n \r\n return {\r\n isRunning: false,\r\n isReady: false,\r\n pid: 0,\r\n };\r\n}\r\n\r\n/**\r\n * Sets default collector configuration environment variables in process.env\r\n * Matches C# CollectorManager.AddCollectorConfigurationToEnvironment()\r\n */\r\nfunction addCollectorConfigurationToEnvironment(): void {\r\n const defaults: Record<string, string> = {\r\n BEAM_CLICKHOUSE_PROCESSOR_TIMEOUT: '5s',\r\n BEAM_CLICKHOUSE_PROCESSOR_BATCH_SIZE: '5000',\r\n BEAM_CLICKHOUSE_EXPORTER_TIMEOUT: '5s',\r\n BEAM_CLICKHOUSE_EXPORTER_QUEUE_SIZE: '1000',\r\n BEAM_CLICKHOUSE_EXPORTER_RETRY_ENABLED: 'true',\r\n BEAM_CLICKHOUSE_EXPORTER_RETRY_INITIAL_INTERVAL: '5s',\r\n BEAM_CLICKHOUSE_EXPORTER_RETRY_MAX_INTERVAL: '30s',\r\n BEAM_CLICKHOUSE_EXPORTER_RETRY_MAX_ELAPSED_TIME: '300s',\r\n };\r\n \r\n // Only set if not already set (matching C# behavior)\r\n for (const [key, defaultValue] of Object.entries(defaults)) {\r\n if (!process.env[key]) {\r\n process.env[key] = defaultValue;\r\n }\r\n }\r\n}\r\n\r\n/**\r\n * Sets ClickHouse credentials in process.env (matching C# CollectorManager.AddAuthEnvironmentVars)\r\n */\r\nexport function addAuthEnvironmentVars(endpoint: string, username: string, password: string): void {\r\n // Always set the values when explicitly provided (matching C# behavior)\r\n // This ensures credentials fetched from API override any existing (potentially empty) values\r\n // In Node.js, process.env is a writable object - setting properties here persists for the lifetime of the process\r\n process.env.BEAM_CLICKHOUSE_ENDPOINT = endpoint;\r\n process.env.BEAM_CLICKHOUSE_USERNAME = username;\r\n process.env.BEAM_CLICKHOUSE_PASSWORD = password;\r\n \r\n // Verify they were actually set (for debugging)\r\n if (!process.env.BEAM_CLICKHOUSE_ENDPOINT || !process.env.BEAM_CLICKHOUSE_USERNAME || !process.env.BEAM_CLICKHOUSE_PASSWORD) {\r\n throw new Error(`Failed to set ClickHouse credentials in process.env - this should never happen in Node.js`);\r\n }\r\n}\r\n\r\n/**\r\n * Starts the collector and waits for it to be ready before returning.\r\n * This ensures Portal logs (structured logs via OTLP) only start appearing AFTER collector is ready.\r\n * Returns the OTLP endpoint when ready, or null if it times out or fails.\r\n */\r\nexport async function startCollectorAndWaitForReady(\r\n env: EnvironmentConfig,\r\n _timeoutMs?: number // Not used - kept for API compatibility\r\n): Promise<string | null> {\r\n // Match C# logic: (this.InDocker() || UseLocalOtel) && !BEAM_DISABLE_STANDARD_OTEL\r\n const isInDocker = process.env.IS_LOCAL !== '1' && process.env.IS_LOCAL !== 'true';\r\n const useLocalOtel = !!process.env.BEAM_LOCAL_OTEL;\r\n const standardOtelEnabled = (isInDocker || useLocalOtel) && !process.env.BEAM_DISABLE_STANDARD_OTEL;\r\n const hasExplicitEndpoint = !!process.env.BEAM_OTEL_EXPORTER_OTLP_ENDPOINT;\r\n\r\n // If OTLP is not needed, return immediately\r\n if (!standardOtelEnabled && !hasExplicitEndpoint) {\r\n return null;\r\n }\r\n\r\n // Create a minimal console logger for initialization messages (no OTLP yet)\r\n const initLogger = pino({\r\n name: 'beamable-otlp-init',\r\n level: 'info',\r\n }, process.stdout);\r\n\r\n initLogger.info('[OTLP] Setting up collector (waiting for readiness before enabling Portal logs)...');\r\n\r\n // Simple linear async/await - no timeouts, no Promise.race complexity\r\n try {\r\n const endpoint = await discoverOrStartCollector(initLogger, standardOtelEnabled, env);\r\n \r\n if (endpoint) {\r\n initLogger.info(`[OTLP] Collector ready at ${endpoint}. Portal logs now enabled.`);\r\n return endpoint;\r\n } else {\r\n initLogger.warn('[OTLP] Collector setup failed, continuing without Portal logs');\r\n return null;\r\n }\r\n } catch (error) {\r\n const errorMsg = error instanceof Error ? error.message : String(error);\r\n initLogger.error(`[OTLP] Collector setup failed: ${errorMsg}`);\r\n return null;\r\n }\r\n}\r\n\r\n/**\r\n * Starts the collector asynchronously in the background (non-blocking).\r\n * This allows the service to start immediately while collector downloads/starts.\r\n * Returns immediately - collector setup happens in background.\r\n * @deprecated Use startCollectorAndWaitForReady() to ensure Portal logs only appear after collector is ready\r\n */\r\nexport function startCollectorAsync(env: EnvironmentConfig): void {\r\n // Match C# logic: (this.InDocker() || UseLocalOtel) && !BEAM_DISABLE_STANDARD_OTEL\r\n const isInDocker = process.env.IS_LOCAL !== '1' && process.env.IS_LOCAL !== 'true';\r\n const useLocalOtel = !!process.env.BEAM_LOCAL_OTEL;\r\n const standardOtelEnabled = (isInDocker || useLocalOtel) && !process.env.BEAM_DISABLE_STANDARD_OTEL;\r\n const hasExplicitEndpoint = !!process.env.BEAM_OTEL_EXPORTER_OTLP_ENDPOINT;\r\n\r\n // If OTLP is not needed, return immediately\r\n if (!standardOtelEnabled && !hasExplicitEndpoint) {\r\n return;\r\n }\r\n\r\n // Create a minimal console logger for initialization messages (no OTLP yet)\r\n const initLogger = pino({\r\n name: 'beamable-otlp-init',\r\n level: 'info',\r\n }, process.stdout);\r\n\r\n initLogger.info('[OTLP] Starting collector setup in background (non-blocking)...');\r\n\r\n // Start collector setup asynchronously - don't wait for it\r\n // The service can start immediately, collector will connect when ready\r\n discoverOrStartCollector(initLogger, standardOtelEnabled, env)\r\n .then((result) => {\r\n if (result) {\r\n initLogger.info(`[OTLP] Collector setup complete in background, endpoint: ${result}`);\r\n // Update the global logger provider if it exists to enable OTLP\r\n // The logger will automatically start using OTLP once the provider is set\r\n } else {\r\n initLogger.warn('[OTLP] Collector setup returned null, OTLP logging will be disabled');\r\n }\r\n })\r\n .catch((error) => {\r\n const errorMsg = error instanceof Error ? error.message : String(error);\r\n initLogger.error(`[OTLP] Failed to setup collector in background: ${errorMsg}`);\r\n });\r\n \r\n // Return immediately - service can start serving requests\r\n}\r\n\r\n/**\r\n * Sets up the collector BEFORE logging is initialized (DEPRECATED - blocking approach).\r\n * This function is kept for backwards compatibility but is no longer recommended.\r\n * Use startCollectorAsync() for non-blocking startup instead.\r\n * @deprecated Use startCollectorAsync() for non-blocking startup\r\n */\r\nexport function setupCollectorBeforeLogging(\r\n env: EnvironmentConfig,\r\n timeoutMs: number = 60000\r\n): string | null {\r\n // Match C# logic: (this.InDocker() || UseLocalOtel) && !BEAM_DISABLE_STANDARD_OTEL\r\n const isInDocker = process.env.IS_LOCAL !== '1' && process.env.IS_LOCAL !== 'true';\r\n const useLocalOtel = !!process.env.BEAM_LOCAL_OTEL;\r\n const standardOtelEnabled = (isInDocker || useLocalOtel) && !process.env.BEAM_DISABLE_STANDARD_OTEL;\r\n const hasExplicitEndpoint = !!process.env.BEAM_OTEL_EXPORTER_OTLP_ENDPOINT;\r\n\r\n // If OTLP is not needed, return immediately\r\n if (!standardOtelEnabled && !hasExplicitEndpoint) {\r\n return null;\r\n }\r\n\r\n // Create a minimal console logger for initialization messages (no OTLP yet)\r\n const initLogger = pino({\r\n name: 'beamable-otlp-init',\r\n level: 'info',\r\n }, process.stdout);\r\n\r\n initLogger.info('[OTLP] Setting up collector before logging initialization...');\r\n\r\n // Use deasync to wait synchronously for collector setup\r\n let endpoint: string | null = null;\r\n let completed = false;\r\n let setupError: string | null = null;\r\n\r\n // Start collector setup promise\r\n discoverOrStartCollector(initLogger, standardOtelEnabled, env)\r\n .then((result) => {\r\n endpoint = result;\r\n completed = true;\r\n if (result) {\r\n initLogger.info(`[OTLP] Collector setup complete, endpoint: ${result}`);\r\n } else {\r\n initLogger.warn('[OTLP] Collector setup returned null, OTLP logging will be disabled');\r\n }\r\n return result;\r\n })\r\n .catch((error) => {\r\n setupError = error instanceof Error ? error.message : String(error);\r\n completed = true;\r\n initLogger.error(`[OTLP] Failed to setup collector: ${setupError}`);\r\n endpoint = null;\r\n });\r\n\r\n // Wait synchronously for collector setup to complete\r\n const startTime = Date.now();\r\n const timeoutId = setTimeout(() => {\r\n if (!completed) {\r\n initLogger.warn(`[OTLP] Collector setup timeout after ${timeoutMs}ms, continuing without OTLP`);\r\n completed = true;\r\n }\r\n }, timeoutMs);\r\n\r\n try {\r\n // Use deasync to wait for completion (allows event loop to process async operations)\r\n deasync.loopWhile(() => {\r\n const elapsed = Date.now() - startTime;\r\n if (elapsed >= timeoutMs) {\r\n return false; // Stop waiting\r\n }\r\n return !completed; // Continue waiting if not completed\r\n });\r\n } catch (error) {\r\n initLogger.error(`[OTLP] Error during collector setup wait: ${error instanceof Error ? error.message : String(error)}`);\r\n }\r\n\r\n clearTimeout(timeoutId);\r\n\r\n if (completed && endpoint) {\r\n initLogger.info('[OTLP] Collector is ready, proceeding with logger creation');\r\n return endpoint;\r\n } else if (setupError) {\r\n initLogger.error(`[OTLP] Collector setup failed: ${setupError}, continuing without OTLP`);\r\n return null;\r\n } else {\r\n initLogger.warn('[OTLP] Collector setup did not complete in time, continuing without OTLP');\r\n return null;\r\n }\r\n}\r\n\r\n/**\r\n * Starts the OpenTelemetry collector process\r\n */\r\nexport async function startCollector(\r\n logger: Logger,\r\n otlpEndpoint?: string,\r\n env?: EnvironmentConfig\r\n): Promise<{ process: ChildProcess; endpoint: string }> {\r\n // Clear any previous errors\r\n globalCollectorStartError = null;\r\n globalCollectorInitError = null;\r\n \r\n // CRITICAL: Fetch credentials FIRST, before resolving collector binary/config\r\n // This ensures credentials are in process.env even if collector download fails\r\n // The credentials will persist in process.env for the lifetime of the process.\r\n // Per Gabriel: \"these OTEL related ones you need to do it yourself at the beginning of the microservice startup\"\r\n let clickhouseEndpoint = process.env.BEAM_CLICKHOUSE_ENDPOINT;\r\n let clickhouseUsername = process.env.BEAM_CLICKHOUSE_USERNAME;\r\n let clickhousePassword = process.env.BEAM_CLICKHOUSE_PASSWORD;\r\n \r\n if ((!clickhouseEndpoint || !clickhouseUsername || !clickhousePassword) && env) {\r\n try {\r\n logger.info('[Collector] Fetching ClickHouse credentials from Beamable API...');\r\n const credentials = await fetchClickHouseCredentials(env);\r\n clickhouseEndpoint = credentials.endpoint;\r\n clickhouseUsername = credentials.username;\r\n clickhousePassword = credentials.password;\r\n \r\n // CRITICAL: Set them in process.env IMMEDIATELY after fetching (matching C# CollectorManager.AddAuthEnvironmentVars)\r\n // In Node.js, process.env is a writable object - setting properties here persists for the lifetime of the process\r\n // This ensures they're available to both the Node.js process and inherited by the collector child process\r\n addAuthEnvironmentVars(clickhouseEndpoint, clickhouseUsername, clickhousePassword);\r\n \r\n // Verify they were set (defensive check - should never fail in Node.js)\r\n const verifyEndpoint = process.env.BEAM_CLICKHOUSE_ENDPOINT;\r\n const verifyUsername = process.env.BEAM_CLICKHOUSE_USERNAME;\r\n const verifyPassword = process.env.BEAM_CLICKHOUSE_PASSWORD;\r\n \r\n if (!verifyEndpoint || !verifyUsername || !verifyPassword) {\r\n logger.error(`[Collector] CRITICAL: Credentials were set but are missing from process.env! This should never happen.`);\r\n throw new Error('Failed to persist ClickHouse credentials in process.env');\r\n }\r\n \r\n logger.info('[Collector] ClickHouse credentials fetched from API and verified in process.env');\r\n } catch (error) {\r\n const errorMsg = `[Collector] Failed to fetch ClickHouse credentials from API: ${error instanceof Error ? error.message : String(error)}`;\r\n logger.error(errorMsg);\r\n throw new Error(errorMsg);\r\n }\r\n }\r\n \r\n // Validate required environment variables (matching C# behavior)\r\n // These must be set before starting the collector, otherwise it will fail\r\n if (!clickhouseEndpoint || !clickhouseUsername || !clickhousePassword) {\r\n const errorMsg = `[Collector] Required ClickHouse credentials are missing. Set BEAM_CLICKHOUSE_ENDPOINT, BEAM_CLICKHOUSE_USERNAME, and BEAM_CLICKHOUSE_PASSWORD, or ensure the API endpoint is accessible.`;\r\n logger.error(errorMsg);\r\n throw new Error(errorMsg);\r\n }\r\n \r\n // Now resolve collector binary and config (after credentials are fetched and set)\r\n logger.info('[Collector] Resolving collector binary and config...');\r\n const collectorInfo = await resolveCollector(true, logger);\r\n \r\n if (!collectorInfo.binaryPath) {\r\n logger.error('[Collector] Binary not found and download failed');\r\n throw new Error('Collector binary not found and download failed');\r\n }\r\n \r\n if (!collectorInfo.configPath) {\r\n logger.error('[Collector] Config not found and download failed');\r\n throw new Error('Collector config not found and download failed');\r\n }\r\n \r\n logger.info(`[Collector] Using binary: ${collectorInfo.binaryPath}`);\r\n logger.info(`[Collector] Using config: ${collectorInfo.configPath}`);\r\n \r\n // Verify binary exists\r\n if (!collectorInfo.binaryPath || !existsSync(collectorInfo.binaryPath)) {\r\n throw new Error(`Collector binary not found at ${collectorInfo.binaryPath}`);\r\n }\r\n \r\n // Check if binary is executable (Unix only - Windows doesn't have execute permission)\r\n if (process.platform !== 'win32') {\r\n try {\r\n const fsPromises = await import('fs/promises');\r\n const stats = await fsPromises.stat(collectorInfo.binaryPath);\r\n const mode = Number(stats.mode);\r\n const isExecutable = !!(mode & parseInt('111', 8)); // Check execute bit\r\n if (!isExecutable) {\r\n logger.warn(`[Collector] Binary may not be executable. Mode: ${mode.toString(8)}`);\r\n // Try to make it executable again\r\n try {\r\n chmodSync(collectorInfo.binaryPath, 0o755);\r\n logger.info(`[Collector] Made binary executable`);\r\n } catch (chmodError) {\r\n logger.error(`[Collector] Failed to make binary executable: ${chmodError instanceof Error ? chmodError.message : String(chmodError)}`);\r\n }\r\n }\r\n } catch (statError) {\r\n logger.warn(`[Collector] Could not check binary permissions: ${statError instanceof Error ? statError.message : String(statError)}`);\r\n }\r\n }\r\n\r\n // Set default collector configuration environment variables in process.env\r\n // This ensures they're available to both the Node.js process and the collector child process\r\n // Matches C# CollectorManager.AddCollectorConfigurationToEnvironment()\r\n addCollectorConfigurationToEnvironment();\r\n \r\n // CRITICAL: Set BEAM_COLLECTOR_DISCOVERY_PORT if not already set (matching C# AddDefaultCollectorHostAndPortFallback)\r\n // This MUST be set before starting the collector\r\n if (!process.env.BEAM_COLLECTOR_DISCOVERY_PORT) {\r\n process.env.BEAM_COLLECTOR_DISCOVERY_PORT = String(DISCOVERY_PORT);\r\n logger.info(`[Collector] Set BEAM_COLLECTOR_DISCOVERY_PORT=${DISCOVERY_PORT} (default)`);\r\n }\r\n \r\n // Determine OTLP endpoint\r\n // Use a free port if not specified (like C# does with PortUtil.FreeEndpoint())\r\n let localEndpoint = otlpEndpoint;\r\n if (!localEndpoint) {\r\n // For now, use default OTLP HTTP port\r\n // In production, this would be discovered or set by Beamable\r\n localEndpoint = '0.0.0.0:4318';\r\n }\r\n localEndpoint = localEndpoint.replace(/^http:\\/\\//, '').replace(/^https:\\/\\//, '');\r\n \r\n // Find a free port for Prometheus (matching C# PortUtil.FreeTcpPort())\r\n // We'll use a simple approach: bind to port 0 and let the OS assign a port\r\n // For now, use a default port (9090 is common for Prometheus) or find a free one\r\n // C# uses PortUtil.FreeTcpPort() which finds an available TCP port\r\n // We'll use a simple helper to find a free port\r\n const prometheusPort = await findFreePort();\r\n \r\n // Set environment variables for collector\r\n // Note: BEAM_CLICKHOUSE_ENDPOINT is for collector → ClickHouse communication\r\n // This is different from the OTLP endpoint (microservice → collector)\r\n // The collector process will inherit all environment variables from process.env automatically\r\n // We only need to set the OTLP endpoint, discovery port, and Prometheus port explicitly here\r\n const collectorEnv: NodeJS.ProcessEnv = {\r\n ...process.env, // Inherit all process.env (including credentials and config we just set)\r\n BEAM_OTLP_HTTP_ENDPOINT: localEndpoint,\r\n BEAM_COLLECTOR_DISCOVERY_PORT: String(DISCOVERY_PORT),\r\n BEAM_COLLECTOR_PROMETHEUS_PORT: String(prometheusPort),\r\n };\r\n \r\n // Start collector process\r\n const collectorProcess = spawn(collectorInfo.binaryPath, ['--config', collectorInfo.configPath], {\r\n env: collectorEnv,\r\n stdio: ['ignore', 'pipe', 'pipe'],\r\n detached: false,\r\n });\r\n \r\n // Store globally to prevent GC and track status\r\n globalCollectorProcess = collectorProcess;\r\n globalCollectorStartError = null;\r\n globalCollectorExitCode = null;\r\n globalCollectorStderr = [];\r\n \r\n // Set up handlers BEFORE we check if process started, so we capture all output\r\n collectorProcess.stdout?.on('data', (data) => {\r\n const outputLine = data.toString().trim();\r\n // Log stdout at info level to see collector output\r\n logger.info(`[Collector OUT] ${outputLine}`);\r\n });\r\n \r\n collectorProcess.stderr?.on('data', (data) => {\r\n const errorLine = data.toString().trim();\r\n globalCollectorStderr.push(errorLine);\r\n // Keep only last 50 lines to prevent memory issues\r\n if (globalCollectorStderr.length > 50) {\r\n globalCollectorStderr.shift();\r\n }\r\n // Log stderr at info level so we can see why collector crashes\r\n logger.info(`[Collector ERR] ${errorLine}`);\r\n });\r\n \r\n collectorProcess.on('error', (err) => {\r\n globalCollectorStartError = err.message;\r\n logger.error(`[Collector] Failed to start: ${err.message}`);\r\n });\r\n \r\n collectorProcess.on('exit', (code, signal) => {\r\n globalCollectorExitCode = code;\r\n logger.warn(`[Collector] Process exited with code ${code}${signal ? `, signal ${signal}` : ''}`);\r\n \r\n // If process exited with an error code, log accumulated stderr\r\n if (code !== 0 && globalCollectorStderr.length > 0) {\r\n logger.error(`[Collector] Exit error - Last ${Math.min(10, globalCollectorStderr.length)} stderr lines:`);\r\n globalCollectorStderr.slice(-10).forEach((line) => {\r\n logger.error(`[Collector ERR] ${line}`);\r\n });\r\n }\r\n \r\n // Don't clear reference immediately - keep it for status checks\r\n // globalCollectorProcess = null; // Keep reference for getCollectorProcessStatus()\r\n });\r\n \r\n logger.info(`[Collector] Started with PID ${collectorProcess.pid}, endpoint: ${localEndpoint}`);\r\n \r\n // Give the process a moment to start and potentially write to stderr\r\n // This helps capture immediate startup errors\r\n // Wait a bit longer to ensure we catch immediate crashes\r\n await new Promise(resolve => setTimeout(resolve, 500));\r\n \r\n // Check if process already exited (immediate crash)\r\n if (globalCollectorExitCode !== null && globalCollectorExitCode !== 0) {\r\n const stderrMsg = globalCollectorStderr.length > 0 \r\n ? ` Stderr: ${globalCollectorStderr.join('; ')}` \r\n : ' No stderr output captured.';\r\n throw new Error(`Collector process exited immediately after startup with code ${globalCollectorExitCode}.${stderrMsg}`);\r\n }\r\n \r\n return {\r\n process: collectorProcess,\r\n endpoint: `http://${localEndpoint}`,\r\n };\r\n}\r\n\r\n/**\r\n * Gets detailed collector status including process info and errors\r\n */\r\nexport function getCollectorProcessStatus(): {\r\n hasProcess: boolean;\r\n pid: number | null;\r\n exitCode: number | null;\r\n startError: string | null;\r\n initError: string | null;\r\n stderr: string[];\r\n} {\r\n return {\r\n hasProcess: globalCollectorProcess !== null,\r\n pid: globalCollectorProcess?.pid ?? null,\r\n exitCode: globalCollectorExitCode,\r\n startError: globalCollectorStartError,\r\n initError: globalCollectorInitError,\r\n stderr: [...globalCollectorStderr], // Return a copy\r\n };\r\n}\r\n\r\n/**\r\n * Discovers or starts the collector and returns the OTLP endpoint\r\n */\r\nexport async function discoverOrStartCollector(\r\n logger: Logger,\r\n standardOtelEnabled: boolean,\r\n env?: EnvironmentConfig\r\n): Promise<string | null> {\r\n if (!standardOtelEnabled) {\r\n return null;\r\n }\r\n \r\n // CRITICAL: Check if collector startup is already in progress\r\n // This prevents duplicate collector starts if this function is called multiple times\r\n // (e.g., if setupCollectorBeforeLogging times out but the promise is still running)\r\n if (globalCollectorStartupPromise) {\r\n logger.info('[Collector] Collector startup already in progress, waiting for existing startup to complete...');\r\n try {\r\n const result = await globalCollectorStartupPromise;\r\n // Clear the promise after it completes (success or failure)\r\n globalCollectorStartupPromise = null;\r\n return result;\r\n } catch (error) {\r\n logger.error(`[Collector] Existing startup promise failed: ${error instanceof Error ? error.message : String(error)}`);\r\n // Clear the promise so we can retry\r\n globalCollectorStartupPromise = null;\r\n // Fall through to start a new one\r\n }\r\n }\r\n \r\n // CRITICAL: Check if we already have a collector process starting/running\r\n // This prevents duplicate collector starts if this function is called multiple times\r\n let existingEndpoint: string | undefined;\r\n if (globalCollectorProcess) {\r\n // Check if process is still alive\r\n const processAlive = globalCollectorProcess.exitCode === null && \r\n globalCollectorProcess.killed === false;\r\n \r\n if (processAlive) {\r\n logger.info(`[Collector] Collector process already exists (PID ${globalCollectorProcess.pid}), waiting for it to be ready...`);\r\n // Use the configured endpoint from environment (we started it earlier)\r\n existingEndpoint = process.env.BEAM_OTLP_HTTP_ENDPOINT ? \r\n `http://${process.env.BEAM_OTLP_HTTP_ENDPOINT}` : \r\n 'http://0.0.0.0:4318';\r\n // Fall through to the wait logic below (don't start a new collector)\r\n } else {\r\n // Process is dead, clear it and start fresh\r\n logger.warn(`[Collector] Previous collector process (PID ${globalCollectorProcess.pid}) is dead, starting new one...`);\r\n globalCollectorProcess = null;\r\n globalCollectorExitCode = null;\r\n globalCollectorStderr = [];\r\n }\r\n }\r\n \r\n // First, check if collector is already running (via UDP discovery)\r\n if (!globalCollectorProcess) {\r\n const status = await isCollectorRunning();\r\n if (status.isRunning && status.isReady && status.otlpEndpoint) {\r\n logger.info(`[Collector] Found running collector at ${status.otlpEndpoint}`);\r\n return `http://${status.otlpEndpoint}`;\r\n }\r\n }\r\n \r\n // Collector not running - start it (or wait for existing one to be ready)\r\n // Wrap the entire startup logic in a promise that we track globally\r\n // This prevents duplicate starts if this function is called multiple times\r\n const startupPromise = (async (): Promise<string | null> => {\r\n try {\r\n // Clear any previous init error\r\n globalCollectorInitError = null;\r\n \r\n let endpoint: string;\r\n if (existingEndpoint) {\r\n // Collector already starting, just wait for it to be ready\r\n endpoint = existingEndpoint;\r\n logger.info(`[Collector] Waiting for existing collector to become ready at ${endpoint}...`);\r\n } else {\r\n // Start a new collector\r\n logger.info('[Collector] Starting OpenTelemetry collector...');\r\n const startResult = await startCollector(logger, undefined, env);\r\n endpoint = startResult.endpoint;\r\n \r\n // Check if collector process exited immediately (crashed)\r\n // Wait a bit longer to see if it crashes right after starting\r\n await new Promise(resolve => setTimeout(resolve, 200));\r\n \r\n if (globalCollectorExitCode !== null && globalCollectorExitCode !== 0) {\r\n const errorMsg = `Collector process exited immediately with code ${globalCollectorExitCode}. ${globalCollectorStderr.length > 0 ? `Stderr: ${globalCollectorStderr.join('; ')}` : 'No stderr output.'}`;\r\n globalCollectorInitError = errorMsg;\r\n logger.error(`[Collector] ${errorMsg}`);\r\n return null;\r\n }\r\n }\r\n \r\n // CRITICAL: Wait for collector to be fully ready before returning\r\n // We'll wait up to 60 seconds, checking every 500ms\r\n // This ensures the collector is actually ready to receive logs before we continue\r\n const maxWaitTime = 60000; // 60 seconds\r\n const checkInterval = 500; // Check every 500ms\r\n const maxChecks = Math.floor(maxWaitTime / checkInterval);\r\n \r\n logger.info('[Collector] Waiting for collector to become ready...');\r\n \r\n for (let i = 0; i < maxChecks; i++) {\r\n await new Promise(resolve => setTimeout(resolve, checkInterval));\r\n \r\n // Check if process exited during wait\r\n if (globalCollectorExitCode !== null && globalCollectorExitCode !== 0) {\r\n const errorMsg = `Collector process exited during startup with code ${globalCollectorExitCode}. ${globalCollectorStderr.length > 0 ? `Stderr: ${globalCollectorStderr.join('; ')}` : 'No stderr output.'}`;\r\n globalCollectorInitError = errorMsg;\r\n logger.error(`[Collector] ${errorMsg}`);\r\n return null;\r\n }\r\n \r\n const newStatus = await isCollectorRunning();\r\n if (newStatus.isRunning && newStatus.isReady) {\r\n logger.info(`[Collector] Collector is ready at ${newStatus.otlpEndpoint || endpoint}`);\r\n return newStatus.otlpEndpoint ? `http://${newStatus.otlpEndpoint}` : endpoint;\r\n }\r\n \r\n // Log progress every 2 seconds\r\n if (i > 0 && i % 4 === 0) {\r\n logger.info(`[Collector] Still waiting for collector to become ready... (${(i * checkInterval) / 1000}s elapsed)`);\r\n }\r\n }\r\n \r\n // Check one more time if process exited\r\n if (globalCollectorExitCode !== null && globalCollectorExitCode !== 0) {\r\n const errorMsg = `Collector process exited with code ${globalCollectorExitCode}. ${globalCollectorStderr.length > 0 ? `Stderr: ${globalCollectorStderr.join('; ')}` : 'No stderr output.'}`;\r\n globalCollectorInitError = errorMsg;\r\n logger.error(`[Collector] ${errorMsg}`);\r\n return null;\r\n }\r\n \r\n // Collector did not become ready within timeout\r\n logger.error(`[Collector] Collector did not become ready within ${maxWaitTime / 1000} seconds`);\r\n return null;\r\n } catch (err) {\r\n const errorMsg = err instanceof Error ? err.message : String(err);\r\n globalCollectorInitError = errorMsg;\r\n logger.error(`[Collector] Failed to start collector: ${errorMsg}`);\r\n return null;\r\n }\r\n })();\r\n \r\n // Store the promise globally so other calls to this function can wait for it\r\n globalCollectorStartupPromise = startupPromise;\r\n \r\n // Clear the promise when it completes (so we don't keep waiting on old promises)\r\n startupPromise.finally(() => {\r\n globalCollectorStartupPromise = null;\r\n });\r\n \r\n return await startupPromise;\r\n}\r\n\r\n"]}
|
|
1
|
+
{"version":3,"file":"collector-manager.js","sourceRoot":"","sources":["../src/collector-manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAgB,MAAM,eAAe,CAAC;AACpD,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AACzE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AACrC,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,MAAM,MAAM,CAAC;AACpC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,KAAK,CAAC;AAEnC,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAmBhD,MAAM,iBAAiB,GAAG,OAAO,CAAC,CAAC,2DAA2D;AAC9F,MAAM,uBAAuB,GAAG,2CAA2C,iBAAiB,EAAE,CAAC;AAC/F,MAAM,cAAc,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,6BAA6B,IAAI,MAAM,EAAE,EAAE,CAAC,CAAC;AAEzF;;GAEG;AACH,SAAS,YAAY;IACnB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,MAAM,GAAG,YAAY,EAAE,CAAC;QAC9B,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,GAAG,EAAE;YACpB,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YAChF,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE;gBAChB,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;oBACb,OAAO,CAAC,IAAI,CAAC,CAAC;gBAChB,CAAC;qBAAM,CAAC;oBACN,qDAAqD;oBACrD,OAAO,CAAC,IAAI,CAAC,CAAC;gBAChB,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QACH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACzB,MAAM,CAAC,GAAG,CAAC,CAAC;QACd,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,mFAAmF;AACnF,IAAI,sBAAsB,GAAwB,IAAI,CAAC;AACvD,IAAI,yBAAyB,GAAkB,IAAI,CAAC;AACpD,IAAI,uBAAuB,GAAkB,IAAI,CAAC;AAClD,IAAI,qBAAqB,GAAa,EAAE,CAAC;AACzC,wEAAwE;AACxE,IAAI,6BAA6B,GAAkC,IAAI,CAAC;AACxE,IAAI,wBAAwB,GAAkB,IAAI,CAAC,CAAC,8CAA8C;AAYlG;;;;GAIG;AACH,SAAS,kBAAkB,CACzB,GAAW,EACX,MAAc,EACd,eAAuB,EACvB,OAAsB,IAAI,EAC1B,UAAkB,GAAG;IAErB,IAAI,UAAU,GAAG,GAAG,MAAM,GAAG,GAAG,GAAG,OAAO,GAAG,eAAe,EAAE,CAAC;IAC/D,IAAI,IAAI,EAAE,CAAC;QACT,UAAU,IAAI,IAAI,CAAC;IACrB,CAAC;IAED,kDAAkD;IAClD,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC3E,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,GAAW;IAClC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,OAAO,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC;IACzC,CAAC;IAAC,MAAM,CAAC;QACP,qCAAqC;QACrC,OAAO,GAAG,CAAC;IACb,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC9C,GAAsB;IAEtB,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,OAAO,GAAG,oCAAoC,CAAC;IACrD,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC;IAEtD,6CAA6C;IAC7C,MAAM,aAAa,GAAG,IAAI,CAAC,EAAE,IAAI,EAAE,mBAAmB,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IACzF,aAAa,CAAC,IAAI,CAAC,gDAAgD,SAAS,KAAK,CAAC,CAAC;IAEnF,6DAA6D;IAC7D,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC;IAClC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,aAAa,CAAC,KAAK,CAAC,sDAAsD,CAAC,CAAC;QAC5E,MAAM,IAAI,KAAK,CAAC,yEAAyE,CAAC,CAAC;IAC7F,CAAC;IACD,aAAa,CAAC,IAAI,CAAC,sDAAsD,CAAC,CAAC;IAE3E,2EAA2E;IAC3E,MAAM,YAAY,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IAC9C,MAAM,SAAS,GAAG,kBAAkB,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,YAAY,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;IAC/E,aAAa,CAAC,IAAI,CAAC,2DAA2D,CAAC,CAAC;IAEhF,6EAA6E;IAC7E,MAAM,OAAO,GAA2B;QACtC,cAAc,EAAE,kBAAkB;QAClC,MAAM,EAAE,kBAAkB;QAC1B,cAAc,EAAE,GAAG,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,EAAE;QACvC,kBAAkB,EAAE,SAAS;KAC9B,CAAC;IAEF,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAClC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE;QACtC,MAAM,EAAE,KAAK;QACb,OAAO;KACR,CAAC,CAAC;IACH,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc,CAAC;IACjD,aAAa,CAAC,IAAI,CAAC,0CAA0C,YAAY,eAAe,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IAE3G,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,eAAe,CAAC,CAAC;QACrE,aAAa,CAAC,KAAK,CAAC,qCAAqC,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QACnG,MAAM,IAAI,KAAK,CAAC,+CAA+C,SAAS,KAAK,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,MAAM,SAAS,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IAC1J,CAAC;IAED,aAAa,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;IAC7D,MAAM,WAAW,GAAG,MAAM,QAAQ,CAAC,IAAI,EAA2B,CAAC;IAEnE,IAAI,CAAC,WAAW,CAAC,QAAQ,IAAI,CAAC,WAAW,CAAC,QAAQ,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC;QAC5E,aAAa,CAAC,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAC/E,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;IACtF,CAAC;IAED,aAAa,CAAC,IAAI,CAAC,6DAA6D,WAAW,CAAC,QAAQ,eAAe,WAAW,CAAC,QAAQ,GAAG,CAAC,CAAC;IAC5I,OAAO,WAAW,CAAC;AACrB,CAAC;AAED;;GAEG;AACH,SAAS,uBAAuB;IAC9B,qEAAqE;IACrE,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,MAAM,CAAC;IAChE,OAAO,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,iBAAiB,CAAC,CAAC;AAChE,CAAC;AAED;;GAEG;AACH,SAAS,sBAAsB;IAC7B,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;IAClC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAE1B,4CAA4C;IAC5C,2EAA2E;IAC3E,IAAI,QAAQ,KAAK,OAAO,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;QAC3C,OAAO,uBAAuB,CAAC;IACjC,CAAC;SAAM,IAAI,QAAQ,KAAK,OAAO,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;QACpD,OAAO,uBAAuB,CAAC;IACjC,CAAC;SAAM,IAAI,QAAQ,KAAK,QAAQ,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;QACnD,OAAO,wBAAwB,CAAC;IAClC,CAAC;SAAM,IAAI,QAAQ,KAAK,QAAQ,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;QACrD,OAAO,wBAAwB,CAAC;IAClC,CAAC;SAAM,IAAI,QAAQ,KAAK,OAAO,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;QAClD,OAAO,6BAA6B,CAAC;IACvC,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,yBAAyB,QAAQ,IAAI,IAAI,EAAE,CAAC,CAAC;AAC/D,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,yBAAyB,CAAC,GAAW,EAAE,UAAkB,EAAE,iBAA0B,KAAK;IACvG,IAAI,QAAkB,CAAC;IACvB,IAAI,CAAC;QACH,wCAAwC;QACxC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC,oBAAoB;QAEnF,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAC1B,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,OAAO,EAAE;oBACP,YAAY,EAAE,wCAAwC;iBACvD;aACF,CAAC,CAAC;YACH,YAAY,CAAC,SAAS,CAAC,CAAC;QAC1B,CAAC;QAAC,OAAO,UAAU,EAAE,CAAC;YACpB,YAAY,CAAC,SAAS,CAAC,CAAC;YACxB,IAAI,UAAU,YAAY,KAAK,EAAE,CAAC;gBAChC,8CAA8C;gBAC9C,IAAI,eAAe,GAAG,EAAE,CAAC;gBACzB,IAAK,UAAkB,CAAC,KAAK,EAAE,CAAC;oBAC9B,MAAM,KAAK,GAAI,UAAkB,CAAC,KAAK,CAAC;oBACxC,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;wBAC3B,MAAM,SAAS,GAAI,KAAa,CAAC,IAAI,CAAC;wBACtC,eAAe,GAAG,YAAY,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,WAAW,SAAS,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC;oBAC5G,CAAC;yBAAM,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;wBACnD,eAAe,GAAG,iBAAiB,KAAK,CAAC,IAAI,GAAG,CAAC;oBACnD,CAAC;gBACH,CAAC;gBAED,uCAAuC;gBACvC,IAAI,UAAU,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;oBACrC,MAAM,IAAI,KAAK,CAAC,uBAAuB,GAAG,wBAAwB,eAAe,EAAE,CAAC,CAAC;gBACvF,CAAC;gBAED,wDAAwD;gBACxD,MAAM,SAAS,GAAI,UAAkB,CAAC,IAAI,IAAI,CAAE,UAAkB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;gBAChF,IAAI,SAAS,EAAE,CAAC;oBACd,IAAI,SAAS,KAAK,WAAW,IAAI,SAAS,KAAK,WAAW,EAAE,CAAC;wBAC3D,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,WAAW,SAAS,iDAAiD,CAAC,CAAC;oBACzH,CAAC;oBACD,IAAI,SAAS,KAAK,cAAc,EAAE,CAAC;wBACjC,MAAM,IAAI,KAAK,CAAC,yBAAyB,GAAG,WAAW,SAAS,6CAA6C,CAAC,CAAC;oBACjH,CAAC;oBACD,IAAI,SAAS,KAAK,WAAW,IAAI,SAAS,KAAK,YAAY,EAAE,CAAC;wBAC5D,MAAM,IAAI,KAAK,CAAC,+BAA+B,GAAG,WAAW,SAAS,wCAAwC,CAAC,CAAC;oBAClH,CAAC;oBACD,IAAI,SAAS,KAAK,kBAAkB,IAAI,SAAS,KAAK,iCAAiC,EAAE,CAAC;wBACxF,MAAM,IAAI,KAAK,CAAC,2CAA2C,GAAG,WAAW,SAAS,MAAM,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC;oBAChH,CAAC;gBACH,CAAC;gBAED,6CAA6C;gBAC7C,MAAM,QAAQ,GAAG,UAAU,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;gBAClD,IAAI,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC/D,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,iDAAiD,eAAe,EAAE,CAAC,CAAC;gBACtH,CAAC;gBACD,IAAI,QAAQ,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,oBAAoB,CAAC,EAAE,CAAC;oBACjF,MAAM,IAAI,KAAK,CAAC,yBAAyB,GAAG,6CAA6C,eAAe,EAAE,CAAC,CAAC;gBAC9G,CAAC;gBACD,IAAI,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;oBACnE,MAAM,IAAI,KAAK,CAAC,yBAAyB,GAAG,wCAAwC,eAAe,EAAE,CAAC,CAAC;gBACzG,CAAC;gBACD,IAAI,QAAQ,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC7F,MAAM,IAAI,KAAK,CAAC,+BAA+B,GAAG,KAAK,UAAU,CAAC,OAAO,GAAG,eAAe,EAAE,CAAC,CAAC;gBACjG,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,KAAK,UAAU,CAAC,IAAI,MAAM,UAAU,CAAC,OAAO,GAAG,eAAe,EAAE,CAAC,CAAC;YACpH,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,KAAK,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC7E,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,6BAA6B;QAC7B,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;gBAC/G,MAAM,KAAK,CAAC,CAAC,2BAA2B;YAC1C,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACxE,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,KAAK,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACxE,CAAC;IAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QACxD,MAAM,IAAI,KAAK,CAAC,QAAQ,QAAQ,CAAC,MAAM,gBAAgB,GAAG,KAAK,QAAQ,CAAC,UAAU,GAAG,SAAS,CAAC,CAAC,CAAC,MAAM,SAAS,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC/I,CAAC;IAED,MAAM,GAAG,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IAChC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACrB,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtC,CAAC;IAED,MAAM,MAAM,GAAG,YAAY,EAAE,CAAC;IAC9B,MAAM,WAAW,GAAG,iBAAiB,CAAC,UAAU,CAAC,CAAC;IAElD,MAAM,QAAQ,CAAC,QAAQ,CAAC,IAAW,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;IAE1D,IAAI,cAAc,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;QACnD,IAAI,CAAC;YACH,SAAS,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QAC/B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,kBAAkB,UAAU,cAAc,EAAE,KAAK,CAAC,CAAC;QACnE,CAAC;IACH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,gBAAgB,CAAC,gBAAyB,IAAI,EAAE,MAAe;IAC5E,MAAM,QAAQ,GAAG,uBAAuB,EAAE,CAAC;IAC3C,MAAM,UAAU,GAAG,sBAAsB,EAAE,CAAC;IAC5C,MAAM,UAAU,GAAG,wBAAwB,CAAC;IAE5C,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAC9C,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAE9C,MAAM,eAAe,GAA8D,EAAE,CAAC;IAEtF,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,aAAa,EAAE,CAAC;QAC7C,MAAM,SAAS,GAAG,GAAG,uBAAuB,IAAI,UAAU,KAAK,CAAC;QAChE,eAAe,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7E,MAAM,EAAE,IAAI,CAAC,0CAA0C,SAAS,EAAE,CAAC,CAAC;IACtE,CAAC;SAAM,IAAI,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAClC,MAAM,EAAE,IAAI,CAAC,gCAAgC,UAAU,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,aAAa,EAAE,CAAC;QAC7C,MAAM,SAAS,GAAG,GAAG,uBAAuB,IAAI,UAAU,KAAK,CAAC;QAChE,eAAe,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC;QAC9E,MAAM,EAAE,IAAI,CAAC,0CAA0C,SAAS,EAAE,CAAC,CAAC;IACtE,CAAC;SAAM,IAAI,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAClC,MAAM,EAAE,IAAI,CAAC,gCAAgC,UAAU,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,mFAAmF;IACnF,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/B,MAAM,EAAE,IAAI,CAAC,2BAA2B,eAAe,CAAC,MAAM,yBAAyB,CAAC,CAAC;QACzF,MAAM,gBAAgB,GAAG,eAAe,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;YAC1D,IAAI,CAAC;gBACH,MAAM,EAAE,IAAI,CAAC,2BAA2B,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC;gBACvD,MAAM,yBAAyB,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;gBACtE,MAAM,EAAE,IAAI,CAAC,6BAA6B,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YACzD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,wEAAwE;gBACxE,MAAM,QAAQ,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACxE,MAAM,EAAE,KAAK,CAAC,kCAAkC,IAAI,CAAC,GAAG,KAAK,QAAQ,EAAE,CAAC,CAAC;gBACzE,yDAAyD;gBACzD,IAAI,KAAK,YAAY,KAAK,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC1D,MAAM,KAAK,CAAC,CAAC,oCAAoC;gBACnD,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,gCAAgC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,SAAS,IAAI,CAAC,GAAG,KAAK,QAAQ,EAAE,CAAC,CAAC;YACzH,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,+CAA+C;QAC/C,MAAM,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IACtC,CAAC;IAED,OAAO;QACL,UAAU,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI;QACtD,UAAU,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI;KACvD,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,uBAAuB,CAAC,YAAoB,IAAI;IACvD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,MAAM,MAAM,GAAG,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,UAAU,GAA8B,EAAE,CAAC;QACjD,IAAI,OAAuB,CAAC;QAE5B,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,EAAE;YAC3B,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,CAA4B,CAAC;gBACtE,2BAA2B;gBAC3B,IAAI,OAAO,CAAC,OAAO,KAAK,iBAAiB,IAAI,OAAO,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;oBACxE,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAC3B,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,sBAAsB;YACxB,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YACtB,YAAY,CAAC,OAAO,CAAC,CAAC;YACtB,MAAM,CAAC,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,CAAC;QAChB,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE;YACf,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAE1B,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;gBACxB,MAAM,CAAC,KAAK,EAAE,CAAC;gBACf,wCAAwC;gBACxC,OAAO,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACxD,CAAC,EAAE,SAAS,CAAC,CAAC;QAChB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,8BAA8B;IAM5C,MAAM,WAAW,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;IAC3D,MAAM,WAAW,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;IAC3D,MAAM,WAAW,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;IAE3D,IAAI,WAAW,IAAI,WAAW,IAAI,WAAW,EAAE,CAAC;QAC9C,OAAO;YACL,WAAW,EAAE,IAAI;YACjB,WAAW,EAAE,IAAI;YACjB,WAAW,EAAE,IAAI;YACjB,MAAM,EAAE,aAAa;SACtB,CAAC;IACJ,CAAC;IAED,OAAO;QACL,WAAW;QACX,WAAW;QACX,WAAW;QACX,MAAM,EAAE,SAAS;KAClB,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB;IACtC,+DAA+D;IAC/D,uFAAuF;IACvF,IAAI,sBAAsB,EAAE,CAAC;QAC3B,kCAAkC;QAClC,IAAI,CAAC;YACH,4DAA4D;YAC5D,MAAM,YAAY,GAAG,sBAAsB,CAAC,QAAQ,KAAK,IAAI;gBACxC,sBAAsB,CAAC,MAAM,KAAK,KAAK,CAAC;YAE7D,IAAI,YAAY,EAAE,CAAC;gBACjB,kEAAkE;gBAClE,2DAA2D;gBAC3D,MAAM,UAAU,GAAG,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACpD,MAAM,OAAO,GAAG,UAAU,CAAC,QAAQ,CAAC,qBAAqB,CAAC;oBAC3C,UAAU,CAAC,QAAQ,CAAC,mCAAmC,CAAC,CAAC;gBAExE,IAAI,OAAO,EAAE,CAAC;oBACZ,sFAAsF;oBACtF,IAAI,CAAC;wBACH,MAAM,UAAU,GAAG,MAAM,uBAAuB,CAAC,IAAI,CAAC,CAAC,CAAC,6CAA6C;wBACrG,IAAI,UAAU,IAAI,UAAU,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;4BAChD,OAAO;gCACL,SAAS,EAAE,IAAI;gCACf,OAAO,EAAE,IAAI;gCACb,GAAG,EAAE,UAAU,CAAC,GAAG,IAAI,sBAAsB,CAAC,GAAG,IAAI,CAAC;gCACtD,YAAY,EAAE,UAAU,CAAC,YAAY;gCACrC,OAAO,EAAE,UAAU,CAAC,OAAO;6BAC5B,CAAC;wBACJ,CAAC;oBACH,CAAC;oBAAC,MAAM,CAAC;wBACP,mFAAmF;wBACnF,0CAA0C;oBAC5C,CAAC;oBAED,yEAAyE;oBACzE,mDAAmD;oBACnD,MAAM,kBAAkB,GAAG,OAAO,CAAC,GAAG,CAAC,uBAAuB,IAAI,cAAc,CAAC;oBACjF,OAAO;wBACL,SAAS,EAAE,IAAI;wBACf,OAAO,EAAE,IAAI;wBACb,GAAG,EAAE,sBAAsB,CAAC,GAAG,IAAI,CAAC;wBACpC,YAAY,EAAE,kBAAkB;wBAChC,OAAO,EAAE,iBAAiB;qBAC3B,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,0CAA0C;QAC5C,CAAC;IACH,CAAC;IAED,gCAAgC;IAChC,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,MAAM,uBAAuB,CAAC,IAAI,CAAC,CAAC;QACvD,IAAI,UAAU,EAAE,CAAC;YACf,OAAO;gBACL,SAAS,EAAE,IAAI;gBACf,OAAO,EAAE,UAAU,CAAC,MAAM,KAAK,OAAO;gBACtC,GAAG,EAAE,UAAU,CAAC,GAAG;gBACnB,YAAY,EAAE,UAAU,CAAC,YAAY;gBACrC,OAAO,EAAE,UAAU,CAAC,OAAO;aAC5B,CAAC;QACJ,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,mDAAmD;IACrD,CAAC;IAED,OAAO;QACL,SAAS,EAAE,KAAK;QAChB,OAAO,EAAE,KAAK;QACd,GAAG,EAAE,CAAC;KACP,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,SAAS,sCAAsC;IAC7C,MAAM,QAAQ,GAA2B;QACvC,iCAAiC,EAAE,IAAI;QACvC,oCAAoC,EAAE,MAAM;QAC5C,gCAAgC,EAAE,IAAI;QACtC,mCAAmC,EAAE,MAAM;QAC3C,sCAAsC,EAAE,MAAM;QAC9C,+CAA+C,EAAE,IAAI;QACrD,2CAA2C,EAAE,KAAK;QAClD,+CAA+C,EAAE,MAAM;KACxD,CAAC;IAEF,qDAAqD;IACrD,KAAK,MAAM,CAAC,GAAG,EAAE,YAAY,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC3D,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,YAAY,CAAC;QAClC,CAAC;IACH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,sBAAsB,CAAC,QAAgB,EAAE,QAAgB,EAAE,QAAgB;IACzF,wEAAwE;IACxE,6FAA6F;IAC7F,kHAAkH;IAClH,OAAO,CAAC,GAAG,CAAC,wBAAwB,GAAG,QAAQ,CAAC;IAChD,OAAO,CAAC,GAAG,CAAC,wBAAwB,GAAG,QAAQ,CAAC;IAChD,OAAO,CAAC,GAAG,CAAC,wBAAwB,GAAG,QAAQ,CAAC;IAEhD,gDAAgD;IAChD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,EAAE,CAAC;QAC5H,MAAM,IAAI,KAAK,CAAC,2FAA2F,CAAC,CAAC;IAC/G,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,6BAA6B,CACjD,GAAsB,EACtB,UAAmB,CAAC,wCAAwC;;IAE5D,mFAAmF;IACnF,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,MAAM,CAAC;IACnF,MAAM,YAAY,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC;IACnD,MAAM,mBAAmB,GAAG,CAAC,UAAU,IAAI,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC;IACpG,MAAM,mBAAmB,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC;IAE3E,4CAA4C;IAC5C,IAAI,CAAC,mBAAmB,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACjD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,4EAA4E;IAC5E,MAAM,UAAU,GAAG,IAAI,CAAC;QACtB,IAAI,EAAE,oBAAoB;QAC1B,KAAK,EAAE,MAAM;KACd,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAEnB,UAAU,CAAC,IAAI,CAAC,oFAAoF,CAAC,CAAC;IACtG,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAElC,sEAAsE;IACtE,IAAI,CAAC;QACH,UAAU,CAAC,IAAI,CAAC,qDAAqD,CAAC,CAAC;QACvE,MAAM,QAAQ,GAAG,MAAM,wBAAwB,CAAC,UAAU,EAAE,mBAAmB,EAAE,GAAG,CAAC,CAAC;QACtF,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc,CAAC;QAE5C,IAAI,QAAQ,EAAE,CAAC;YACb,UAAU,CAAC,IAAI,CAAC,6BAA6B,QAAQ,oCAAoC,OAAO,KAAK,CAAC,CAAC;YACvG,OAAO,QAAQ,CAAC;QAClB,CAAC;aAAM,CAAC;YACN,UAAU,CAAC,IAAI,CAAC,wEAAwE,OAAO,KAAK,CAAC,CAAC;YACtG,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc,CAAC;QAC5C,MAAM,QAAQ,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACxE,UAAU,CAAC,KAAK,CAAC,uCAAuC,OAAO,OAAO,QAAQ,EAAE,CAAC,CAAC;QAClF,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB,CAAC,GAAsB;IACxD,mFAAmF;IACnF,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,MAAM,CAAC;IACnF,MAAM,YAAY,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC;IACnD,MAAM,mBAAmB,GAAG,CAAC,UAAU,IAAI,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC;IACpG,MAAM,mBAAmB,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC;IAE3E,4CAA4C;IAC5C,IAAI,CAAC,mBAAmB,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACjD,OAAO;IACT,CAAC;IAED,4EAA4E;IAC5E,MAAM,UAAU,GAAG,IAAI,CAAC;QACtB,IAAI,EAAE,oBAAoB;QAC1B,KAAK,EAAE,MAAM;KACd,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAEnB,UAAU,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;IAEnF,2DAA2D;IAC3D,uEAAuE;IACvE,wBAAwB,CAAC,UAAU,EAAE,mBAAmB,EAAE,GAAG,CAAC;SAC3D,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;QACf,IAAI,MAAM,EAAE,CAAC;YACX,UAAU,CAAC,IAAI,CAAC,4DAA4D,MAAM,EAAE,CAAC,CAAC;YACtF,gEAAgE;YAChE,0EAA0E;QAC5E,CAAC;aAAM,CAAC;YACN,UAAU,CAAC,IAAI,CAAC,qEAAqE,CAAC,CAAC;QACzF,CAAC;IACH,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;QACf,MAAM,QAAQ,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACxE,UAAU,CAAC,KAAK,CAAC,mDAAmD,QAAQ,EAAE,CAAC,CAAC;IAClF,CAAC,CAAC,CAAC;IAEL,0DAA0D;AAC5D,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,2BAA2B,CACzC,GAAsB,EACtB,YAAoB,KAAK;IAEzB,mFAAmF;IACnF,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,MAAM,CAAC;IACnF,MAAM,YAAY,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC;IACnD,MAAM,mBAAmB,GAAG,CAAC,UAAU,IAAI,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC;IACpG,MAAM,mBAAmB,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC;IAE3E,4CAA4C;IAC5C,IAAI,CAAC,mBAAmB,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACjD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,4EAA4E;IAC5E,MAAM,UAAU,GAAG,IAAI,CAAC;QACtB,IAAI,EAAE,oBAAoB;QAC1B,KAAK,EAAE,MAAM;KACd,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAEnB,UAAU,CAAC,IAAI,CAAC,8DAA8D,CAAC,CAAC;IAEhF,wDAAwD;IACxD,IAAI,QAAQ,GAAkB,IAAI,CAAC;IACnC,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,UAAU,GAAkB,IAAI,CAAC;IAErC,gCAAgC;IAChC,wBAAwB,CAAC,UAAU,EAAE,mBAAmB,EAAE,GAAG,CAAC;SAC3D,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;QACf,QAAQ,GAAG,MAAM,CAAC;QAClB,SAAS,GAAG,IAAI,CAAC;QACjB,IAAI,MAAM,EAAE,CAAC;YACX,UAAU,CAAC,IAAI,CAAC,8CAA8C,MAAM,EAAE,CAAC,CAAC;QAC1E,CAAC;aAAM,CAAC;YACN,UAAU,CAAC,IAAI,CAAC,qEAAqE,CAAC,CAAC;QACzF,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;QACf,UAAU,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACpE,SAAS,GAAG,IAAI,CAAC;QACjB,UAAU,CAAC,KAAK,CAAC,qCAAqC,UAAU,EAAE,CAAC,CAAC;QACpE,QAAQ,GAAG,IAAI,CAAC;IAClB,CAAC,CAAC,CAAC;IAEL,qDAAqD;IACrD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE;QAChC,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,UAAU,CAAC,IAAI,CAAC,wCAAwC,SAAS,6BAA6B,CAAC,CAAC;YAChG,SAAS,GAAG,IAAI,CAAC;QACnB,CAAC;IACH,CAAC,EAAE,SAAS,CAAC,CAAC;IAEd,IAAI,CAAC;QACH,qFAAqF;QACrF,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE;YACrB,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YACvC,IAAI,OAAO,IAAI,SAAS,EAAE,CAAC;gBACzB,OAAO,KAAK,CAAC,CAAC,eAAe;YAC/B,CAAC;YACD,OAAO,CAAC,SAAS,CAAC,CAAC,oCAAoC;QACzD,CAAC,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,UAAU,CAAC,KAAK,CAAC,6CAA6C,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC1H,CAAC;IAED,YAAY,CAAC,SAAS,CAAC,CAAC;IAExB,IAAI,SAAS,IAAI,QAAQ,EAAE,CAAC;QAC1B,UAAU,CAAC,IAAI,CAAC,4DAA4D,CAAC,CAAC;QAC9E,OAAO,QAAQ,CAAC;IAClB,CAAC;SAAM,IAAI,UAAU,EAAE,CAAC;QACtB,UAAU,CAAC,KAAK,CAAC,kCAAkC,UAAU,2BAA2B,CAAC,CAAC;QAC1F,OAAO,IAAI,CAAC;IACd,CAAC;SAAM,CAAC;QACN,UAAU,CAAC,IAAI,CAAC,0EAA0E,CAAC,CAAC;QAC5F,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAAc,EACd,YAAqB,EACrB,GAAuB;IAEvB,4BAA4B;IAC5B,yBAAyB,GAAG,IAAI,CAAC;IACjC,wBAAwB,GAAG,IAAI,CAAC;IAEhC,8EAA8E;IAC9E,+EAA+E;IAC/E,+EAA+E;IAC/E,iHAAiH;IACjH,IAAI,kBAAkB,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;IAC9D,IAAI,kBAAkB,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;IAC9D,IAAI,kBAAkB,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;IAE9D,IAAI,CAAC,CAAC,kBAAkB,IAAI,CAAC,kBAAkB,IAAI,CAAC,kBAAkB,CAAC,IAAI,GAAG,EAAE,CAAC;QAC/E,IAAI,CAAC;YACH,MAAM,CAAC,IAAI,CAAC,kEAAkE,CAAC,CAAC;YAChF,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACjC,MAAM,WAAW,GAAG,MAAM,0BAA0B,CAAC,GAAG,CAAC,CAAC;YAC1D,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,aAAa,CAAC;YAC/C,MAAM,CAAC,IAAI,CAAC,yDAAyD,WAAW,IAAI,CAAC,CAAC;YACtF,kBAAkB,GAAG,WAAW,CAAC,QAAQ,CAAC;YAC1C,kBAAkB,GAAG,WAAW,CAAC,QAAQ,CAAC;YAC1C,kBAAkB,GAAG,WAAW,CAAC,QAAQ,CAAC;YAE1C,qHAAqH;YACrH,kHAAkH;YAClH,0GAA0G;YAC1G,sBAAsB,CAAC,kBAAkB,EAAE,kBAAkB,EAAE,kBAAkB,CAAC,CAAC;YAEnF,wEAAwE;YACxE,MAAM,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;YAC5D,MAAM,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;YAC5D,MAAM,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;YAE5D,IAAI,CAAC,cAAc,IAAI,CAAC,cAAc,IAAI,CAAC,cAAc,EAAE,CAAC;gBAC1D,MAAM,CAAC,KAAK,CAAC,wGAAwG,CAAC,CAAC;gBACvH,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;YAC7E,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,iFAAiF,CAAC,CAAC;QACjG,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,QAAQ,GAAG,gEAAgE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YAC1I,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,iEAAiE;IACjE,0EAA0E;IAC1E,IAAI,CAAC,kBAAkB,IAAI,CAAC,kBAAkB,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACtE,MAAM,QAAQ,GAAG,0LAA0L,CAAC;QAC5M,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC5B,CAAC;IAED,kFAAkF;IAClF,MAAM,CAAC,IAAI,CAAC,8DAA8D,CAAC,CAAC;IAC5E,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACpC,MAAM,aAAa,GAAG,MAAM,gBAAgB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC3D,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,gBAAgB,CAAC;IACrD,MAAM,CAAC,IAAI,CAAC,+DAA+D,cAAc,IAAI,CAAC,CAAC;IAE/F,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,CAAC;QAC9B,MAAM,CAAC,KAAK,CAAC,kDAAkD,CAAC,CAAC;QACjE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;IACpE,CAAC;IAED,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,CAAC;QAC9B,MAAM,CAAC,KAAK,CAAC,kDAAkD,CAAC,CAAC;QACjE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;IACpE,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,6BAA6B,aAAa,CAAC,UAAU,EAAE,CAAC,CAAC;IACrE,MAAM,CAAC,IAAI,CAAC,6BAA6B,aAAa,CAAC,UAAU,EAAE,CAAC,CAAC;IAErE,uBAAuB;IACvB,IAAI,CAAC,aAAa,CAAC,UAAU,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC;QACvE,MAAM,IAAI,KAAK,CAAC,iCAAiC,aAAa,CAAC,UAAU,EAAE,CAAC,CAAC;IAC/E,CAAC;IAED,sFAAsF;IACtF,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;QACjC,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;YAC/C,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;YAC9D,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAChC,MAAM,YAAY,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,oBAAoB;YACxE,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,MAAM,CAAC,IAAI,CAAC,mDAAmD,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACnF,kCAAkC;gBAClC,IAAI,CAAC;oBACH,SAAS,CAAC,aAAa,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;oBAC3C,MAAM,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;gBACpD,CAAC;gBAAC,OAAO,UAAU,EAAE,CAAC;oBACpB,MAAM,CAAC,KAAK,CAAC,iDAAiD,UAAU,YAAY,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;gBACzI,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,SAAS,EAAE,CAAC;YACnB,MAAM,CAAC,IAAI,CAAC,mDAAmD,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACvI,CAAC;IACH,CAAC;IAED,2EAA2E;IAC3E,6FAA6F;IAC7F,uEAAuE;IACvE,sCAAsC,EAAE,CAAC;IAEzC,sHAAsH;IACtH,iDAAiD;IACjD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,6BAA6B,EAAE,CAAC;QAC/C,OAAO,CAAC,GAAG,CAAC,6BAA6B,GAAG,MAAM,CAAC,cAAc,CAAC,CAAC;QACnE,MAAM,CAAC,IAAI,CAAC,iDAAiD,cAAc,YAAY,CAAC,CAAC;IAC3F,CAAC;IAED,0BAA0B;IAC1B,+EAA+E;IAC/E,IAAI,aAAa,GAAG,YAAY,CAAC;IACjC,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,sCAAsC;QACtC,6DAA6D;QAC7D,aAAa,GAAG,cAAc,CAAC;IACjC,CAAC;IACD,aAAa,GAAG,aAAa,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;IAEnF,uEAAuE;IACvE,2EAA2E;IAC3E,iFAAiF;IACjF,mEAAmE;IACnE,gDAAgD;IAChD,MAAM,cAAc,GAAG,MAAM,YAAY,EAAE,CAAC;IAE5C,0CAA0C;IAC1C,6EAA6E;IAC7E,sEAAsE;IACtE,8FAA8F;IAC9F,6FAA6F;IAC7F,MAAM,YAAY,GAAsB;QACtC,GAAG,OAAO,CAAC,GAAG,EAAE,yEAAyE;QACzF,uBAAuB,EAAE,aAAa;QACtC,6BAA6B,EAAE,MAAM,CAAC,cAAc,CAAC;QACrD,8BAA8B,EAAE,MAAM,CAAC,cAAc,CAAC;KACvD,CAAC;IAEF,0BAA0B;IAC1B,MAAM,gBAAgB,GAAG,KAAK,CAAC,aAAa,CAAC,UAAU,EAAE,CAAC,UAAU,EAAE,aAAa,CAAC,UAAU,CAAC,EAAE;QAC/F,GAAG,EAAE,YAAY;QACjB,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;QACjC,QAAQ,EAAE,KAAK;KAChB,CAAC,CAAC;IAEH,gDAAgD;IAChD,sBAAsB,GAAG,gBAAgB,CAAC;IAC1C,yBAAyB,GAAG,IAAI,CAAC;IACjC,uBAAuB,GAAG,IAAI,CAAC;IAC/B,qBAAqB,GAAG,EAAE,CAAC;IAE3B,+EAA+E;IAC/E,gBAAgB,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;QAC3C,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC;QAC1C,mDAAmD;QACnD,MAAM,CAAC,IAAI,CAAC,mBAAmB,UAAU,EAAE,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,gBAAgB,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;QAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC;QACzC,qBAAqB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtC,mDAAmD;QACnD,IAAI,qBAAqB,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YACtC,qBAAqB,CAAC,KAAK,EAAE,CAAC;QAChC,CAAC;QACD,+DAA+D;QAC/D,MAAM,CAAC,IAAI,CAAC,mBAAmB,SAAS,EAAE,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;IAEH,gBAAgB,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;QACnC,yBAAyB,GAAG,GAAG,CAAC,OAAO,CAAC;QACxC,MAAM,CAAC,KAAK,CAAC,gCAAgC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;IAEH,gBAAgB,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE;QAC3C,uBAAuB,GAAG,IAAI,CAAC;QAC/B,MAAM,CAAC,IAAI,CAAC,wCAAwC,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,YAAY,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAEjG,+DAA+D;QAC/D,IAAI,IAAI,KAAK,CAAC,IAAI,qBAAqB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnD,MAAM,CAAC,KAAK,CAAC,iCAAiC,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,qBAAqB,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;YAC1G,qBAAqB,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;gBAChD,MAAM,CAAC,KAAK,CAAC,mBAAmB,IAAI,EAAE,CAAC,CAAC;YAC1C,CAAC,CAAC,CAAC;QACL,CAAC;QAED,gEAAgE;QAChE,mFAAmF;IACrF,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,IAAI,CAAC,gCAAgC,gBAAgB,CAAC,GAAG,eAAe,aAAa,EAAE,CAAC,CAAC;IAEhG,qEAAqE;IACrE,8CAA8C;IAC9C,yDAAyD;IACzD,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;IAEvD,oDAAoD;IACpD,IAAI,uBAAuB,KAAK,IAAI,IAAI,uBAAuB,KAAK,CAAC,EAAE,CAAC;QACtE,MAAM,SAAS,GAAG,qBAAqB,CAAC,MAAM,GAAG,CAAC;YAChD,CAAC,CAAC,YAAY,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YAChD,CAAC,CAAC,6BAA6B,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,gEAAgE,uBAAuB,IAAI,SAAS,EAAE,CAAC,CAAC;IAC1H,CAAC;IAED,OAAO;QACL,OAAO,EAAE,gBAAgB;QACzB,QAAQ,EAAE,UAAU,aAAa,EAAE;KACpC,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,yBAAyB;IAQvC,OAAO;QACL,UAAU,EAAE,sBAAsB,KAAK,IAAI;QAC3C,GAAG,EAAE,sBAAsB,EAAE,GAAG,IAAI,IAAI;QACxC,QAAQ,EAAE,uBAAuB;QACjC,UAAU,EAAE,yBAAyB;QACrC,SAAS,EAAE,wBAAwB;QACnC,MAAM,EAAE,CAAC,GAAG,qBAAqB,CAAC,EAAE,gBAAgB;KACrD,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,MAAc,EACd,mBAA4B,EAC5B,GAAuB;IAEvB,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,8DAA8D;IAC9D,qFAAqF;IACrF,oFAAoF;IACpF,IAAI,6BAA6B,EAAE,CAAC;QAClC,MAAM,CAAC,IAAI,CAAC,gGAAgG,CAAC,CAAC;QAC9G,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,6BAA6B,CAAC;YACnD,4DAA4D;YAC5D,6BAA6B,GAAG,IAAI,CAAC;YACrC,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,gDAAgD,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACvH,oCAAoC;YACpC,6BAA6B,GAAG,IAAI,CAAC;YACrC,kCAAkC;QACpC,CAAC;IACH,CAAC;IAED,0EAA0E;IAC1E,qFAAqF;IACrF,IAAI,gBAAoC,CAAC;IACzC,IAAI,sBAAsB,EAAE,CAAC;QAC3B,kCAAkC;QAClC,MAAM,YAAY,GAAG,sBAAsB,CAAC,QAAQ,KAAK,IAAI;YACxC,sBAAsB,CAAC,MAAM,KAAK,KAAK,CAAC;QAE7D,IAAI,YAAY,EAAE,CAAC;YACjB,MAAM,CAAC,IAAI,CAAC,qDAAqD,sBAAsB,CAAC,GAAG,kCAAkC,CAAC,CAAC;YAC/H,uEAAuE;YACvE,gBAAgB,GAAG,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;gBACtD,UAAU,OAAO,CAAC,GAAG,CAAC,uBAAuB,EAAE,CAAC,CAAC;gBACjD,qBAAqB,CAAC;YACxB,qEAAqE;QACvE,CAAC;aAAM,CAAC;YACN,4CAA4C;YAC5C,MAAM,CAAC,IAAI,CAAC,+CAA+C,sBAAsB,CAAC,GAAG,gCAAgC,CAAC,CAAC;YACvH,sBAAsB,GAAG,IAAI,CAAC;YAC9B,uBAAuB,GAAG,IAAI,CAAC;YAC/B,qBAAqB,GAAG,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,mEAAmE;IACnE,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC5B,MAAM,CAAC,IAAI,CAAC,kEAAkE,CAAC,CAAC;QAChF,MAAM,MAAM,GAAG,MAAM,kBAAkB,EAAE,CAAC;QAC1C,MAAM,CAAC,IAAI,CAAC,+CAA+C,MAAM,CAAC,SAAS,aAAa,MAAM,CAAC,OAAO,cAAc,MAAM,CAAC,YAAY,IAAI,MAAM,EAAE,CAAC,CAAC;QACrJ,IAAI,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;YAC9D,MAAM,CAAC,IAAI,CAAC,0CAA0C,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC;YAC7E,OAAO,UAAU,MAAM,CAAC,YAAY,EAAE,CAAC;QACzC,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,gEAAgE,CAAC,CAAC;IAChF,CAAC;IAED,0EAA0E;IAC1E,oEAAoE;IACpE,2EAA2E;IAC3E,MAAM,cAAc,GAAG,CAAC,KAAK,IAA4B,EAAE;QACzD,IAAI,CAAC;YACH,gCAAgC;YAChC,wBAAwB,GAAG,IAAI,CAAC;YAEhC,IAAI,QAAgB,CAAC;YACrB,IAAI,gBAAgB,EAAE,CAAC;gBACrB,2DAA2D;gBAC3D,QAAQ,GAAG,gBAAgB,CAAC;gBAC5B,MAAM,CAAC,IAAI,CAAC,iEAAiE,QAAQ,KAAK,CAAC,CAAC;YAChG,CAAC;iBAAM,CAAC;gBACN,wBAAwB;gBACxB,MAAM,CAAC,IAAI,CAAC,qDAAqD,CAAC,CAAC;gBACnE,MAAM,kBAAkB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACtC,MAAM,WAAW,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC;gBACjE,MAAM,qBAAqB,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,kBAAkB,CAAC;gBAC9D,MAAM,CAAC,IAAI,CAAC,4CAA4C,qBAAqB,iBAAiB,WAAW,CAAC,QAAQ,EAAE,CAAC,CAAC;gBACtH,QAAQ,GAAG,WAAW,CAAC,QAAQ,CAAC;gBAEhC,0DAA0D;gBAC1D,MAAM,CAAC,IAAI,CAAC,+DAA+D,CAAC,CAAC;gBAC7E,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;gBAEvD,IAAI,uBAAuB,KAAK,IAAI,IAAI,uBAAuB,KAAK,CAAC,EAAE,CAAC;oBACtE,MAAM,QAAQ,GAAG,kDAAkD,uBAAuB,KAAK,qBAAqB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,mBAAmB,EAAE,CAAC;oBACxM,wBAAwB,GAAG,QAAQ,CAAC;oBACpC,MAAM,CAAC,KAAK,CAAC,eAAe,QAAQ,EAAE,CAAC,CAAC;oBACxC,OAAO,IAAI,CAAC;gBACd,CAAC;gBACD,MAAM,CAAC,IAAI,CAAC,kFAAkF,CAAC,CAAC;YAClG,CAAC;YAED,kEAAkE;YAClE,oDAAoD;YACpD,kFAAkF;YAClF,MAAM,WAAW,GAAG,KAAK,CAAC,CAAC,aAAa;YACxC,MAAM,aAAa,GAAG,GAAG,CAAC,CAAC,oBAAoB;YAC/C,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,aAAa,CAAC,CAAC;YAE1D,MAAM,CAAC,IAAI,CAAC,qEAAqE,aAAa,WAAW,WAAW,GAAG,IAAI,OAAO,CAAC,CAAC;YACpI,MAAM,kBAAkB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAEtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;gBACnC,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC,CAAC;gBACjE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,kBAAkB,CAAC;gBAEhD,sCAAsC;gBACtC,IAAI,uBAAuB,KAAK,IAAI,IAAI,uBAAuB,KAAK,CAAC,EAAE,CAAC;oBACtE,MAAM,QAAQ,GAAG,qDAAqD,uBAAuB,KAAK,qBAAqB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,mBAAmB,EAAE,CAAC;oBAC3M,wBAAwB,GAAG,QAAQ,CAAC;oBACpC,MAAM,CAAC,KAAK,CAAC,eAAe,QAAQ,WAAW,OAAO,KAAK,CAAC,CAAC;oBAC7D,OAAO,IAAI,CAAC;gBACd,CAAC;gBAED,MAAM,CAAC,IAAI,CAAC,gDAAgD,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,sBAAsB,CAAC,GAAG,CAAC,IAAI,SAAS,GAAG,CAAC,CAAC;gBACpI,MAAM,SAAS,GAAG,MAAM,kBAAkB,EAAE,CAAC;gBAC7C,MAAM,CAAC,IAAI,CAAC,2CAA2C,SAAS,CAAC,SAAS,aAAa,SAAS,CAAC,OAAO,SAAS,SAAS,CAAC,GAAG,cAAc,SAAS,CAAC,YAAY,IAAI,MAAM,EAAE,CAAC,CAAC;gBAEhL,IAAI,SAAS,CAAC,SAAS,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;oBAC7C,MAAM,CAAC,IAAI,CAAC,qCAAqC,SAAS,CAAC,YAAY,IAAI,QAAQ,iBAAiB,OAAO,KAAK,CAAC,CAAC;oBAClH,OAAO,SAAS,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,SAAS,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC;gBAChF,CAAC;gBAED,6CAA6C;gBAC7C,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;oBACzB,MAAM,CAAC,IAAI,CAAC,+DAA+D,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;gBACtH,CAAC;YACH,CAAC;YAED,wCAAwC;YACxC,IAAI,uBAAuB,KAAK,IAAI,IAAI,uBAAuB,KAAK,CAAC,EAAE,CAAC;gBACtE,MAAM,QAAQ,GAAG,sCAAsC,uBAAuB,KAAK,qBAAqB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,mBAAmB,EAAE,CAAC;gBAC5L,wBAAwB,GAAG,QAAQ,CAAC;gBACpC,MAAM,CAAC,KAAK,CAAC,eAAe,QAAQ,EAAE,CAAC,CAAC;gBACxC,OAAO,IAAI,CAAC;YACd,CAAC;YAEC,gDAAgD;YAChD,MAAM,CAAC,KAAK,CAAC,qDAAqD,WAAW,GAAG,IAAI,UAAU,CAAC,CAAC;YAChG,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,QAAQ,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAClE,wBAAwB,GAAG,QAAQ,CAAC;YACpC,MAAM,CAAC,KAAK,CAAC,0CAA0C,QAAQ,EAAE,CAAC,CAAC;YACnE,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC,CAAC,EAAE,CAAC;IAEL,6EAA6E;IAC7E,6BAA6B,GAAG,cAAc,CAAC;IAE/C,iFAAiF;IACjF,cAAc,CAAC,OAAO,CAAC,GAAG,EAAE;QAC1B,6BAA6B,GAAG,IAAI,CAAC;IACvC,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,cAAc,CAAC;AAC9B,CAAC","sourcesContent":["import { spawn, ChildProcess } from 'child_process';\r\nimport { createWriteStream, existsSync, chmodSync, mkdirSync } from 'fs';\r\nimport { join, dirname } from 'path';\r\nimport { pipeline } from 'stream/promises';\r\nimport { createGunzip } from 'zlib';\r\nimport { createHash } from 'node:crypto';\r\nimport { createServer } from 'net';\r\nimport type { Logger } from 'pino';\r\nimport pino from 'pino';\r\nimport dgram from 'dgram';\r\nimport deasync from 'deasync';\r\nimport type { EnvironmentConfig } from './types.js';\r\nimport { hostToHttpUrl } from './utils/urls.js';\r\n\r\n// Protocol is httpprotobuf \r\n\r\ninterface CollectorDiscoveryEntry {\r\n version: string;\r\n status: string;\r\n pid: number;\r\n otlpEndpoint: string;\r\n}\r\n\r\nexport interface CollectorStatus {\r\n isRunning: boolean;\r\n isReady: boolean;\r\n pid: number;\r\n otlpEndpoint?: string;\r\n version?: string;\r\n}\r\n\r\nconst COLLECTOR_VERSION = '1.0.1'; // Match C# collector version (from collector-version.json)\r\nconst COLLECTOR_DOWNLOAD_BASE = `https://collectors.beamable.com/version/${COLLECTOR_VERSION}`;\r\nconst DISCOVERY_PORT = parseInt(process.env.BEAM_COLLECTOR_DISCOVERY_PORT || '8688', 10);\r\n\r\n/**\r\n * Finds a free TCP port (matching C# PortUtil.FreeTcpPort())\r\n */\r\nfunction findFreePort(): Promise<number> {\r\n return new Promise((resolve, reject) => {\r\n const server = createServer();\r\n server.listen(0, () => {\r\n const address = server.address();\r\n const port = typeof address === 'object' && address !== null ? address.port : 0;\r\n server.close(() => {\r\n if (port > 0) {\r\n resolve(port);\r\n } else {\r\n // Fallback to a default port if something went wrong\r\n resolve(9090);\r\n }\r\n });\r\n });\r\n server.on('error', (err) => {\r\n reject(err);\r\n });\r\n });\r\n}\r\n\r\n// Global storage for collector process and errors (to prevent GC and track status)\r\nlet globalCollectorProcess: ChildProcess | null = null;\r\nlet globalCollectorStartError: string | null = null;\r\nlet globalCollectorExitCode: number | null = null;\r\nlet globalCollectorStderr: string[] = [];\r\n// Track if collector startup is in progress to prevent duplicate starts\r\nlet globalCollectorStartupPromise: Promise<string | null> | null = null;\r\nlet globalCollectorInitError: string | null = null; // Tracks errors from discoverOrStartCollector\r\n\r\n/**\r\n * ClickHouse credentials response from Beamable API\r\n */\r\ninterface ClickHouseCredentials {\r\n endpoint: string;\r\n expiresAt?: string;\r\n password: string;\r\n username: string;\r\n}\r\n\r\n/**\r\n * Calculates Beamable signature for signed requests\r\n * Signature format: MD5(secret + pid + version + uriPathAndQuery + body) as Base64\r\n * Matches C# SignedRequesterHelper.CalculateSignature\r\n */\r\nfunction calculateSignature(\r\n pid: string,\r\n secret: string,\r\n uriPathAndQuery: string,\r\n body: string | null = null,\r\n version: string = '1'\r\n): string {\r\n let dataToSign = `${secret}${pid}${version}${uriPathAndQuery}`;\r\n if (body) {\r\n dataToSign += body;\r\n }\r\n \r\n // MD5 hash to Base64 (matching C# implementation)\r\n const hash = createHash('md5').update(dataToSign, 'utf8').digest('base64');\r\n return hash;\r\n}\r\n\r\n/**\r\n * Extracts path and query from a URL\r\n */\r\nfunction getPathAndQuery(url: string): string {\r\n try {\r\n const urlObj = new URL(url);\r\n return urlObj.pathname + urlObj.search;\r\n } catch {\r\n // If URL parsing fails, return as-is\r\n return url;\r\n }\r\n}\r\n\r\n/**\r\n * Fetches ClickHouse credentials from Beamable API\r\n * GET /api/beamo/otel/auth/writer/config\r\n * Requires signed request authentication (X-BEAM-SIGNATURE header)\r\n */\r\nexport async function fetchClickHouseCredentials(\r\n env: EnvironmentConfig\r\n): Promise<ClickHouseCredentials> {\r\n const apiUrl = hostToHttpUrl(env.host);\r\n const uriPath = '/api/beamo/otel/auth/writer/config';\r\n const configUrl = new URL(uriPath, apiUrl).toString();\r\n \r\n // Create a minimal logger for verbose output\r\n const verboseLogger = pino({ name: 'fetch-credentials', level: 'info' }, process.stdout);\r\n verboseLogger.info(`[Credentials] Starting credential fetch from ${configUrl}...`);\r\n \r\n // Get secret from environment (required for signed requests)\r\n const secret = process.env.SECRET;\r\n if (!secret) {\r\n verboseLogger.error('[Credentials] SECRET environment variable is missing');\r\n throw new Error('SECRET environment variable is required to fetch ClickHouse credentials');\r\n }\r\n verboseLogger.info('[Credentials] SECRET found, calculating signature...');\r\n \r\n // Calculate signature for signed request (matching C# HttpSignedRequester)\r\n const pathAndQuery = getPathAndQuery(uriPath);\r\n const signature = calculateSignature(env.pid, secret, pathAndQuery, null, '1');\r\n verboseLogger.info('[Credentials] Signature calculated, making API request...');\r\n \r\n // Build headers with signed authentication (matching C# HttpSignedRequester)\r\n const headers: Record<string, string> = {\r\n 'Content-Type': 'application/json',\r\n Accept: 'application/json',\r\n 'X-BEAM-SCOPE': `${env.cid}.${env.pid}`,\r\n 'X-BEAM-SIGNATURE': signature,\r\n };\r\n \r\n const fetchStartTime = Date.now();\r\n const response = await fetch(configUrl, {\r\n method: 'GET',\r\n headers,\r\n });\r\n const fetchElapsed = Date.now() - fetchStartTime;\r\n verboseLogger.info(`[Credentials] API request completed in ${fetchElapsed}ms, status: ${response.status}`);\r\n \r\n if (!response.ok) {\r\n const errorText = await response.text().catch(() => 'Unknown error');\r\n verboseLogger.error(`[Credentials] API request failed: ${response.status} ${response.statusText}`);\r\n throw new Error(`Failed to fetch ClickHouse credentials from ${configUrl}: ${response.status} ${response.statusText} - ${errorText.substring(0, 200)}`);\r\n }\r\n \r\n verboseLogger.info('[Credentials] Parsing response JSON...');\r\n const credentials = await response.json() as ClickHouseCredentials;\r\n \r\n if (!credentials.endpoint || !credentials.username || !credentials.password) {\r\n verboseLogger.error('[Credentials] Invalid response: missing required fields');\r\n throw new Error('Invalid ClickHouse credentials response: missing required fields');\r\n }\r\n \r\n verboseLogger.info(`[Credentials] Successfully fetched credentials (endpoint: ${credentials.endpoint}, username: ${credentials.username})`);\r\n return credentials;\r\n}\r\n\r\n/**\r\n * Gets the collector storage directory (similar to C# LocalApplicationData/beam/collectors/version)\r\n */\r\nfunction getCollectorStoragePath(): string {\r\n // Use temp directory for now - in containers this should be writable\r\n const tempDir = process.env.TMPDIR || process.env.TMP || '/tmp';\r\n return join(tempDir, 'beam', 'collectors', COLLECTOR_VERSION);\r\n}\r\n\r\n/**\r\n * Gets the collector binary name for the current platform\r\n */\r\nfunction getCollectorBinaryName(): string {\r\n const platform = process.platform;\r\n const arch = process.arch;\r\n \r\n // Match C# naming: collector-{osArchSuffix}\r\n // C# returns \"collector-linux-amd64\", not \"beamable-collector-linux-amd64\"\r\n if (platform === 'linux' && arch === 'x64') {\r\n return 'collector-linux-amd64';\r\n } else if (platform === 'linux' && arch === 'arm64') {\r\n return 'collector-linux-arm64';\r\n } else if (platform === 'darwin' && arch === 'x64') {\r\n return 'collector-darwin-amd64';\r\n } else if (platform === 'darwin' && arch === 'arm64') {\r\n return 'collector-darwin-arm64';\r\n } else if (platform === 'win32' && arch === 'x64') {\r\n return 'collector-windows-amd64.exe';\r\n }\r\n \r\n throw new Error(`Unsupported platform: ${platform} ${arch}`);\r\n}\r\n\r\n/**\r\n * Downloads and decompresses a gzipped file\r\n */\r\nasync function downloadAndDecompressGzip(url: string, outputPath: string, makeExecutable: boolean = false): Promise<void> {\r\n let response: Response;\r\n try {\r\n // Add timeout and better error handling\r\n const controller = new AbortController();\r\n const timeoutId = setTimeout(() => controller.abort(), 30000); // 30 second timeout\r\n \r\n try {\r\n response = await fetch(url, {\r\n signal: controller.signal,\r\n headers: {\r\n 'User-Agent': 'Beamable-Node-Microservice-Runtime/1.0',\r\n },\r\n });\r\n clearTimeout(timeoutId);\r\n } catch (fetchError) {\r\n clearTimeout(timeoutId);\r\n if (fetchError instanceof Error) {\r\n // Extract underlying error details from cause\r\n let underlyingError = '';\r\n if ((fetchError as any).cause) {\r\n const cause = (fetchError as any).cause;\r\n if (cause instanceof Error) {\r\n const causeCode = (cause as any).code;\r\n underlyingError = ` (cause: ${cause.name} - ${cause.message}${causeCode ? ` [code: ${causeCode}]` : ''})`;\r\n } else if (typeof cause === 'object' && cause.code) {\r\n underlyingError = ` (cause code: ${cause.code})`;\r\n }\r\n }\r\n \r\n // Provide more context about the error\r\n if (fetchError.name === 'AbortError') {\r\n throw new Error(`Timeout downloading ${url} (30s limit exceeded)${underlyingError}`);\r\n }\r\n \r\n // Check error code if available (Node.js system errors)\r\n const errorCode = (fetchError as any).code || ((fetchError as any).cause?.code);\r\n if (errorCode) {\r\n if (errorCode === 'ENOTFOUND' || errorCode === 'EAI_AGAIN') {\r\n throw new Error(`DNS resolution failed for ${url} (code: ${errorCode}). Check network connectivity and DNS settings.`);\r\n }\r\n if (errorCode === 'ECONNREFUSED') {\r\n throw new Error(`Connection refused to ${url} (code: ${errorCode}). Server may be down or firewall blocking.`);\r\n }\r\n if (errorCode === 'ETIMEDOUT' || errorCode === 'ECONNRESET') {\r\n throw new Error(`Connection timeout/reset to ${url} (code: ${errorCode}). Network may be slow or unreachable.`);\r\n }\r\n if (errorCode === 'CERT_HAS_EXPIRED' || errorCode === 'UNABLE_TO_VERIFY_LEAF_SIGNATURE') {\r\n throw new Error(`SSL/TLS certificate error connecting to ${url} (code: ${errorCode}): ${fetchError.message}`);\r\n }\r\n }\r\n \r\n // Check for common network errors in message\r\n const errorMsg = fetchError.message.toLowerCase();\r\n if (errorMsg.includes('enotfound') || errorMsg.includes('dns')) {\r\n throw new Error(`DNS resolution failed for ${url}. Check network connectivity and DNS settings.${underlyingError}`);\r\n }\r\n if (errorMsg.includes('econnrefused') || errorMsg.includes('connection refused')) {\r\n throw new Error(`Connection refused to ${url}. Server may be down or firewall blocking.${underlyingError}`);\r\n }\r\n if (errorMsg.includes('etimedout') || errorMsg.includes('timeout')) {\r\n throw new Error(`Connection timeout to ${url}. Network may be slow or unreachable.${underlyingError}`);\r\n }\r\n if (errorMsg.includes('certificate') || errorMsg.includes('ssl') || errorMsg.includes('tls')) {\r\n throw new Error(`SSL/TLS error connecting to ${url}: ${fetchError.message}${underlyingError}`);\r\n }\r\n throw new Error(`Network error downloading ${url}: ${fetchError.name} - ${fetchError.message}${underlyingError}`);\r\n }\r\n throw new Error(`Network error downloading ${url}: ${String(fetchError)}`);\r\n }\r\n } catch (error) {\r\n // Re-throw with more context\r\n if (error instanceof Error) {\r\n if (error.message.includes('Timeout') || error.message.includes('DNS') || error.message.includes('Connection')) {\r\n throw error; // Already has good context\r\n }\r\n throw new Error(`Network error downloading ${url}: ${error.message}`);\r\n }\r\n throw new Error(`Network error downloading ${url}: ${String(error)}`);\r\n }\r\n \r\n if (!response.ok) {\r\n const errorText = await response.text().catch(() => '');\r\n throw new Error(`HTTP ${response.status} downloading ${url}: ${response.statusText}${errorText ? ` - ${errorText.substring(0, 200)}` : ''}`);\r\n }\r\n \r\n const dir = dirname(outputPath);\r\n if (!existsSync(dir)) {\r\n mkdirSync(dir, { recursive: true });\r\n }\r\n \r\n const gunzip = createGunzip();\r\n const writeStream = createWriteStream(outputPath);\r\n \r\n await pipeline(response.body as any, gunzip, writeStream);\r\n \r\n if (makeExecutable && process.platform !== 'win32') {\r\n try {\r\n chmodSync(outputPath, 0o755);\r\n } catch (error) {\r\n console.error(`Failed to make ${outputPath} executable:`, error);\r\n }\r\n }\r\n}\r\n\r\n/**\r\n * Resolves the collector binary and config, downloading if needed\r\n */\r\nasync function resolveCollector(allowDownload: boolean = true, logger?: Logger): Promise<{ binaryPath: string | null; configPath: string | null }> {\r\n const basePath = getCollectorStoragePath();\r\n const binaryName = getCollectorBinaryName();\r\n const configName = 'clickhouse-config.yaml';\r\n \r\n const binaryPath = join(basePath, binaryName);\r\n const configPath = join(basePath, configName);\r\n \r\n const itemsToDownload: Array<{ url: string; path: string; executable: boolean }> = [];\r\n \r\n if (!existsSync(binaryPath) && allowDownload) {\r\n const binaryUrl = `${COLLECTOR_DOWNLOAD_BASE}/${binaryName}.gz`;\r\n itemsToDownload.push({ url: binaryUrl, path: binaryPath, executable: true });\r\n logger?.info(`[Collector] Will download binary from: ${binaryUrl}`);\r\n } else if (existsSync(binaryPath)) {\r\n logger?.info(`[Collector] Binary found at: ${binaryPath}`);\r\n }\r\n \r\n if (!existsSync(configPath) && allowDownload) {\r\n const configUrl = `${COLLECTOR_DOWNLOAD_BASE}/${configName}.gz`;\r\n itemsToDownload.push({ url: configUrl, path: configPath, executable: false });\r\n logger?.info(`[Collector] Will download config from: ${configUrl}`);\r\n } else if (existsSync(configPath)) {\r\n logger?.info(`[Collector] Config found at: ${configPath}`);\r\n }\r\n \r\n // Download items in parallel for faster startup (binary and config simultaneously)\r\n if (itemsToDownload.length > 0) {\r\n logger?.info(`[Collector] Downloading ${itemsToDownload.length} item(s) in parallel...`);\r\n const downloadPromises = itemsToDownload.map(async (item) => {\r\n try {\r\n logger?.info(`[Collector] Downloading ${item.url}...`);\r\n await downloadAndDecompressGzip(item.url, item.path, item.executable);\r\n logger?.info(`[Collector] Downloaded to ${item.path}`);\r\n } catch (error) {\r\n // Don't wrap the error again if it already contains the URL and details\r\n const errorMsg = error instanceof Error ? error.message : String(error);\r\n logger?.error(`[Collector] Failed to download ${item.url}: ${errorMsg}`);\r\n // Only wrap if the error doesn't already contain the URL\r\n if (error instanceof Error && errorMsg.includes(item.url)) {\r\n throw error; // Error already has URL and context\r\n }\r\n throw new Error(`Failed to download collector ${item.executable ? 'binary' : 'config'} from ${item.url}: ${errorMsg}`);\r\n }\r\n });\r\n \r\n // Wait for all downloads to complete (or fail)\r\n await Promise.all(downloadPromises);\r\n }\r\n \r\n return {\r\n binaryPath: existsSync(binaryPath) ? binaryPath : null,\r\n configPath: existsSync(configPath) ? configPath : null,\r\n };\r\n}\r\n\r\n/**\r\n * Discovers collector via UDP broadcast\r\n */\r\nfunction discoverCollectorViaUDP(timeoutMs: number = 5000): Promise<CollectorDiscoveryEntry | null> {\r\n return new Promise((resolve) => {\r\n const socket = dgram.createSocket('udp4');\r\n const discovered: CollectorDiscoveryEntry[] = [];\r\n let timeout: NodeJS.Timeout;\r\n \r\n socket.on('message', (msg) => {\r\n try {\r\n const message = JSON.parse(msg.toString()) as CollectorDiscoveryEntry;\r\n // Check if version matches\r\n if (message.version === COLLECTOR_VERSION && message.status === 'READY') {\r\n discovered.push(message);\r\n }\r\n } catch (error) {\r\n // Ignore parse errors\r\n }\r\n });\r\n \r\n socket.on('error', () => {\r\n clearTimeout(timeout);\r\n socket.close();\r\n resolve(null);\r\n });\r\n \r\n socket.bind(() => {\r\n socket.setBroadcast(true);\r\n \r\n timeout = setTimeout(() => {\r\n socket.close();\r\n // Return the first discovered collector\r\n resolve(discovered.length > 0 ? discovered[0] : null);\r\n }, timeoutMs);\r\n });\r\n });\r\n}\r\n\r\n/**\r\n * Gets the current status of ClickHouse credentials (from env or API)\r\n */\r\nexport function getClickHouseCredentialsStatus(): {\r\n hasEndpoint: boolean;\r\n hasUsername: boolean;\r\n hasPassword: boolean;\r\n source: 'environment' | 'api' | 'missing';\r\n} {\r\n const hasEndpoint = !!process.env.BEAM_CLICKHOUSE_ENDPOINT;\r\n const hasUsername = !!process.env.BEAM_CLICKHOUSE_USERNAME;\r\n const hasPassword = !!process.env.BEAM_CLICKHOUSE_PASSWORD;\r\n \r\n if (hasEndpoint && hasUsername && hasPassword) {\r\n return {\r\n hasEndpoint: true,\r\n hasUsername: true,\r\n hasPassword: true,\r\n source: 'environment',\r\n };\r\n }\r\n \r\n return {\r\n hasEndpoint,\r\n hasUsername,\r\n hasPassword,\r\n source: 'missing',\r\n };\r\n}\r\n\r\n/**\r\n * Checks if collector is already running via UDP discovery\r\n */\r\nexport async function isCollectorRunning(): Promise<CollectorStatus> {\r\n // First, check if we have a global collector process reference\r\n // If the process is running and we've seen \"Everything is ready\" in stderr, it's ready\r\n if (globalCollectorProcess) {\r\n // Check if process is still alive\r\n try {\r\n // Process is alive if we can access it and it hasn't exited\r\n const processAlive = globalCollectorProcess.exitCode === null && \r\n globalCollectorProcess.killed === false;\r\n \r\n if (processAlive) {\r\n // Check if we've seen the \"Everything is ready\" message in stderr\r\n // This is a reliable indicator that the collector is ready\r\n const stderrText = globalCollectorStderr.join('\\n');\r\n const isReady = stderrText.includes('Everything is ready') || \r\n stderrText.includes('Begin running and processing data');\r\n \r\n if (isReady) {\r\n // Try UDP discovery to get the endpoint, but if it fails, use the configured endpoint\r\n try {\r\n const discovered = await discoverCollectorViaUDP(1000); // Shorter timeout since we know it's running\r\n if (discovered && discovered.status === 'READY') {\r\n return {\r\n isRunning: true,\r\n isReady: true,\r\n pid: discovered.pid || globalCollectorProcess.pid || 0,\r\n otlpEndpoint: discovered.otlpEndpoint,\r\n version: discovered.version,\r\n };\r\n }\r\n } catch {\r\n // UDP discovery failed, but process is alive and has \"Everything is ready\" message\r\n // Use the configured endpoint as fallback\r\n }\r\n \r\n // Process is alive and we've seen \"Everything is ready\" - treat as ready\r\n // Extract endpoint from environment or use default\r\n const configuredEndpoint = process.env.BEAM_OTLP_HTTP_ENDPOINT || '0.0.0.0:4318';\r\n return {\r\n isRunning: true,\r\n isReady: true,\r\n pid: globalCollectorProcess.pid || 0,\r\n otlpEndpoint: configuredEndpoint,\r\n version: COLLECTOR_VERSION,\r\n };\r\n }\r\n }\r\n } catch {\r\n // Process might have exited or is invalid\r\n }\r\n }\r\n \r\n // Try UDP discovery as fallback\r\n try {\r\n const discovered = await discoverCollectorViaUDP(2000);\r\n if (discovered) {\r\n return {\r\n isRunning: true,\r\n isReady: discovered.status === 'READY',\r\n pid: discovered.pid,\r\n otlpEndpoint: discovered.otlpEndpoint,\r\n version: discovered.version,\r\n };\r\n }\r\n } catch (error) {\r\n // Discovery failed, collector probably not running\r\n }\r\n \r\n return {\r\n isRunning: false,\r\n isReady: false,\r\n pid: 0,\r\n };\r\n}\r\n\r\n/**\r\n * Sets default collector configuration environment variables in process.env\r\n * Matches C# CollectorManager.AddCollectorConfigurationToEnvironment()\r\n */\r\nfunction addCollectorConfigurationToEnvironment(): void {\r\n const defaults: Record<string, string> = {\r\n BEAM_CLICKHOUSE_PROCESSOR_TIMEOUT: '5s',\r\n BEAM_CLICKHOUSE_PROCESSOR_BATCH_SIZE: '5000',\r\n BEAM_CLICKHOUSE_EXPORTER_TIMEOUT: '5s',\r\n BEAM_CLICKHOUSE_EXPORTER_QUEUE_SIZE: '1000',\r\n BEAM_CLICKHOUSE_EXPORTER_RETRY_ENABLED: 'true',\r\n BEAM_CLICKHOUSE_EXPORTER_RETRY_INITIAL_INTERVAL: '5s',\r\n BEAM_CLICKHOUSE_EXPORTER_RETRY_MAX_INTERVAL: '30s',\r\n BEAM_CLICKHOUSE_EXPORTER_RETRY_MAX_ELAPSED_TIME: '300s',\r\n };\r\n \r\n // Only set if not already set (matching C# behavior)\r\n for (const [key, defaultValue] of Object.entries(defaults)) {\r\n if (!process.env[key]) {\r\n process.env[key] = defaultValue;\r\n }\r\n }\r\n}\r\n\r\n/**\r\n * Sets ClickHouse credentials in process.env (matching C# CollectorManager.AddAuthEnvironmentVars)\r\n */\r\nexport function addAuthEnvironmentVars(endpoint: string, username: string, password: string): void {\r\n // Always set the values when explicitly provided (matching C# behavior)\r\n // This ensures credentials fetched from API override any existing (potentially empty) values\r\n // In Node.js, process.env is a writable object - setting properties here persists for the lifetime of the process\r\n process.env.BEAM_CLICKHOUSE_ENDPOINT = endpoint;\r\n process.env.BEAM_CLICKHOUSE_USERNAME = username;\r\n process.env.BEAM_CLICKHOUSE_PASSWORD = password;\r\n \r\n // Verify they were actually set (for debugging)\r\n if (!process.env.BEAM_CLICKHOUSE_ENDPOINT || !process.env.BEAM_CLICKHOUSE_USERNAME || !process.env.BEAM_CLICKHOUSE_PASSWORD) {\r\n throw new Error(`Failed to set ClickHouse credentials in process.env - this should never happen in Node.js`);\r\n }\r\n}\r\n\r\n/**\r\n * Starts the collector and waits for it to be ready before returning.\r\n * This ensures Portal logs (structured logs via OTLP) only start appearing AFTER collector is ready.\r\n * Returns the OTLP endpoint when ready, or null if it times out or fails.\r\n */\r\nexport async function startCollectorAndWaitForReady(\r\n env: EnvironmentConfig,\r\n _timeoutMs?: number // Not used - kept for API compatibility\r\n): Promise<string | null> {\r\n // Match C# logic: (this.InDocker() || UseLocalOtel) && !BEAM_DISABLE_STANDARD_OTEL\r\n const isInDocker = process.env.IS_LOCAL !== '1' && process.env.IS_LOCAL !== 'true';\r\n const useLocalOtel = !!process.env.BEAM_LOCAL_OTEL;\r\n const standardOtelEnabled = (isInDocker || useLocalOtel) && !process.env.BEAM_DISABLE_STANDARD_OTEL;\r\n const hasExplicitEndpoint = !!process.env.BEAM_OTEL_EXPORTER_OTLP_ENDPOINT;\r\n\r\n // If OTLP is not needed, return immediately\r\n if (!standardOtelEnabled && !hasExplicitEndpoint) {\r\n return null;\r\n }\r\n\r\n // Create a minimal console logger for initialization messages (no OTLP yet)\r\n const initLogger = pino({\r\n name: 'beamable-otlp-init',\r\n level: 'info',\r\n }, process.stdout);\r\n\r\n initLogger.info('[OTLP] Setting up collector (waiting for readiness before enabling Portal logs)...');\r\n const setupStartTime = Date.now();\r\n\r\n // Simple linear async/await - no timeouts, no Promise.race complexity\r\n try {\r\n initLogger.info('[OTLP] Step 1: Discovering or starting collector...');\r\n const endpoint = await discoverOrStartCollector(initLogger, standardOtelEnabled, env);\r\n const elapsed = Date.now() - setupStartTime;\r\n \r\n if (endpoint) {\r\n initLogger.info(`[OTLP] Collector ready at ${endpoint}. Portal logs now enabled. (took ${elapsed}ms)`);\r\n return endpoint;\r\n } else {\r\n initLogger.warn(`[OTLP] Collector setup failed, continuing without Portal logs. (took ${elapsed}ms)`);\r\n return null;\r\n }\r\n } catch (error) {\r\n const elapsed = Date.now() - setupStartTime;\r\n const errorMsg = error instanceof Error ? error.message : String(error);\r\n initLogger.error(`[OTLP] Collector setup failed after ${elapsed}ms: ${errorMsg}`);\r\n return null;\r\n }\r\n}\r\n\r\n/**\r\n * Starts the collector asynchronously in the background (non-blocking).\r\n * This allows the service to start immediately while collector downloads/starts.\r\n * Returns immediately - collector setup happens in background.\r\n * @deprecated Use startCollectorAndWaitForReady() to ensure Portal logs only appear after collector is ready\r\n */\r\nexport function startCollectorAsync(env: EnvironmentConfig): void {\r\n // Match C# logic: (this.InDocker() || UseLocalOtel) && !BEAM_DISABLE_STANDARD_OTEL\r\n const isInDocker = process.env.IS_LOCAL !== '1' && process.env.IS_LOCAL !== 'true';\r\n const useLocalOtel = !!process.env.BEAM_LOCAL_OTEL;\r\n const standardOtelEnabled = (isInDocker || useLocalOtel) && !process.env.BEAM_DISABLE_STANDARD_OTEL;\r\n const hasExplicitEndpoint = !!process.env.BEAM_OTEL_EXPORTER_OTLP_ENDPOINT;\r\n\r\n // If OTLP is not needed, return immediately\r\n if (!standardOtelEnabled && !hasExplicitEndpoint) {\r\n return;\r\n }\r\n\r\n // Create a minimal console logger for initialization messages (no OTLP yet)\r\n const initLogger = pino({\r\n name: 'beamable-otlp-init',\r\n level: 'info',\r\n }, process.stdout);\r\n\r\n initLogger.info('[OTLP] Starting collector setup in background (non-blocking)...');\r\n\r\n // Start collector setup asynchronously - don't wait for it\r\n // The service can start immediately, collector will connect when ready\r\n discoverOrStartCollector(initLogger, standardOtelEnabled, env)\r\n .then((result) => {\r\n if (result) {\r\n initLogger.info(`[OTLP] Collector setup complete in background, endpoint: ${result}`);\r\n // Update the global logger provider if it exists to enable OTLP\r\n // The logger will automatically start using OTLP once the provider is set\r\n } else {\r\n initLogger.warn('[OTLP] Collector setup returned null, OTLP logging will be disabled');\r\n }\r\n })\r\n .catch((error) => {\r\n const errorMsg = error instanceof Error ? error.message : String(error);\r\n initLogger.error(`[OTLP] Failed to setup collector in background: ${errorMsg}`);\r\n });\r\n \r\n // Return immediately - service can start serving requests\r\n}\r\n\r\n/**\r\n * Sets up the collector BEFORE logging is initialized (DEPRECATED - blocking approach).\r\n * This function is kept for backwards compatibility but is no longer recommended.\r\n * Use startCollectorAsync() for non-blocking startup instead.\r\n * @deprecated Use startCollectorAsync() for non-blocking startup\r\n */\r\nexport function setupCollectorBeforeLogging(\r\n env: EnvironmentConfig,\r\n timeoutMs: number = 60000\r\n): string | null {\r\n // Match C# logic: (this.InDocker() || UseLocalOtel) && !BEAM_DISABLE_STANDARD_OTEL\r\n const isInDocker = process.env.IS_LOCAL !== '1' && process.env.IS_LOCAL !== 'true';\r\n const useLocalOtel = !!process.env.BEAM_LOCAL_OTEL;\r\n const standardOtelEnabled = (isInDocker || useLocalOtel) && !process.env.BEAM_DISABLE_STANDARD_OTEL;\r\n const hasExplicitEndpoint = !!process.env.BEAM_OTEL_EXPORTER_OTLP_ENDPOINT;\r\n\r\n // If OTLP is not needed, return immediately\r\n if (!standardOtelEnabled && !hasExplicitEndpoint) {\r\n return null;\r\n }\r\n\r\n // Create a minimal console logger for initialization messages (no OTLP yet)\r\n const initLogger = pino({\r\n name: 'beamable-otlp-init',\r\n level: 'info',\r\n }, process.stdout);\r\n\r\n initLogger.info('[OTLP] Setting up collector before logging initialization...');\r\n\r\n // Use deasync to wait synchronously for collector setup\r\n let endpoint: string | null = null;\r\n let completed = false;\r\n let setupError: string | null = null;\r\n\r\n // Start collector setup promise\r\n discoverOrStartCollector(initLogger, standardOtelEnabled, env)\r\n .then((result) => {\r\n endpoint = result;\r\n completed = true;\r\n if (result) {\r\n initLogger.info(`[OTLP] Collector setup complete, endpoint: ${result}`);\r\n } else {\r\n initLogger.warn('[OTLP] Collector setup returned null, OTLP logging will be disabled');\r\n }\r\n return result;\r\n })\r\n .catch((error) => {\r\n setupError = error instanceof Error ? error.message : String(error);\r\n completed = true;\r\n initLogger.error(`[OTLP] Failed to setup collector: ${setupError}`);\r\n endpoint = null;\r\n });\r\n\r\n // Wait synchronously for collector setup to complete\r\n const startTime = Date.now();\r\n const timeoutId = setTimeout(() => {\r\n if (!completed) {\r\n initLogger.warn(`[OTLP] Collector setup timeout after ${timeoutMs}ms, continuing without OTLP`);\r\n completed = true;\r\n }\r\n }, timeoutMs);\r\n\r\n try {\r\n // Use deasync to wait for completion (allows event loop to process async operations)\r\n deasync.loopWhile(() => {\r\n const elapsed = Date.now() - startTime;\r\n if (elapsed >= timeoutMs) {\r\n return false; // Stop waiting\r\n }\r\n return !completed; // Continue waiting if not completed\r\n });\r\n } catch (error) {\r\n initLogger.error(`[OTLP] Error during collector setup wait: ${error instanceof Error ? error.message : String(error)}`);\r\n }\r\n\r\n clearTimeout(timeoutId);\r\n\r\n if (completed && endpoint) {\r\n initLogger.info('[OTLP] Collector is ready, proceeding with logger creation');\r\n return endpoint;\r\n } else if (setupError) {\r\n initLogger.error(`[OTLP] Collector setup failed: ${setupError}, continuing without OTLP`);\r\n return null;\r\n } else {\r\n initLogger.warn('[OTLP] Collector setup did not complete in time, continuing without OTLP');\r\n return null;\r\n }\r\n}\r\n\r\n/**\r\n * Starts the OpenTelemetry collector process\r\n */\r\nexport async function startCollector(\r\n logger: Logger,\r\n otlpEndpoint?: string,\r\n env?: EnvironmentConfig\r\n): Promise<{ process: ChildProcess; endpoint: string }> {\r\n // Clear any previous errors\r\n globalCollectorStartError = null;\r\n globalCollectorInitError = null;\r\n \r\n // CRITICAL: Fetch credentials FIRST, before resolving collector binary/config\r\n // This ensures credentials are in process.env even if collector download fails\r\n // The credentials will persist in process.env for the lifetime of the process.\r\n // Per Gabriel: \"these OTEL related ones you need to do it yourself at the beginning of the microservice startup\"\r\n let clickhouseEndpoint = process.env.BEAM_CLICKHOUSE_ENDPOINT;\r\n let clickhouseUsername = process.env.BEAM_CLICKHOUSE_USERNAME;\r\n let clickhousePassword = process.env.BEAM_CLICKHOUSE_PASSWORD;\r\n \r\n if ((!clickhouseEndpoint || !clickhouseUsername || !clickhousePassword) && env) {\r\n try {\r\n logger.info('[Collector] Fetching ClickHouse credentials from Beamable API...');\r\n const credStartTime = Date.now();\r\n const credentials = await fetchClickHouseCredentials(env);\r\n const credElapsed = Date.now() - credStartTime;\r\n logger.info(`[Collector] ClickHouse credentials fetch completed in ${credElapsed}ms`);\r\n clickhouseEndpoint = credentials.endpoint;\r\n clickhouseUsername = credentials.username;\r\n clickhousePassword = credentials.password;\r\n \r\n // CRITICAL: Set them in process.env IMMEDIATELY after fetching (matching C# CollectorManager.AddAuthEnvironmentVars)\r\n // In Node.js, process.env is a writable object - setting properties here persists for the lifetime of the process\r\n // This ensures they're available to both the Node.js process and inherited by the collector child process\r\n addAuthEnvironmentVars(clickhouseEndpoint, clickhouseUsername, clickhousePassword);\r\n \r\n // Verify they were set (defensive check - should never fail in Node.js)\r\n const verifyEndpoint = process.env.BEAM_CLICKHOUSE_ENDPOINT;\r\n const verifyUsername = process.env.BEAM_CLICKHOUSE_USERNAME;\r\n const verifyPassword = process.env.BEAM_CLICKHOUSE_PASSWORD;\r\n \r\n if (!verifyEndpoint || !verifyUsername || !verifyPassword) {\r\n logger.error(`[Collector] CRITICAL: Credentials were set but are missing from process.env! This should never happen.`);\r\n throw new Error('Failed to persist ClickHouse credentials in process.env');\r\n }\r\n \r\n logger.info('[Collector] ClickHouse credentials fetched from API and verified in process.env');\r\n } catch (error) {\r\n const errorMsg = `[Collector] Failed to fetch ClickHouse credentials from API: ${error instanceof Error ? error.message : String(error)}`;\r\n logger.error(errorMsg);\r\n throw new Error(errorMsg);\r\n }\r\n }\r\n \r\n // Validate required environment variables (matching C# behavior)\r\n // These must be set before starting the collector, otherwise it will fail\r\n if (!clickhouseEndpoint || !clickhouseUsername || !clickhousePassword) {\r\n const errorMsg = `[Collector] Required ClickHouse credentials are missing. Set BEAM_CLICKHOUSE_ENDPOINT, BEAM_CLICKHOUSE_USERNAME, and BEAM_CLICKHOUSE_PASSWORD, or ensure the API endpoint is accessible.`;\r\n logger.error(errorMsg);\r\n throw new Error(errorMsg);\r\n }\r\n \r\n // Now resolve collector binary and config (after credentials are fetched and set)\r\n logger.info('[Collector] Step 2: Resolving collector binary and config...');\r\n const resolveStartTime = Date.now();\r\n const collectorInfo = await resolveCollector(true, logger);\r\n const resolveElapsed = Date.now() - resolveStartTime;\r\n logger.info(`[Collector] Collector binary/config resolution completed in ${resolveElapsed}ms`);\r\n \r\n if (!collectorInfo.binaryPath) {\r\n logger.error('[Collector] Binary not found and download failed');\r\n throw new Error('Collector binary not found and download failed');\r\n }\r\n \r\n if (!collectorInfo.configPath) {\r\n logger.error('[Collector] Config not found and download failed');\r\n throw new Error('Collector config not found and download failed');\r\n }\r\n \r\n logger.info(`[Collector] Using binary: ${collectorInfo.binaryPath}`);\r\n logger.info(`[Collector] Using config: ${collectorInfo.configPath}`);\r\n \r\n // Verify binary exists\r\n if (!collectorInfo.binaryPath || !existsSync(collectorInfo.binaryPath)) {\r\n throw new Error(`Collector binary not found at ${collectorInfo.binaryPath}`);\r\n }\r\n \r\n // Check if binary is executable (Unix only - Windows doesn't have execute permission)\r\n if (process.platform !== 'win32') {\r\n try {\r\n const fsPromises = await import('fs/promises');\r\n const stats = await fsPromises.stat(collectorInfo.binaryPath);\r\n const mode = Number(stats.mode);\r\n const isExecutable = !!(mode & parseInt('111', 8)); // Check execute bit\r\n if (!isExecutable) {\r\n logger.warn(`[Collector] Binary may not be executable. Mode: ${mode.toString(8)}`);\r\n // Try to make it executable again\r\n try {\r\n chmodSync(collectorInfo.binaryPath, 0o755);\r\n logger.info(`[Collector] Made binary executable`);\r\n } catch (chmodError) {\r\n logger.error(`[Collector] Failed to make binary executable: ${chmodError instanceof Error ? chmodError.message : String(chmodError)}`);\r\n }\r\n }\r\n } catch (statError) {\r\n logger.warn(`[Collector] Could not check binary permissions: ${statError instanceof Error ? statError.message : String(statError)}`);\r\n }\r\n }\r\n\r\n // Set default collector configuration environment variables in process.env\r\n // This ensures they're available to both the Node.js process and the collector child process\r\n // Matches C# CollectorManager.AddCollectorConfigurationToEnvironment()\r\n addCollectorConfigurationToEnvironment();\r\n \r\n // CRITICAL: Set BEAM_COLLECTOR_DISCOVERY_PORT if not already set (matching C# AddDefaultCollectorHostAndPortFallback)\r\n // This MUST be set before starting the collector\r\n if (!process.env.BEAM_COLLECTOR_DISCOVERY_PORT) {\r\n process.env.BEAM_COLLECTOR_DISCOVERY_PORT = String(DISCOVERY_PORT);\r\n logger.info(`[Collector] Set BEAM_COLLECTOR_DISCOVERY_PORT=${DISCOVERY_PORT} (default)`);\r\n }\r\n \r\n // Determine OTLP endpoint\r\n // Use a free port if not specified (like C# does with PortUtil.FreeEndpoint())\r\n let localEndpoint = otlpEndpoint;\r\n if (!localEndpoint) {\r\n // For now, use default OTLP HTTP port\r\n // In production, this would be discovered or set by Beamable\r\n localEndpoint = '0.0.0.0:4318';\r\n }\r\n localEndpoint = localEndpoint.replace(/^http:\\/\\//, '').replace(/^https:\\/\\//, '');\r\n \r\n // Find a free port for Prometheus (matching C# PortUtil.FreeTcpPort())\r\n // We'll use a simple approach: bind to port 0 and let the OS assign a port\r\n // For now, use a default port (9090 is common for Prometheus) or find a free one\r\n // C# uses PortUtil.FreeTcpPort() which finds an available TCP port\r\n // We'll use a simple helper to find a free port\r\n const prometheusPort = await findFreePort();\r\n \r\n // Set environment variables for collector\r\n // Note: BEAM_CLICKHOUSE_ENDPOINT is for collector → ClickHouse communication\r\n // This is different from the OTLP endpoint (microservice → collector)\r\n // The collector process will inherit all environment variables from process.env automatically\r\n // We only need to set the OTLP endpoint, discovery port, and Prometheus port explicitly here\r\n const collectorEnv: NodeJS.ProcessEnv = {\r\n ...process.env, // Inherit all process.env (including credentials and config we just set)\r\n BEAM_OTLP_HTTP_ENDPOINT: localEndpoint,\r\n BEAM_COLLECTOR_DISCOVERY_PORT: String(DISCOVERY_PORT),\r\n BEAM_COLLECTOR_PROMETHEUS_PORT: String(prometheusPort),\r\n };\r\n \r\n // Start collector process\r\n const collectorProcess = spawn(collectorInfo.binaryPath, ['--config', collectorInfo.configPath], {\r\n env: collectorEnv,\r\n stdio: ['ignore', 'pipe', 'pipe'],\r\n detached: false,\r\n });\r\n \r\n // Store globally to prevent GC and track status\r\n globalCollectorProcess = collectorProcess;\r\n globalCollectorStartError = null;\r\n globalCollectorExitCode = null;\r\n globalCollectorStderr = [];\r\n \r\n // Set up handlers BEFORE we check if process started, so we capture all output\r\n collectorProcess.stdout?.on('data', (data) => {\r\n const outputLine = data.toString().trim();\r\n // Log stdout at info level to see collector output\r\n logger.info(`[Collector OUT] ${outputLine}`);\r\n });\r\n \r\n collectorProcess.stderr?.on('data', (data) => {\r\n const errorLine = data.toString().trim();\r\n globalCollectorStderr.push(errorLine);\r\n // Keep only last 50 lines to prevent memory issues\r\n if (globalCollectorStderr.length > 50) {\r\n globalCollectorStderr.shift();\r\n }\r\n // Log stderr at info level so we can see why collector crashes\r\n logger.info(`[Collector ERR] ${errorLine}`);\r\n });\r\n \r\n collectorProcess.on('error', (err) => {\r\n globalCollectorStartError = err.message;\r\n logger.error(`[Collector] Failed to start: ${err.message}`);\r\n });\r\n \r\n collectorProcess.on('exit', (code, signal) => {\r\n globalCollectorExitCode = code;\r\n logger.warn(`[Collector] Process exited with code ${code}${signal ? `, signal ${signal}` : ''}`);\r\n \r\n // If process exited with an error code, log accumulated stderr\r\n if (code !== 0 && globalCollectorStderr.length > 0) {\r\n logger.error(`[Collector] Exit error - Last ${Math.min(10, globalCollectorStderr.length)} stderr lines:`);\r\n globalCollectorStderr.slice(-10).forEach((line) => {\r\n logger.error(`[Collector ERR] ${line}`);\r\n });\r\n }\r\n \r\n // Don't clear reference immediately - keep it for status checks\r\n // globalCollectorProcess = null; // Keep reference for getCollectorProcessStatus()\r\n });\r\n \r\n logger.info(`[Collector] Started with PID ${collectorProcess.pid}, endpoint: ${localEndpoint}`);\r\n \r\n // Give the process a moment to start and potentially write to stderr\r\n // This helps capture immediate startup errors\r\n // Wait a bit longer to ensure we catch immediate crashes\r\n await new Promise(resolve => setTimeout(resolve, 500));\r\n \r\n // Check if process already exited (immediate crash)\r\n if (globalCollectorExitCode !== null && globalCollectorExitCode !== 0) {\r\n const stderrMsg = globalCollectorStderr.length > 0 \r\n ? ` Stderr: ${globalCollectorStderr.join('; ')}` \r\n : ' No stderr output captured.';\r\n throw new Error(`Collector process exited immediately after startup with code ${globalCollectorExitCode}.${stderrMsg}`);\r\n }\r\n \r\n return {\r\n process: collectorProcess,\r\n endpoint: `http://${localEndpoint}`,\r\n };\r\n}\r\n\r\n/**\r\n * Gets detailed collector status including process info and errors\r\n */\r\nexport function getCollectorProcessStatus(): {\r\n hasProcess: boolean;\r\n pid: number | null;\r\n exitCode: number | null;\r\n startError: string | null;\r\n initError: string | null;\r\n stderr: string[];\r\n} {\r\n return {\r\n hasProcess: globalCollectorProcess !== null,\r\n pid: globalCollectorProcess?.pid ?? null,\r\n exitCode: globalCollectorExitCode,\r\n startError: globalCollectorStartError,\r\n initError: globalCollectorInitError,\r\n stderr: [...globalCollectorStderr], // Return a copy\r\n };\r\n}\r\n\r\n/**\r\n * Discovers or starts the collector and returns the OTLP endpoint\r\n */\r\nexport async function discoverOrStartCollector(\r\n logger: Logger,\r\n standardOtelEnabled: boolean,\r\n env?: EnvironmentConfig\r\n): Promise<string | null> {\r\n if (!standardOtelEnabled) {\r\n return null;\r\n }\r\n \r\n // CRITICAL: Check if collector startup is already in progress\r\n // This prevents duplicate collector starts if this function is called multiple times\r\n // (e.g., if setupCollectorBeforeLogging times out but the promise is still running)\r\n if (globalCollectorStartupPromise) {\r\n logger.info('[Collector] Collector startup already in progress, waiting for existing startup to complete...');\r\n try {\r\n const result = await globalCollectorStartupPromise;\r\n // Clear the promise after it completes (success or failure)\r\n globalCollectorStartupPromise = null;\r\n return result;\r\n } catch (error) {\r\n logger.error(`[Collector] Existing startup promise failed: ${error instanceof Error ? error.message : String(error)}`);\r\n // Clear the promise so we can retry\r\n globalCollectorStartupPromise = null;\r\n // Fall through to start a new one\r\n }\r\n }\r\n \r\n // CRITICAL: Check if we already have a collector process starting/running\r\n // This prevents duplicate collector starts if this function is called multiple times\r\n let existingEndpoint: string | undefined;\r\n if (globalCollectorProcess) {\r\n // Check if process is still alive\r\n const processAlive = globalCollectorProcess.exitCode === null && \r\n globalCollectorProcess.killed === false;\r\n \r\n if (processAlive) {\r\n logger.info(`[Collector] Collector process already exists (PID ${globalCollectorProcess.pid}), waiting for it to be ready...`);\r\n // Use the configured endpoint from environment (we started it earlier)\r\n existingEndpoint = process.env.BEAM_OTLP_HTTP_ENDPOINT ? \r\n `http://${process.env.BEAM_OTLP_HTTP_ENDPOINT}` : \r\n 'http://0.0.0.0:4318';\r\n // Fall through to the wait logic below (don't start a new collector)\r\n } else {\r\n // Process is dead, clear it and start fresh\r\n logger.warn(`[Collector] Previous collector process (PID ${globalCollectorProcess.pid}) is dead, starting new one...`);\r\n globalCollectorProcess = null;\r\n globalCollectorExitCode = null;\r\n globalCollectorStderr = [];\r\n }\r\n }\r\n \r\n // First, check if collector is already running (via UDP discovery)\r\n if (!globalCollectorProcess) {\r\n logger.info('[Collector] Checking for existing collector via UDP discovery...');\r\n const status = await isCollectorRunning();\r\n logger.info(`[Collector] UDP discovery result: isRunning=${status.isRunning}, isReady=${status.isReady}, endpoint=${status.otlpEndpoint || 'none'}`);\r\n if (status.isRunning && status.isReady && status.otlpEndpoint) {\r\n logger.info(`[Collector] Found running collector at ${status.otlpEndpoint}`);\r\n return `http://${status.otlpEndpoint}`;\r\n }\r\n logger.info('[Collector] No existing collector found, will start new one...');\r\n }\r\n \r\n // Collector not running - start it (or wait for existing one to be ready)\r\n // Wrap the entire startup logic in a promise that we track globally\r\n // This prevents duplicate starts if this function is called multiple times\r\n const startupPromise = (async (): Promise<string | null> => {\r\n try {\r\n // Clear any previous init error\r\n globalCollectorInitError = null;\r\n \r\n let endpoint: string;\r\n if (existingEndpoint) {\r\n // Collector already starting, just wait for it to be ready\r\n endpoint = existingEndpoint;\r\n logger.info(`[Collector] Waiting for existing collector to become ready at ${endpoint}...`);\r\n } else {\r\n // Start a new collector\r\n logger.info('[Collector] Starting new OpenTelemetry collector...');\r\n const startCollectorTime = Date.now();\r\n const startResult = await startCollector(logger, undefined, env);\r\n const startCollectorElapsed = Date.now() - startCollectorTime;\r\n logger.info(`[Collector] Collector process started in ${startCollectorElapsed}ms, endpoint: ${startResult.endpoint}`);\r\n endpoint = startResult.endpoint;\r\n \r\n // Check if collector process exited immediately (crashed)\r\n logger.info('[Collector] Checking if collector process is still running...');\r\n await new Promise(resolve => setTimeout(resolve, 200));\r\n \r\n if (globalCollectorExitCode !== null && globalCollectorExitCode !== 0) {\r\n const errorMsg = `Collector process exited immediately with code ${globalCollectorExitCode}. ${globalCollectorStderr.length > 0 ? `Stderr: ${globalCollectorStderr.join('; ')}` : 'No stderr output.'}`;\r\n globalCollectorInitError = errorMsg;\r\n logger.error(`[Collector] ${errorMsg}`);\r\n return null;\r\n }\r\n logger.info('[Collector] Collector process is still running, proceeding to readiness check...');\r\n }\r\n \r\n // CRITICAL: Wait for collector to be fully ready before returning\r\n // We'll wait up to 60 seconds, checking every 500ms\r\n // This ensures the collector is actually ready to receive logs before we continue\r\n const maxWaitTime = 60000; // 60 seconds\r\n const checkInterval = 500; // Check every 500ms\r\n const maxChecks = Math.floor(maxWaitTime / checkInterval);\r\n \r\n logger.info(`[Collector] Waiting for collector to become ready (checking every ${checkInterval}ms, max ${maxWaitTime / 1000}s)...`);\r\n const readinessStartTime = Date.now();\r\n \r\n for (let i = 0; i < maxChecks; i++) {\r\n await new Promise(resolve => setTimeout(resolve, checkInterval));\r\n const elapsed = Date.now() - readinessStartTime;\r\n \r\n // Check if process exited during wait\r\n if (globalCollectorExitCode !== null && globalCollectorExitCode !== 0) {\r\n const errorMsg = `Collector process exited during startup with code ${globalCollectorExitCode}. ${globalCollectorStderr.length > 0 ? `Stderr: ${globalCollectorStderr.join('; ')}` : 'No stderr output.'}`;\r\n globalCollectorInitError = errorMsg;\r\n logger.error(`[Collector] ${errorMsg} (after ${elapsed}ms)`);\r\n return null;\r\n }\r\n \r\n logger.info(`[Collector] Checking collector readiness... (${(elapsed / 1000).toFixed(1)}s elapsed, attempt ${i + 1}/${maxChecks})`);\r\n const newStatus = await isCollectorRunning();\r\n logger.info(`[Collector] Collector status: isRunning=${newStatus.isRunning}, isReady=${newStatus.isReady}, pid=${newStatus.pid}, endpoint=${newStatus.otlpEndpoint || 'none'}`);\r\n \r\n if (newStatus.isRunning && newStatus.isReady) {\r\n logger.info(`[Collector] Collector is ready at ${newStatus.otlpEndpoint || endpoint} (ready after ${elapsed}ms)`);\r\n return newStatus.otlpEndpoint ? `http://${newStatus.otlpEndpoint}` : endpoint;\r\n }\r\n \r\n // Log progress every second (every 2 checks)\r\n if (i > 0 && i % 2 === 0) {\r\n logger.info(`[Collector] Still waiting for collector to become ready... (${(elapsed / 1000).toFixed(1)}s elapsed)`);\r\n }\r\n }\r\n \r\n // Check one more time if process exited\r\n if (globalCollectorExitCode !== null && globalCollectorExitCode !== 0) {\r\n const errorMsg = `Collector process exited with code ${globalCollectorExitCode}. ${globalCollectorStderr.length > 0 ? `Stderr: ${globalCollectorStderr.join('; ')}` : 'No stderr output.'}`;\r\n globalCollectorInitError = errorMsg;\r\n logger.error(`[Collector] ${errorMsg}`);\r\n return null;\r\n }\r\n \r\n // Collector did not become ready within timeout\r\n logger.error(`[Collector] Collector did not become ready within ${maxWaitTime / 1000} seconds`);\r\n return null;\r\n } catch (err) {\r\n const errorMsg = err instanceof Error ? err.message : String(err);\r\n globalCollectorInitError = errorMsg;\r\n logger.error(`[Collector] Failed to start collector: ${errorMsg}`);\r\n return null;\r\n }\r\n })();\r\n \r\n // Store the promise globally so other calls to this function can wait for it\r\n globalCollectorStartupPromise = startupPromise;\r\n \r\n // Clear the promise when it completes (so we don't keep waiting on old promises)\r\n startupPromise.finally(() => {\r\n globalCollectorStartupPromise = null;\r\n });\r\n \r\n return await startupPromise;\r\n}\r\n\r\n"]}
|
package/package.json
CHANGED
package/src/collector-manager.ts
CHANGED
|
@@ -123,15 +123,22 @@ export async function fetchClickHouseCredentials(
|
|
|
123
123
|
const uriPath = '/api/beamo/otel/auth/writer/config';
|
|
124
124
|
const configUrl = new URL(uriPath, apiUrl).toString();
|
|
125
125
|
|
|
126
|
+
// Create a minimal logger for verbose output
|
|
127
|
+
const verboseLogger = pino({ name: 'fetch-credentials', level: 'info' }, process.stdout);
|
|
128
|
+
verboseLogger.info(`[Credentials] Starting credential fetch from ${configUrl}...`);
|
|
129
|
+
|
|
126
130
|
// Get secret from environment (required for signed requests)
|
|
127
131
|
const secret = process.env.SECRET;
|
|
128
132
|
if (!secret) {
|
|
133
|
+
verboseLogger.error('[Credentials] SECRET environment variable is missing');
|
|
129
134
|
throw new Error('SECRET environment variable is required to fetch ClickHouse credentials');
|
|
130
135
|
}
|
|
136
|
+
verboseLogger.info('[Credentials] SECRET found, calculating signature...');
|
|
131
137
|
|
|
132
138
|
// Calculate signature for signed request (matching C# HttpSignedRequester)
|
|
133
139
|
const pathAndQuery = getPathAndQuery(uriPath);
|
|
134
140
|
const signature = calculateSignature(env.pid, secret, pathAndQuery, null, '1');
|
|
141
|
+
verboseLogger.info('[Credentials] Signature calculated, making API request...');
|
|
135
142
|
|
|
136
143
|
// Build headers with signed authentication (matching C# HttpSignedRequester)
|
|
137
144
|
const headers: Record<string, string> = {
|
|
@@ -141,22 +148,29 @@ export async function fetchClickHouseCredentials(
|
|
|
141
148
|
'X-BEAM-SIGNATURE': signature,
|
|
142
149
|
};
|
|
143
150
|
|
|
151
|
+
const fetchStartTime = Date.now();
|
|
144
152
|
const response = await fetch(configUrl, {
|
|
145
153
|
method: 'GET',
|
|
146
154
|
headers,
|
|
147
155
|
});
|
|
156
|
+
const fetchElapsed = Date.now() - fetchStartTime;
|
|
157
|
+
verboseLogger.info(`[Credentials] API request completed in ${fetchElapsed}ms, status: ${response.status}`);
|
|
148
158
|
|
|
149
159
|
if (!response.ok) {
|
|
150
160
|
const errorText = await response.text().catch(() => 'Unknown error');
|
|
161
|
+
verboseLogger.error(`[Credentials] API request failed: ${response.status} ${response.statusText}`);
|
|
151
162
|
throw new Error(`Failed to fetch ClickHouse credentials from ${configUrl}: ${response.status} ${response.statusText} - ${errorText.substring(0, 200)}`);
|
|
152
163
|
}
|
|
153
164
|
|
|
165
|
+
verboseLogger.info('[Credentials] Parsing response JSON...');
|
|
154
166
|
const credentials = await response.json() as ClickHouseCredentials;
|
|
155
167
|
|
|
156
168
|
if (!credentials.endpoint || !credentials.username || !credentials.password) {
|
|
169
|
+
verboseLogger.error('[Credentials] Invalid response: missing required fields');
|
|
157
170
|
throw new Error('Invalid ClickHouse credentials response: missing required fields');
|
|
158
171
|
}
|
|
159
172
|
|
|
173
|
+
verboseLogger.info(`[Credentials] Successfully fetched credentials (endpoint: ${credentials.endpoint}, username: ${credentials.username})`);
|
|
160
174
|
return credentials;
|
|
161
175
|
}
|
|
162
176
|
|
|
@@ -575,21 +589,25 @@ export async function startCollectorAndWaitForReady(
|
|
|
575
589
|
}, process.stdout);
|
|
576
590
|
|
|
577
591
|
initLogger.info('[OTLP] Setting up collector (waiting for readiness before enabling Portal logs)...');
|
|
592
|
+
const setupStartTime = Date.now();
|
|
578
593
|
|
|
579
594
|
// Simple linear async/await - no timeouts, no Promise.race complexity
|
|
580
595
|
try {
|
|
596
|
+
initLogger.info('[OTLP] Step 1: Discovering or starting collector...');
|
|
581
597
|
const endpoint = await discoverOrStartCollector(initLogger, standardOtelEnabled, env);
|
|
598
|
+
const elapsed = Date.now() - setupStartTime;
|
|
582
599
|
|
|
583
600
|
if (endpoint) {
|
|
584
|
-
initLogger.info(`[OTLP] Collector ready at ${endpoint}. Portal logs now enabled
|
|
601
|
+
initLogger.info(`[OTLP] Collector ready at ${endpoint}. Portal logs now enabled. (took ${elapsed}ms)`);
|
|
585
602
|
return endpoint;
|
|
586
603
|
} else {
|
|
587
|
-
initLogger.warn(
|
|
604
|
+
initLogger.warn(`[OTLP] Collector setup failed, continuing without Portal logs. (took ${elapsed}ms)`);
|
|
588
605
|
return null;
|
|
589
606
|
}
|
|
590
607
|
} catch (error) {
|
|
608
|
+
const elapsed = Date.now() - setupStartTime;
|
|
591
609
|
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
592
|
-
initLogger.error(`[OTLP] Collector setup failed: ${errorMsg}`);
|
|
610
|
+
initLogger.error(`[OTLP] Collector setup failed after ${elapsed}ms: ${errorMsg}`);
|
|
593
611
|
return null;
|
|
594
612
|
}
|
|
595
613
|
}
|
|
@@ -752,7 +770,10 @@ export async function startCollector(
|
|
|
752
770
|
if ((!clickhouseEndpoint || !clickhouseUsername || !clickhousePassword) && env) {
|
|
753
771
|
try {
|
|
754
772
|
logger.info('[Collector] Fetching ClickHouse credentials from Beamable API...');
|
|
773
|
+
const credStartTime = Date.now();
|
|
755
774
|
const credentials = await fetchClickHouseCredentials(env);
|
|
775
|
+
const credElapsed = Date.now() - credStartTime;
|
|
776
|
+
logger.info(`[Collector] ClickHouse credentials fetch completed in ${credElapsed}ms`);
|
|
756
777
|
clickhouseEndpoint = credentials.endpoint;
|
|
757
778
|
clickhouseUsername = credentials.username;
|
|
758
779
|
clickhousePassword = credentials.password;
|
|
@@ -789,8 +810,11 @@ export async function startCollector(
|
|
|
789
810
|
}
|
|
790
811
|
|
|
791
812
|
// Now resolve collector binary and config (after credentials are fetched and set)
|
|
792
|
-
logger.info('[Collector] Resolving collector binary and config...');
|
|
813
|
+
logger.info('[Collector] Step 2: Resolving collector binary and config...');
|
|
814
|
+
const resolveStartTime = Date.now();
|
|
793
815
|
const collectorInfo = await resolveCollector(true, logger);
|
|
816
|
+
const resolveElapsed = Date.now() - resolveStartTime;
|
|
817
|
+
logger.info(`[Collector] Collector binary/config resolution completed in ${resolveElapsed}ms`);
|
|
794
818
|
|
|
795
819
|
if (!collectorInfo.binaryPath) {
|
|
796
820
|
logger.error('[Collector] Binary not found and download failed');
|
|
@@ -1023,11 +1047,14 @@ export async function discoverOrStartCollector(
|
|
|
1023
1047
|
|
|
1024
1048
|
// First, check if collector is already running (via UDP discovery)
|
|
1025
1049
|
if (!globalCollectorProcess) {
|
|
1050
|
+
logger.info('[Collector] Checking for existing collector via UDP discovery...');
|
|
1026
1051
|
const status = await isCollectorRunning();
|
|
1052
|
+
logger.info(`[Collector] UDP discovery result: isRunning=${status.isRunning}, isReady=${status.isReady}, endpoint=${status.otlpEndpoint || 'none'}`);
|
|
1027
1053
|
if (status.isRunning && status.isReady && status.otlpEndpoint) {
|
|
1028
1054
|
logger.info(`[Collector] Found running collector at ${status.otlpEndpoint}`);
|
|
1029
1055
|
return `http://${status.otlpEndpoint}`;
|
|
1030
1056
|
}
|
|
1057
|
+
logger.info('[Collector] No existing collector found, will start new one...');
|
|
1031
1058
|
}
|
|
1032
1059
|
|
|
1033
1060
|
// Collector not running - start it (or wait for existing one to be ready)
|
|
@@ -1045,12 +1072,15 @@ export async function discoverOrStartCollector(
|
|
|
1045
1072
|
logger.info(`[Collector] Waiting for existing collector to become ready at ${endpoint}...`);
|
|
1046
1073
|
} else {
|
|
1047
1074
|
// Start a new collector
|
|
1048
|
-
logger.info('[Collector] Starting OpenTelemetry collector...');
|
|
1075
|
+
logger.info('[Collector] Starting new OpenTelemetry collector...');
|
|
1076
|
+
const startCollectorTime = Date.now();
|
|
1049
1077
|
const startResult = await startCollector(logger, undefined, env);
|
|
1078
|
+
const startCollectorElapsed = Date.now() - startCollectorTime;
|
|
1079
|
+
logger.info(`[Collector] Collector process started in ${startCollectorElapsed}ms, endpoint: ${startResult.endpoint}`);
|
|
1050
1080
|
endpoint = startResult.endpoint;
|
|
1051
1081
|
|
|
1052
1082
|
// Check if collector process exited immediately (crashed)
|
|
1053
|
-
|
|
1083
|
+
logger.info('[Collector] Checking if collector process is still running...');
|
|
1054
1084
|
await new Promise(resolve => setTimeout(resolve, 200));
|
|
1055
1085
|
|
|
1056
1086
|
if (globalCollectorExitCode !== null && globalCollectorExitCode !== 0) {
|
|
@@ -1059,6 +1089,7 @@ export async function discoverOrStartCollector(
|
|
|
1059
1089
|
logger.error(`[Collector] ${errorMsg}`);
|
|
1060
1090
|
return null;
|
|
1061
1091
|
}
|
|
1092
|
+
logger.info('[Collector] Collector process is still running, proceeding to readiness check...');
|
|
1062
1093
|
}
|
|
1063
1094
|
|
|
1064
1095
|
// CRITICAL: Wait for collector to be fully ready before returning
|
|
@@ -1068,28 +1099,33 @@ export async function discoverOrStartCollector(
|
|
|
1068
1099
|
const checkInterval = 500; // Check every 500ms
|
|
1069
1100
|
const maxChecks = Math.floor(maxWaitTime / checkInterval);
|
|
1070
1101
|
|
|
1071
|
-
logger.info(
|
|
1102
|
+
logger.info(`[Collector] Waiting for collector to become ready (checking every ${checkInterval}ms, max ${maxWaitTime / 1000}s)...`);
|
|
1103
|
+
const readinessStartTime = Date.now();
|
|
1072
1104
|
|
|
1073
1105
|
for (let i = 0; i < maxChecks; i++) {
|
|
1074
1106
|
await new Promise(resolve => setTimeout(resolve, checkInterval));
|
|
1107
|
+
const elapsed = Date.now() - readinessStartTime;
|
|
1075
1108
|
|
|
1076
1109
|
// Check if process exited during wait
|
|
1077
1110
|
if (globalCollectorExitCode !== null && globalCollectorExitCode !== 0) {
|
|
1078
1111
|
const errorMsg = `Collector process exited during startup with code ${globalCollectorExitCode}. ${globalCollectorStderr.length > 0 ? `Stderr: ${globalCollectorStderr.join('; ')}` : 'No stderr output.'}`;
|
|
1079
1112
|
globalCollectorInitError = errorMsg;
|
|
1080
|
-
logger.error(`[Collector] ${errorMsg}`);
|
|
1113
|
+
logger.error(`[Collector] ${errorMsg} (after ${elapsed}ms)`);
|
|
1081
1114
|
return null;
|
|
1082
1115
|
}
|
|
1083
1116
|
|
|
1117
|
+
logger.info(`[Collector] Checking collector readiness... (${(elapsed / 1000).toFixed(1)}s elapsed, attempt ${i + 1}/${maxChecks})`);
|
|
1084
1118
|
const newStatus = await isCollectorRunning();
|
|
1119
|
+
logger.info(`[Collector] Collector status: isRunning=${newStatus.isRunning}, isReady=${newStatus.isReady}, pid=${newStatus.pid}, endpoint=${newStatus.otlpEndpoint || 'none'}`);
|
|
1120
|
+
|
|
1085
1121
|
if (newStatus.isRunning && newStatus.isReady) {
|
|
1086
|
-
logger.info(`[Collector] Collector is ready at ${newStatus.otlpEndpoint || endpoint}`);
|
|
1122
|
+
logger.info(`[Collector] Collector is ready at ${newStatus.otlpEndpoint || endpoint} (ready after ${elapsed}ms)`);
|
|
1087
1123
|
return newStatus.otlpEndpoint ? `http://${newStatus.otlpEndpoint}` : endpoint;
|
|
1088
1124
|
}
|
|
1089
1125
|
|
|
1090
|
-
// Log progress every 2
|
|
1091
|
-
if (i > 0 && i %
|
|
1092
|
-
logger.info(`[Collector] Still waiting for collector to become ready... (${(
|
|
1126
|
+
// Log progress every second (every 2 checks)
|
|
1127
|
+
if (i > 0 && i % 2 === 0) {
|
|
1128
|
+
logger.info(`[Collector] Still waiting for collector to become ready... (${(elapsed / 1000).toFixed(1)}s elapsed)`);
|
|
1093
1129
|
}
|
|
1094
1130
|
}
|
|
1095
1131
|
|