@omen.foundation/node-microservice-runtime 0.1.40 → 0.1.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/collector-manager.cjs +41 -7
- package/dist/collector-manager.d.ts.map +1 -1
- package/dist/collector-manager.js +48 -8
- package/dist/collector-manager.js.map +1 -1
- package/package.json +1 -1
- package/src/collector-manager.ts +51 -8
- package/test-downloads/collector-test +0 -0
- package/test-downloads/collector-test.gz +0 -0
|
@@ -100,6 +100,7 @@ function getCollectorBinaryName() {
|
|
|
100
100
|
throw new Error(`Unsupported platform: ${platform} ${arch}`);
|
|
101
101
|
}
|
|
102
102
|
async function downloadAndDecompressGzip(url, outputPath, makeExecutable = false) {
|
|
103
|
+
var _a;
|
|
103
104
|
let response;
|
|
104
105
|
try {
|
|
105
106
|
const controller = new AbortController();
|
|
@@ -116,23 +117,49 @@ async function downloadAndDecompressGzip(url, outputPath, makeExecutable = false
|
|
|
116
117
|
catch (fetchError) {
|
|
117
118
|
clearTimeout(timeoutId);
|
|
118
119
|
if (fetchError instanceof Error) {
|
|
120
|
+
let underlyingError = '';
|
|
121
|
+
if (fetchError.cause) {
|
|
122
|
+
const cause = fetchError.cause;
|
|
123
|
+
if (cause instanceof Error) {
|
|
124
|
+
const causeCode = cause.code;
|
|
125
|
+
underlyingError = ` (cause: ${cause.name} - ${cause.message}${causeCode ? ` [code: ${causeCode}]` : ''})`;
|
|
126
|
+
}
|
|
127
|
+
else if (typeof cause === 'object' && cause.code) {
|
|
128
|
+
underlyingError = ` (cause code: ${cause.code})`;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
119
131
|
if (fetchError.name === 'AbortError') {
|
|
120
|
-
throw new Error(`Timeout downloading ${url} (30s limit exceeded)`);
|
|
132
|
+
throw new Error(`Timeout downloading ${url} (30s limit exceeded)${underlyingError}`);
|
|
133
|
+
}
|
|
134
|
+
const errorCode = fetchError.code || ((_a = fetchError.cause) === null || _a === void 0 ? void 0 : _a.code);
|
|
135
|
+
if (errorCode) {
|
|
136
|
+
if (errorCode === 'ENOTFOUND' || errorCode === 'EAI_AGAIN') {
|
|
137
|
+
throw new Error(`DNS resolution failed for ${url} (code: ${errorCode}). Check network connectivity and DNS settings.`);
|
|
138
|
+
}
|
|
139
|
+
if (errorCode === 'ECONNREFUSED') {
|
|
140
|
+
throw new Error(`Connection refused to ${url} (code: ${errorCode}). Server may be down or firewall blocking.`);
|
|
141
|
+
}
|
|
142
|
+
if (errorCode === 'ETIMEDOUT' || errorCode === 'ECONNRESET') {
|
|
143
|
+
throw new Error(`Connection timeout/reset to ${url} (code: ${errorCode}). Network may be slow or unreachable.`);
|
|
144
|
+
}
|
|
145
|
+
if (errorCode === 'CERT_HAS_EXPIRED' || errorCode === 'UNABLE_TO_VERIFY_LEAF_SIGNATURE') {
|
|
146
|
+
throw new Error(`SSL/TLS certificate error connecting to ${url} (code: ${errorCode}): ${fetchError.message}`);
|
|
147
|
+
}
|
|
121
148
|
}
|
|
122
149
|
const errorMsg = fetchError.message.toLowerCase();
|
|
123
150
|
if (errorMsg.includes('enotfound') || errorMsg.includes('dns')) {
|
|
124
|
-
throw new Error(`DNS resolution failed for ${url}. Check network connectivity and DNS settings
|
|
151
|
+
throw new Error(`DNS resolution failed for ${url}. Check network connectivity and DNS settings.${underlyingError}`);
|
|
125
152
|
}
|
|
126
153
|
if (errorMsg.includes('econnrefused') || errorMsg.includes('connection refused')) {
|
|
127
|
-
throw new Error(`Connection refused to ${url}. Server may be down or firewall blocking
|
|
154
|
+
throw new Error(`Connection refused to ${url}. Server may be down or firewall blocking.${underlyingError}`);
|
|
128
155
|
}
|
|
129
156
|
if (errorMsg.includes('etimedout') || errorMsg.includes('timeout')) {
|
|
130
|
-
throw new Error(`Connection timeout to ${url}. Network may be slow or unreachable
|
|
157
|
+
throw new Error(`Connection timeout to ${url}. Network may be slow or unreachable.${underlyingError}`);
|
|
131
158
|
}
|
|
132
159
|
if (errorMsg.includes('certificate') || errorMsg.includes('ssl') || errorMsg.includes('tls')) {
|
|
133
|
-
throw new Error(`SSL/TLS error connecting to ${url}: ${fetchError.message}`);
|
|
160
|
+
throw new Error(`SSL/TLS error connecting to ${url}: ${fetchError.message}${underlyingError}`);
|
|
134
161
|
}
|
|
135
|
-
throw new Error(`Network error downloading ${url}: ${fetchError.name} - ${fetchError.message}`);
|
|
162
|
+
throw new Error(`Network error downloading ${url}: ${fetchError.name} - ${fetchError.message}${underlyingError}`);
|
|
136
163
|
}
|
|
137
164
|
throw new Error(`Network error downloading ${url}: ${String(fetchError)}`);
|
|
138
165
|
}
|
|
@@ -198,6 +225,9 @@ async function resolveCollector(allowDownload = true, logger) {
|
|
|
198
225
|
catch (error) {
|
|
199
226
|
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
200
227
|
logger === null || logger === void 0 ? void 0 : logger.error(`[Collector] Failed to download ${item.url}: ${errorMsg}`);
|
|
228
|
+
if (error instanceof Error && errorMsg.includes(item.url)) {
|
|
229
|
+
throw error;
|
|
230
|
+
}
|
|
201
231
|
throw new Error(`Failed to download collector ${item.executable ? 'binary' : 'config'} from ${item.url}: ${errorMsg}`);
|
|
202
232
|
}
|
|
203
233
|
}
|
|
@@ -348,6 +378,10 @@ async function startCollector(logger, otlpEndpoint, env) {
|
|
|
348
378
|
logger.info(`[Collector] Using binary: ${collectorInfo.binaryPath}`);
|
|
349
379
|
logger.info(`[Collector] Using config: ${collectorInfo.configPath}`);
|
|
350
380
|
addCollectorConfigurationToEnvironment();
|
|
381
|
+
if (!process.env.BEAM_COLLECTOR_DISCOVERY_PORT) {
|
|
382
|
+
process.env.BEAM_COLLECTOR_DISCOVERY_PORT = String(DISCOVERY_PORT);
|
|
383
|
+
logger.info(`[Collector] Set BEAM_COLLECTOR_DISCOVERY_PORT=${DISCOVERY_PORT} (default)`);
|
|
384
|
+
}
|
|
351
385
|
let localEndpoint = otlpEndpoint;
|
|
352
386
|
if (!localEndpoint) {
|
|
353
387
|
localEndpoint = '0.0.0.0:4318';
|
|
@@ -376,7 +410,7 @@ async function startCollector(logger, otlpEndpoint, env) {
|
|
|
376
410
|
if (globalCollectorStderr.length > 50) {
|
|
377
411
|
globalCollectorStderr.shift();
|
|
378
412
|
}
|
|
379
|
-
logger.
|
|
413
|
+
logger.info(`[Collector ERR] ${errorLine}`);
|
|
380
414
|
});
|
|
381
415
|
collectorProcess.on('error', (err) => {
|
|
382
416
|
globalCollectorStartError = err.message;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"collector-manager.d.ts","sourceRoot":"","sources":["../src/collector-manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAS,YAAY,EAAE,MAAM,eAAe,CAAC;AAMpD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAEnC,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAYpD,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAeD;;GAEG;AACH,UAAU,qBAAqB;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAqCD;;;;GAIG;AACH,wBAAsB,0BAA0B,CAC9C,GAAG,EAAE,iBAAiB,GACrB,OAAO,CAAC,qBAAqB,CAAC,CAwChC;
|
|
1
|
+
{"version":3,"file":"collector-manager.d.ts","sourceRoot":"","sources":["../src/collector-manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAS,YAAY,EAAE,MAAM,eAAe,CAAC;AAMpD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAEnC,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAYpD,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAeD;;GAEG;AACH,UAAU,qBAAqB;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAqCD;;;;GAIG;AACH,wBAAsB,0BAA0B,CAC9C,GAAG,EAAE,iBAAiB,GACrB,OAAO,CAAC,qBAAqB,CAAC,CAwChC;AA0OD;;GAEG;AACH,wBAAgB,8BAA8B,IAAI;IAChD,WAAW,EAAE,OAAO,CAAC;IACrB,WAAW,EAAE,OAAO,CAAC;IACrB,WAAW,EAAE,OAAO,CAAC;IACrB,MAAM,EAAE,aAAa,GAAG,KAAK,GAAG,SAAS,CAAC;CAC3C,CAoBA;AAED;;GAEG;AACH,wBAAsB,kBAAkB,IAAI,OAAO,CAAC,eAAe,CAAC,CAqBnE;AA0BD;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI,CAYjG;AAED;;GAEG;AACH,wBAAsB,cAAc,CAClC,MAAM,EAAE,MAAM,EACd,YAAY,CAAC,EAAE,MAAM,EACrB,GAAG,CAAC,EAAE,iBAAiB,GACtB,OAAO,CAAC;IAAE,OAAO,EAAE,YAAY,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC,CAmJtD;AAED;;GAEG;AACH,wBAAgB,yBAAyB,IAAI;IAC3C,UAAU,EAAE,OAAO,CAAC;IACpB,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACnB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB,CASA;AAED;;GAEG;AACH,wBAAsB,wBAAwB,CAC5C,MAAM,EAAE,MAAM,EACd,mBAAmB,EAAE,OAAO,EAC5B,GAAG,CAAC,EAAE,iBAAiB,GACtB,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAwCxB"}
|
|
@@ -136,25 +136,53 @@ async function downloadAndDecompressGzip(url, outputPath, makeExecutable = false
|
|
|
136
136
|
catch (fetchError) {
|
|
137
137
|
clearTimeout(timeoutId);
|
|
138
138
|
if (fetchError instanceof Error) {
|
|
139
|
+
// Extract underlying error details from cause
|
|
140
|
+
let underlyingError = '';
|
|
141
|
+
if (fetchError.cause) {
|
|
142
|
+
const cause = fetchError.cause;
|
|
143
|
+
if (cause instanceof Error) {
|
|
144
|
+
const causeCode = cause.code;
|
|
145
|
+
underlyingError = ` (cause: ${cause.name} - ${cause.message}${causeCode ? ` [code: ${causeCode}]` : ''})`;
|
|
146
|
+
}
|
|
147
|
+
else if (typeof cause === 'object' && cause.code) {
|
|
148
|
+
underlyingError = ` (cause code: ${cause.code})`;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
139
151
|
// Provide more context about the error
|
|
140
152
|
if (fetchError.name === 'AbortError') {
|
|
141
|
-
throw new Error(`Timeout downloading ${url} (30s limit exceeded)`);
|
|
153
|
+
throw new Error(`Timeout downloading ${url} (30s limit exceeded)${underlyingError}`);
|
|
154
|
+
}
|
|
155
|
+
// Check error code if available (Node.js system errors)
|
|
156
|
+
const errorCode = fetchError.code || (fetchError.cause?.code);
|
|
157
|
+
if (errorCode) {
|
|
158
|
+
if (errorCode === 'ENOTFOUND' || errorCode === 'EAI_AGAIN') {
|
|
159
|
+
throw new Error(`DNS resolution failed for ${url} (code: ${errorCode}). Check network connectivity and DNS settings.`);
|
|
160
|
+
}
|
|
161
|
+
if (errorCode === 'ECONNREFUSED') {
|
|
162
|
+
throw new Error(`Connection refused to ${url} (code: ${errorCode}). Server may be down or firewall blocking.`);
|
|
163
|
+
}
|
|
164
|
+
if (errorCode === 'ETIMEDOUT' || errorCode === 'ECONNRESET') {
|
|
165
|
+
throw new Error(`Connection timeout/reset to ${url} (code: ${errorCode}). Network may be slow or unreachable.`);
|
|
166
|
+
}
|
|
167
|
+
if (errorCode === 'CERT_HAS_EXPIRED' || errorCode === 'UNABLE_TO_VERIFY_LEAF_SIGNATURE') {
|
|
168
|
+
throw new Error(`SSL/TLS certificate error connecting to ${url} (code: ${errorCode}): ${fetchError.message}`);
|
|
169
|
+
}
|
|
142
170
|
}
|
|
143
|
-
// Check for common network errors
|
|
171
|
+
// Check for common network errors in message
|
|
144
172
|
const errorMsg = fetchError.message.toLowerCase();
|
|
145
173
|
if (errorMsg.includes('enotfound') || errorMsg.includes('dns')) {
|
|
146
|
-
throw new Error(`DNS resolution failed for ${url}. Check network connectivity and DNS settings
|
|
174
|
+
throw new Error(`DNS resolution failed for ${url}. Check network connectivity and DNS settings.${underlyingError}`);
|
|
147
175
|
}
|
|
148
176
|
if (errorMsg.includes('econnrefused') || errorMsg.includes('connection refused')) {
|
|
149
|
-
throw new Error(`Connection refused to ${url}. Server may be down or firewall blocking
|
|
177
|
+
throw new Error(`Connection refused to ${url}. Server may be down or firewall blocking.${underlyingError}`);
|
|
150
178
|
}
|
|
151
179
|
if (errorMsg.includes('etimedout') || errorMsg.includes('timeout')) {
|
|
152
|
-
throw new Error(`Connection timeout to ${url}. Network may be slow or unreachable
|
|
180
|
+
throw new Error(`Connection timeout to ${url}. Network may be slow or unreachable.${underlyingError}`);
|
|
153
181
|
}
|
|
154
182
|
if (errorMsg.includes('certificate') || errorMsg.includes('ssl') || errorMsg.includes('tls')) {
|
|
155
|
-
throw new Error(`SSL/TLS error connecting to ${url}: ${fetchError.message}`);
|
|
183
|
+
throw new Error(`SSL/TLS error connecting to ${url}: ${fetchError.message}${underlyingError}`);
|
|
156
184
|
}
|
|
157
|
-
throw new Error(`Network error downloading ${url}: ${fetchError.name} - ${fetchError.message}`);
|
|
185
|
+
throw new Error(`Network error downloading ${url}: ${fetchError.name} - ${fetchError.message}${underlyingError}`);
|
|
158
186
|
}
|
|
159
187
|
throw new Error(`Network error downloading ${url}: ${String(fetchError)}`);
|
|
160
188
|
}
|
|
@@ -222,8 +250,13 @@ async function resolveCollector(allowDownload = true, logger) {
|
|
|
222
250
|
logger?.info(`[Collector] Downloaded to ${item.path}`);
|
|
223
251
|
}
|
|
224
252
|
catch (error) {
|
|
253
|
+
// Don't wrap the error again if it already contains the URL and details
|
|
225
254
|
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
226
255
|
logger?.error(`[Collector] Failed to download ${item.url}: ${errorMsg}`);
|
|
256
|
+
// Only wrap if the error doesn't already contain the URL
|
|
257
|
+
if (error instanceof Error && errorMsg.includes(item.url)) {
|
|
258
|
+
throw error; // Error already has URL and context
|
|
259
|
+
}
|
|
227
260
|
throw new Error(`Failed to download collector ${item.executable ? 'binary' : 'config'} from ${item.url}: ${errorMsg}`);
|
|
228
261
|
}
|
|
229
262
|
}
|
|
@@ -416,6 +449,12 @@ export async function startCollector(logger, otlpEndpoint, env) {
|
|
|
416
449
|
// This ensures they're available to both the Node.js process and the collector child process
|
|
417
450
|
// Matches C# CollectorManager.AddCollectorConfigurationToEnvironment()
|
|
418
451
|
addCollectorConfigurationToEnvironment();
|
|
452
|
+
// CRITICAL: Set BEAM_COLLECTOR_DISCOVERY_PORT if not already set (matching C# AddDefaultCollectorHostAndPortFallback)
|
|
453
|
+
// This MUST be set before starting the collector
|
|
454
|
+
if (!process.env.BEAM_COLLECTOR_DISCOVERY_PORT) {
|
|
455
|
+
process.env.BEAM_COLLECTOR_DISCOVERY_PORT = String(DISCOVERY_PORT);
|
|
456
|
+
logger.info(`[Collector] Set BEAM_COLLECTOR_DISCOVERY_PORT=${DISCOVERY_PORT} (default)`);
|
|
457
|
+
}
|
|
419
458
|
// Determine OTLP endpoint
|
|
420
459
|
// Use a free port if not specified (like C# does with PortUtil.FreeEndpoint())
|
|
421
460
|
let localEndpoint = otlpEndpoint;
|
|
@@ -456,7 +495,8 @@ export async function startCollector(logger, otlpEndpoint, env) {
|
|
|
456
495
|
if (globalCollectorStderr.length > 50) {
|
|
457
496
|
globalCollectorStderr.shift();
|
|
458
497
|
}
|
|
459
|
-
|
|
498
|
+
// Log stderr at info level so we can see why collector crashes
|
|
499
|
+
logger.info(`[Collector ERR] ${errorLine}`);
|
|
460
500
|
});
|
|
461
501
|
collectorProcess.on('error', (err) => {
|
|
462
502
|
globalCollectorStartError = err.message;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"collector-manager.js","sourceRoot":"","sources":["../src/collector-manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAgB,MAAM,eAAe,CAAC;AACpD,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AACzE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AACrC,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,MAAM,MAAM,CAAC;AACpC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAmBhD,MAAM,iBAAiB,GAAG,OAAO,CAAC,CAAC,2DAA2D;AAC9F,MAAM,uBAAuB,GAAG,2CAA2C,iBAAiB,EAAE,CAAC;AAC/F,MAAM,cAAc,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,6BAA6B,IAAI,MAAM,EAAE,EAAE,CAAC,CAAC;AAEzF,mFAAmF;AACnF,IAAI,sBAAsB,GAAwB,IAAI,CAAC;AACvD,IAAI,yBAAyB,GAAkB,IAAI,CAAC;AACpD,IAAI,uBAAuB,GAAkB,IAAI,CAAC;AAClD,IAAI,qBAAqB,GAAa,EAAE,CAAC;AACzC,IAAI,wBAAwB,GAAkB,IAAI,CAAC,CAAC,8CAA8C;AAClG,MAAM,eAAe,GAAG,GAAG,CAAC,CAAC,eAAe;AAC5C,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAY9B;;;;GAIG;AACH,SAAS,kBAAkB,CACzB,GAAW,EACX,MAAc,EACd,eAAuB,EACvB,OAAsB,IAAI,EAC1B,UAAkB,GAAG;IAErB,IAAI,UAAU,GAAG,GAAG,MAAM,GAAG,GAAG,GAAG,OAAO,GAAG,eAAe,EAAE,CAAC;IAC/D,IAAI,IAAI,EAAE,CAAC;QACT,UAAU,IAAI,IAAI,CAAC;IACrB,CAAC;IAED,kDAAkD;IAClD,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC3E,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,GAAW;IAClC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,OAAO,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC;IACzC,CAAC;IAAC,MAAM,CAAC;QACP,qCAAqC;QACrC,OAAO,GAAG,CAAC;IACb,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC9C,GAAsB;IAEtB,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,OAAO,GAAG,oCAAoC,CAAC;IACrD,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC;IAEtD,6DAA6D;IAC7D,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC;IAClC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CAAC,yEAAyE,CAAC,CAAC;IAC7F,CAAC;IAED,2EAA2E;IAC3E,MAAM,YAAY,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IAC9C,MAAM,SAAS,GAAG,kBAAkB,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,YAAY,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;IAE/E,6EAA6E;IAC7E,MAAM,OAAO,GAA2B;QACtC,cAAc,EAAE,kBAAkB;QAClC,MAAM,EAAE,kBAAkB;QAC1B,cAAc,EAAE,GAAG,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,EAAE;QACvC,kBAAkB,EAAE,SAAS;KAC9B,CAAC;IAEF,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE;QACtC,MAAM,EAAE,KAAK;QACb,OAAO;KACR,CAAC,CAAC;IAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,eAAe,CAAC,CAAC;QACrE,MAAM,IAAI,KAAK,CAAC,+CAA+C,SAAS,KAAK,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,MAAM,SAAS,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IAC1J,CAAC;IAED,MAAM,WAAW,GAAG,MAAM,QAAQ,CAAC,IAAI,EAA2B,CAAC;IAEnE,IAAI,CAAC,WAAW,CAAC,QAAQ,IAAI,CAAC,WAAW,CAAC,QAAQ,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC;QAC5E,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;IACtF,CAAC;IAED,OAAO,WAAW,CAAC;AACrB,CAAC;AAED;;GAEG;AACH,SAAS,uBAAuB;IAC9B,qEAAqE;IACrE,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,MAAM,CAAC;IAChE,OAAO,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,iBAAiB,CAAC,CAAC;AAChE,CAAC;AAED;;GAEG;AACH,SAAS,sBAAsB;IAC7B,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;IAClC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAE1B,4CAA4C;IAC5C,2EAA2E;IAC3E,IAAI,QAAQ,KAAK,OAAO,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;QAC3C,OAAO,uBAAuB,CAAC;IACjC,CAAC;SAAM,IAAI,QAAQ,KAAK,OAAO,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;QACpD,OAAO,uBAAuB,CAAC;IACjC,CAAC;SAAM,IAAI,QAAQ,KAAK,QAAQ,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;QACnD,OAAO,wBAAwB,CAAC;IAClC,CAAC;SAAM,IAAI,QAAQ,KAAK,QAAQ,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;QACrD,OAAO,wBAAwB,CAAC;IAClC,CAAC;SAAM,IAAI,QAAQ,KAAK,OAAO,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;QAClD,OAAO,6BAA6B,CAAC;IACvC,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,yBAAyB,QAAQ,IAAI,IAAI,EAAE,CAAC,CAAC;AAC/D,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,yBAAyB,CAAC,GAAW,EAAE,UAAkB,EAAE,iBAA0B,KAAK;IACvG,IAAI,QAAkB,CAAC;IACvB,IAAI,CAAC;QACH,wCAAwC;QACxC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC,oBAAoB;QAEnF,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAC1B,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,OAAO,EAAE;oBACP,YAAY,EAAE,wCAAwC;iBACvD;aACF,CAAC,CAAC;YACH,YAAY,CAAC,SAAS,CAAC,CAAC;QAC1B,CAAC;QAAC,OAAO,UAAU,EAAE,CAAC;YACpB,YAAY,CAAC,SAAS,CAAC,CAAC;YACxB,IAAI,UAAU,YAAY,KAAK,EAAE,CAAC;gBAChC,uCAAuC;gBACvC,IAAI,UAAU,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;oBACrC,MAAM,IAAI,KAAK,CAAC,uBAAuB,GAAG,uBAAuB,CAAC,CAAC;gBACrE,CAAC;gBACD,kCAAkC;gBAClC,MAAM,QAAQ,GAAG,UAAU,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;gBAClD,IAAI,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC/D,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,gDAAgD,CAAC,CAAC;gBACpG,CAAC;gBACD,IAAI,QAAQ,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,oBAAoB,CAAC,EAAE,CAAC;oBACjF,MAAM,IAAI,KAAK,CAAC,yBAAyB,GAAG,4CAA4C,CAAC,CAAC;gBAC5F,CAAC;gBACD,IAAI,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;oBACnE,MAAM,IAAI,KAAK,CAAC,yBAAyB,GAAG,uCAAuC,CAAC,CAAC;gBACvF,CAAC;gBACD,IAAI,QAAQ,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC7F,MAAM,IAAI,KAAK,CAAC,+BAA+B,GAAG,KAAK,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC/E,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,KAAK,UAAU,CAAC,IAAI,MAAM,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC;YAClG,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,KAAK,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC7E,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,6BAA6B;QAC7B,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;gBAC/G,MAAM,KAAK,CAAC,CAAC,2BAA2B;YAC1C,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACxE,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,KAAK,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACxE,CAAC;IAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QACxD,MAAM,IAAI,KAAK,CAAC,QAAQ,QAAQ,CAAC,MAAM,gBAAgB,GAAG,KAAK,QAAQ,CAAC,UAAU,GAAG,SAAS,CAAC,CAAC,CAAC,MAAM,SAAS,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC/I,CAAC;IAED,MAAM,GAAG,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IAChC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACrB,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtC,CAAC;IAED,MAAM,MAAM,GAAG,YAAY,EAAE,CAAC;IAC9B,MAAM,WAAW,GAAG,iBAAiB,CAAC,UAAU,CAAC,CAAC;IAElD,MAAM,QAAQ,CAAC,QAAQ,CAAC,IAAW,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;IAE1D,IAAI,cAAc,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;QACnD,IAAI,CAAC;YACH,SAAS,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QAC/B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,kBAAkB,UAAU,cAAc,EAAE,KAAK,CAAC,CAAC;QACnE,CAAC;IACH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,gBAAgB,CAAC,gBAAyB,IAAI,EAAE,MAAe;IAC5E,MAAM,QAAQ,GAAG,uBAAuB,EAAE,CAAC;IAC3C,MAAM,UAAU,GAAG,sBAAsB,EAAE,CAAC;IAC5C,MAAM,UAAU,GAAG,wBAAwB,CAAC;IAE5C,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAC9C,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAE9C,MAAM,eAAe,GAA8D,EAAE,CAAC;IAEtF,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,aAAa,EAAE,CAAC;QAC7C,MAAM,SAAS,GAAG,GAAG,uBAAuB,IAAI,UAAU,KAAK,CAAC;QAChE,eAAe,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7E,MAAM,EAAE,IAAI,CAAC,0CAA0C,SAAS,EAAE,CAAC,CAAC;IACtE,CAAC;SAAM,IAAI,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAClC,MAAM,EAAE,IAAI,CAAC,gCAAgC,UAAU,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,aAAa,EAAE,CAAC;QAC7C,MAAM,SAAS,GAAG,GAAG,uBAAuB,IAAI,UAAU,KAAK,CAAC;QAChE,eAAe,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC;QAC9E,MAAM,EAAE,IAAI,CAAC,0CAA0C,SAAS,EAAE,CAAC,CAAC;IACtE,CAAC;SAAM,IAAI,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAClC,MAAM,EAAE,IAAI,CAAC,gCAAgC,UAAU,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,eAAe,EAAE,CAAC;QACnC,IAAI,CAAC;YACH,MAAM,EAAE,IAAI,CAAC,2BAA2B,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC;YACvD,MAAM,yBAAyB,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YACtE,MAAM,EAAE,IAAI,CAAC,6BAA6B,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QACzD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,QAAQ,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACxE,MAAM,EAAE,KAAK,CAAC,kCAAkC,IAAI,CAAC,GAAG,KAAK,QAAQ,EAAE,CAAC,CAAC;YACzE,MAAM,IAAI,KAAK,CAAC,gCAAgC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,SAAS,IAAI,CAAC,GAAG,KAAK,QAAQ,EAAE,CAAC,CAAC;QACzH,CAAC;IACH,CAAC;IAED,OAAO;QACL,UAAU,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI;QACtD,UAAU,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI;KACvD,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,uBAAuB,CAAC,YAAoB,IAAI;IACvD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,MAAM,MAAM,GAAG,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,UAAU,GAA8B,EAAE,CAAC;QACjD,IAAI,OAAuB,CAAC;QAE5B,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,EAAE;YAC3B,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,CAA4B,CAAC;gBACtE,2BAA2B;gBAC3B,IAAI,OAAO,CAAC,OAAO,KAAK,iBAAiB,IAAI,OAAO,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;oBACxE,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAC3B,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,sBAAsB;YACxB,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YACtB,YAAY,CAAC,OAAO,CAAC,CAAC;YACtB,MAAM,CAAC,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,CAAC;QAChB,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE;YACf,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAE1B,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;gBACxB,MAAM,CAAC,KAAK,EAAE,CAAC;gBACf,wCAAwC;gBACxC,OAAO,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACxD,CAAC,EAAE,SAAS,CAAC,CAAC;QAChB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,8BAA8B;IAM5C,MAAM,WAAW,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;IAC3D,MAAM,WAAW,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;IAC3D,MAAM,WAAW,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;IAE3D,IAAI,WAAW,IAAI,WAAW,IAAI,WAAW,EAAE,CAAC;QAC9C,OAAO;YACL,WAAW,EAAE,IAAI;YACjB,WAAW,EAAE,IAAI;YACjB,WAAW,EAAE,IAAI;YACjB,MAAM,EAAE,aAAa;SACtB,CAAC;IACJ,CAAC;IAED,OAAO;QACL,WAAW;QACX,WAAW;QACX,WAAW;QACX,MAAM,EAAE,SAAS;KAClB,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB;IACtC,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,MAAM,uBAAuB,CAAC,IAAI,CAAC,CAAC;QACvD,IAAI,UAAU,EAAE,CAAC;YACf,OAAO;gBACL,SAAS,EAAE,IAAI;gBACf,OAAO,EAAE,UAAU,CAAC,MAAM,KAAK,OAAO;gBACtC,GAAG,EAAE,UAAU,CAAC,GAAG;gBACnB,YAAY,EAAE,UAAU,CAAC,YAAY;gBACrC,OAAO,EAAE,UAAU,CAAC,OAAO;aAC5B,CAAC;QACJ,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,mDAAmD;IACrD,CAAC;IAED,OAAO;QACL,SAAS,EAAE,KAAK;QAChB,OAAO,EAAE,KAAK;QACd,GAAG,EAAE,CAAC;KACP,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,SAAS,sCAAsC;IAC7C,MAAM,QAAQ,GAA2B;QACvC,iCAAiC,EAAE,IAAI;QACvC,oCAAoC,EAAE,MAAM;QAC5C,gCAAgC,EAAE,IAAI;QACtC,mCAAmC,EAAE,MAAM;QAC3C,sCAAsC,EAAE,MAAM;QAC9C,+CAA+C,EAAE,IAAI;QACrD,2CAA2C,EAAE,KAAK;QAClD,+CAA+C,EAAE,MAAM;KACxD,CAAC;IAEF,qDAAqD;IACrD,KAAK,MAAM,CAAC,GAAG,EAAE,YAAY,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC3D,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,YAAY,CAAC;QAClC,CAAC;IACH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,sBAAsB,CAAC,QAAgB,EAAE,QAAgB,EAAE,QAAgB;IACzF,wEAAwE;IACxE,6FAA6F;IAC7F,kHAAkH;IAClH,OAAO,CAAC,GAAG,CAAC,wBAAwB,GAAG,QAAQ,CAAC;IAChD,OAAO,CAAC,GAAG,CAAC,wBAAwB,GAAG,QAAQ,CAAC;IAChD,OAAO,CAAC,GAAG,CAAC,wBAAwB,GAAG,QAAQ,CAAC;IAEhD,gDAAgD;IAChD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,EAAE,CAAC;QAC5H,MAAM,IAAI,KAAK,CAAC,2FAA2F,CAAC,CAAC;IAC/G,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAAc,EACd,YAAqB,EACrB,GAAuB;IAEvB,4BAA4B;IAC5B,yBAAyB,GAAG,IAAI,CAAC;IACjC,wBAAwB,GAAG,IAAI,CAAC;IAEhC,8EAA8E;IAC9E,+EAA+E;IAC/E,+EAA+E;IAC/E,iHAAiH;IACjH,IAAI,kBAAkB,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;IAC9D,IAAI,kBAAkB,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;IAC9D,IAAI,kBAAkB,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;IAE9D,IAAI,CAAC,CAAC,kBAAkB,IAAI,CAAC,kBAAkB,IAAI,CAAC,kBAAkB,CAAC,IAAI,GAAG,EAAE,CAAC;QAC/E,IAAI,CAAC;YACH,MAAM,CAAC,IAAI,CAAC,kEAAkE,CAAC,CAAC;YAChF,MAAM,WAAW,GAAG,MAAM,0BAA0B,CAAC,GAAG,CAAC,CAAC;YAC1D,kBAAkB,GAAG,WAAW,CAAC,QAAQ,CAAC;YAC1C,kBAAkB,GAAG,WAAW,CAAC,QAAQ,CAAC;YAC1C,kBAAkB,GAAG,WAAW,CAAC,QAAQ,CAAC;YAE1C,qHAAqH;YACrH,kHAAkH;YAClH,0GAA0G;YAC1G,sBAAsB,CAAC,kBAAkB,EAAE,kBAAkB,EAAE,kBAAkB,CAAC,CAAC;YAEnF,wEAAwE;YACxE,MAAM,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;YAC5D,MAAM,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;YAC5D,MAAM,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;YAE5D,IAAI,CAAC,cAAc,IAAI,CAAC,cAAc,IAAI,CAAC,cAAc,EAAE,CAAC;gBAC1D,MAAM,CAAC,KAAK,CAAC,wGAAwG,CAAC,CAAC;gBACvH,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;YAC7E,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,iFAAiF,CAAC,CAAC;QACjG,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,QAAQ,GAAG,gEAAgE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YAC1I,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,iEAAiE;IACjE,0EAA0E;IAC1E,IAAI,CAAC,kBAAkB,IAAI,CAAC,kBAAkB,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACtE,MAAM,QAAQ,GAAG,0LAA0L,CAAC;QAC5M,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC5B,CAAC;IAED,kFAAkF;IAClF,MAAM,CAAC,IAAI,CAAC,sDAAsD,CAAC,CAAC;IACpE,MAAM,aAAa,GAAG,MAAM,gBAAgB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAE3D,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,CAAC;QAC9B,MAAM,CAAC,KAAK,CAAC,kDAAkD,CAAC,CAAC;QACjE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;IACpE,CAAC;IAED,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,CAAC;QAC9B,MAAM,CAAC,KAAK,CAAC,kDAAkD,CAAC,CAAC;QACjE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;IACpE,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,6BAA6B,aAAa,CAAC,UAAU,EAAE,CAAC,CAAC;IACrE,MAAM,CAAC,IAAI,CAAC,6BAA6B,aAAa,CAAC,UAAU,EAAE,CAAC,CAAC;IAErE,2EAA2E;IAC3E,6FAA6F;IAC7F,uEAAuE;IACvE,sCAAsC,EAAE,CAAC;IAEzC,0BAA0B;IAC1B,+EAA+E;IAC/E,IAAI,aAAa,GAAG,YAAY,CAAC;IACjC,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,sCAAsC;QACtC,6DAA6D;QAC7D,aAAa,GAAG,cAAc,CAAC;IACjC,CAAC;IACD,aAAa,GAAG,aAAa,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;IAEnF,0CAA0C;IAC1C,6EAA6E;IAC7E,sEAAsE;IACtE,8FAA8F;IAC9F,2EAA2E;IAC3E,MAAM,YAAY,GAAsB;QACtC,GAAG,OAAO,CAAC,GAAG,EAAE,yEAAyE;QACzF,uBAAuB,EAAE,aAAa;QACtC,6BAA6B,EAAE,MAAM,CAAC,cAAc,CAAC;KACtD,CAAC;IAEF,0BAA0B;IAC1B,MAAM,gBAAgB,GAAG,KAAK,CAAC,aAAa,CAAC,UAAU,EAAE,CAAC,UAAU,EAAE,aAAa,CAAC,UAAU,CAAC,EAAE;QAC/F,GAAG,EAAE,YAAY;QACjB,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;QACjC,QAAQ,EAAE,KAAK;KAChB,CAAC,CAAC;IAEH,gDAAgD;IAChD,sBAAsB,GAAG,gBAAgB,CAAC;IAC1C,yBAAyB,GAAG,IAAI,CAAC;IACjC,uBAAuB,GAAG,IAAI,CAAC;IAC/B,qBAAqB,GAAG,EAAE,CAAC;IAE3B,gBAAgB,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;QAC3C,MAAM,CAAC,KAAK,CAAC,eAAe,IAAI,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IACxD,CAAC,CAAC,CAAC;IAEH,gBAAgB,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;QAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC;QACzC,qBAAqB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtC,mDAAmD;QACnD,IAAI,qBAAqB,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YACtC,qBAAqB,CAAC,KAAK,EAAE,CAAC;QAChC,CAAC;QACD,MAAM,CAAC,KAAK,CAAC,mBAAmB,SAAS,EAAE,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,gBAAgB,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;QACnC,yBAAyB,GAAG,GAAG,CAAC,OAAO,CAAC;QACxC,MAAM,CAAC,KAAK,CAAC,gCAAgC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;IAEH,gBAAgB,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;QACnC,uBAAuB,GAAG,IAAI,CAAC;QAC/B,sBAAsB,GAAG,IAAI,CAAC,CAAC,qCAAqC;QACpE,MAAM,CAAC,IAAI,CAAC,wCAAwC,IAAI,EAAE,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,IAAI,CAAC,gCAAgC,gBAAgB,CAAC,GAAG,eAAe,aAAa,EAAE,CAAC,CAAC;IAEhG,OAAO;QACL,OAAO,EAAE,gBAAgB;QACzB,QAAQ,EAAE,UAAU,aAAa,EAAE;KACpC,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,yBAAyB;IAQvC,OAAO;QACL,UAAU,EAAE,sBAAsB,KAAK,IAAI;QAC3C,GAAG,EAAE,sBAAsB,EAAE,GAAG,IAAI,IAAI;QACxC,QAAQ,EAAE,uBAAuB;QACjC,UAAU,EAAE,yBAAyB;QACrC,SAAS,EAAE,wBAAwB;QACnC,MAAM,EAAE,CAAC,GAAG,qBAAqB,CAAC,EAAE,gBAAgB;KACrD,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,MAAc,EACd,mBAA4B,EAC5B,GAAuB;IAEvB,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,+CAA+C;IAC/C,MAAM,MAAM,GAAG,MAAM,kBAAkB,EAAE,CAAC;IAC1C,IAAI,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;QAC9D,MAAM,CAAC,IAAI,CAAC,0CAA0C,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC;QAC7E,OAAO,UAAU,MAAM,CAAC,YAAY,EAAE,CAAC;IACzC,CAAC;IAED,mCAAmC;IACnC,IAAI,CAAC;QACH,gCAAgC;QAChC,wBAAwB,GAAG,IAAI,CAAC;QAEhC,MAAM,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;QAC/D,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC;QAElE,qDAAqD;QACrD,qBAAqB;QACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,kBAAkB,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC,CAAC;YACnE,MAAM,SAAS,GAAG,MAAM,kBAAkB,EAAE,CAAC;YAC7C,IAAI,SAAS,CAAC,SAAS,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;gBAC7C,MAAM,CAAC,IAAI,CAAC,qCAAqC,SAAS,CAAC,YAAY,IAAI,QAAQ,EAAE,CAAC,CAAC;gBACvF,OAAO,SAAS,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,SAAS,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC;YAChF,CAAC;QACH,CAAC;QAED,4EAA4E;QAC5E,MAAM,CAAC,IAAI,CAAC,4EAA4E,CAAC,CAAC;QAC1F,OAAO,QAAQ,CAAC;IAClB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,QAAQ,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAClE,wBAAwB,GAAG,QAAQ,CAAC;QACpC,MAAM,CAAC,KAAK,CAAC,0CAA0C,QAAQ,EAAE,CAAC,CAAC;QACnE,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC","sourcesContent":["import { spawn, ChildProcess } from 'child_process';\r\nimport { createWriteStream, existsSync, chmodSync, mkdirSync } from 'fs';\r\nimport { join, dirname } from 'path';\r\nimport { pipeline } from 'stream/promises';\r\nimport { createGunzip } from 'zlib';\r\nimport { createHash } from 'node:crypto';\r\nimport type { Logger } from 'pino';\r\nimport dgram from 'dgram';\r\nimport type { EnvironmentConfig } from './types.js';\r\nimport { hostToHttpUrl } from './utils/urls.js';\r\n\r\n// Protocol is httpprotobuf \r\n\r\ninterface CollectorDiscoveryEntry {\r\n version: string;\r\n status: string;\r\n pid: number;\r\n otlpEndpoint: string;\r\n}\r\n\r\nexport interface CollectorStatus {\r\n isRunning: boolean;\r\n isReady: boolean;\r\n pid: number;\r\n otlpEndpoint?: string;\r\n version?: string;\r\n}\r\n\r\nconst COLLECTOR_VERSION = '1.0.1'; // Match C# collector version (from collector-version.json)\r\nconst COLLECTOR_DOWNLOAD_BASE = `https://collectors.beamable.com/version/${COLLECTOR_VERSION}`;\r\nconst DISCOVERY_PORT = parseInt(process.env.BEAM_COLLECTOR_DISCOVERY_PORT || '8688', 10);\r\n\r\n// Global storage for collector process and errors (to prevent GC and track status)\r\nlet globalCollectorProcess: ChildProcess | null = null;\r\nlet globalCollectorStartError: string | null = null;\r\nlet globalCollectorExitCode: number | null = null;\r\nlet globalCollectorStderr: string[] = [];\r\nlet globalCollectorInitError: string | null = null; // Tracks errors from discoverOrStartCollector\r\nconst DISCOVERY_DELAY = 100; // milliseconds\r\nconst DISCOVERY_ATTEMPTS = 10;\r\n\r\n/**\r\n * ClickHouse credentials response from Beamable API\r\n */\r\ninterface ClickHouseCredentials {\r\n endpoint: string;\r\n expiresAt?: string;\r\n password: string;\r\n username: string;\r\n}\r\n\r\n/**\r\n * Calculates Beamable signature for signed requests\r\n * Signature format: MD5(secret + pid + version + uriPathAndQuery + body) as Base64\r\n * Matches C# SignedRequesterHelper.CalculateSignature\r\n */\r\nfunction calculateSignature(\r\n pid: string,\r\n secret: string,\r\n uriPathAndQuery: string,\r\n body: string | null = null,\r\n version: string = '1'\r\n): string {\r\n let dataToSign = `${secret}${pid}${version}${uriPathAndQuery}`;\r\n if (body) {\r\n dataToSign += body;\r\n }\r\n \r\n // MD5 hash to Base64 (matching C# implementation)\r\n const hash = createHash('md5').update(dataToSign, 'utf8').digest('base64');\r\n return hash;\r\n}\r\n\r\n/**\r\n * Extracts path and query from a URL\r\n */\r\nfunction getPathAndQuery(url: string): string {\r\n try {\r\n const urlObj = new URL(url);\r\n return urlObj.pathname + urlObj.search;\r\n } catch {\r\n // If URL parsing fails, return as-is\r\n return url;\r\n }\r\n}\r\n\r\n/**\r\n * Fetches ClickHouse credentials from Beamable API\r\n * GET /api/beamo/otel/auth/writer/config\r\n * Requires signed request authentication (X-BEAM-SIGNATURE header)\r\n */\r\nexport async function fetchClickHouseCredentials(\r\n env: EnvironmentConfig\r\n): Promise<ClickHouseCredentials> {\r\n const apiUrl = hostToHttpUrl(env.host);\r\n const uriPath = '/api/beamo/otel/auth/writer/config';\r\n const configUrl = new URL(uriPath, apiUrl).toString();\r\n \r\n // Get secret from environment (required for signed requests)\r\n const secret = process.env.SECRET;\r\n if (!secret) {\r\n throw new Error('SECRET environment variable is required to fetch ClickHouse credentials');\r\n }\r\n \r\n // Calculate signature for signed request (matching C# HttpSignedRequester)\r\n const pathAndQuery = getPathAndQuery(uriPath);\r\n const signature = calculateSignature(env.pid, secret, pathAndQuery, null, '1');\r\n \r\n // Build headers with signed authentication (matching C# HttpSignedRequester)\r\n const headers: Record<string, string> = {\r\n 'Content-Type': 'application/json',\r\n Accept: 'application/json',\r\n 'X-BEAM-SCOPE': `${env.cid}.${env.pid}`,\r\n 'X-BEAM-SIGNATURE': signature,\r\n };\r\n \r\n const response = await fetch(configUrl, {\r\n method: 'GET',\r\n headers,\r\n });\r\n \r\n if (!response.ok) {\r\n const errorText = await response.text().catch(() => 'Unknown error');\r\n throw new Error(`Failed to fetch ClickHouse credentials from ${configUrl}: ${response.status} ${response.statusText} - ${errorText.substring(0, 200)}`);\r\n }\r\n \r\n const credentials = await response.json() as ClickHouseCredentials;\r\n \r\n if (!credentials.endpoint || !credentials.username || !credentials.password) {\r\n throw new Error('Invalid ClickHouse credentials response: missing required fields');\r\n }\r\n \r\n return credentials;\r\n}\r\n\r\n/**\r\n * Gets the collector storage directory (similar to C# LocalApplicationData/beam/collectors/version)\r\n */\r\nfunction getCollectorStoragePath(): string {\r\n // Use temp directory for now - in containers this should be writable\r\n const tempDir = process.env.TMPDIR || process.env.TMP || '/tmp';\r\n return join(tempDir, 'beam', 'collectors', COLLECTOR_VERSION);\r\n}\r\n\r\n/**\r\n * Gets the collector binary name for the current platform\r\n */\r\nfunction getCollectorBinaryName(): string {\r\n const platform = process.platform;\r\n const arch = process.arch;\r\n \r\n // Match C# naming: collector-{osArchSuffix}\r\n // C# returns \"collector-linux-amd64\", not \"beamable-collector-linux-amd64\"\r\n if (platform === 'linux' && arch === 'x64') {\r\n return 'collector-linux-amd64';\r\n } else if (platform === 'linux' && arch === 'arm64') {\r\n return 'collector-linux-arm64';\r\n } else if (platform === 'darwin' && arch === 'x64') {\r\n return 'collector-darwin-amd64';\r\n } else if (platform === 'darwin' && arch === 'arm64') {\r\n return 'collector-darwin-arm64';\r\n } else if (platform === 'win32' && arch === 'x64') {\r\n return 'collector-windows-amd64.exe';\r\n }\r\n \r\n throw new Error(`Unsupported platform: ${platform} ${arch}`);\r\n}\r\n\r\n/**\r\n * Downloads and decompresses a gzipped file\r\n */\r\nasync function downloadAndDecompressGzip(url: string, outputPath: string, makeExecutable: boolean = false): Promise<void> {\r\n let response: Response;\r\n try {\r\n // Add timeout and better error handling\r\n const controller = new AbortController();\r\n const timeoutId = setTimeout(() => controller.abort(), 30000); // 30 second timeout\r\n \r\n try {\r\n response = await fetch(url, {\r\n signal: controller.signal,\r\n headers: {\r\n 'User-Agent': 'Beamable-Node-Microservice-Runtime/1.0',\r\n },\r\n });\r\n clearTimeout(timeoutId);\r\n } catch (fetchError) {\r\n clearTimeout(timeoutId);\r\n if (fetchError instanceof Error) {\r\n // Provide more context about the error\r\n if (fetchError.name === 'AbortError') {\r\n throw new Error(`Timeout downloading ${url} (30s limit exceeded)`);\r\n }\r\n // Check for common network errors\r\n const errorMsg = fetchError.message.toLowerCase();\r\n if (errorMsg.includes('enotfound') || errorMsg.includes('dns')) {\r\n throw new Error(`DNS resolution failed for ${url}. Check network connectivity and DNS settings.`);\r\n }\r\n if (errorMsg.includes('econnrefused') || errorMsg.includes('connection refused')) {\r\n throw new Error(`Connection refused to ${url}. Server may be down or firewall blocking.`);\r\n }\r\n if (errorMsg.includes('etimedout') || errorMsg.includes('timeout')) {\r\n throw new Error(`Connection timeout to ${url}. Network may be slow or unreachable.`);\r\n }\r\n if (errorMsg.includes('certificate') || errorMsg.includes('ssl') || errorMsg.includes('tls')) {\r\n throw new Error(`SSL/TLS error connecting to ${url}: ${fetchError.message}`);\r\n }\r\n throw new Error(`Network error downloading ${url}: ${fetchError.name} - ${fetchError.message}`);\r\n }\r\n throw new Error(`Network error downloading ${url}: ${String(fetchError)}`);\r\n }\r\n } catch (error) {\r\n // Re-throw with more context\r\n if (error instanceof Error) {\r\n if (error.message.includes('Timeout') || error.message.includes('DNS') || error.message.includes('Connection')) {\r\n throw error; // Already has good context\r\n }\r\n throw new Error(`Network error downloading ${url}: ${error.message}`);\r\n }\r\n throw new Error(`Network error downloading ${url}: ${String(error)}`);\r\n }\r\n \r\n if (!response.ok) {\r\n const errorText = await response.text().catch(() => '');\r\n throw new Error(`HTTP ${response.status} downloading ${url}: ${response.statusText}${errorText ? ` - ${errorText.substring(0, 200)}` : ''}`);\r\n }\r\n \r\n const dir = dirname(outputPath);\r\n if (!existsSync(dir)) {\r\n mkdirSync(dir, { recursive: true });\r\n }\r\n \r\n const gunzip = createGunzip();\r\n const writeStream = createWriteStream(outputPath);\r\n \r\n await pipeline(response.body as any, gunzip, writeStream);\r\n \r\n if (makeExecutable && process.platform !== 'win32') {\r\n try {\r\n chmodSync(outputPath, 0o755);\r\n } catch (error) {\r\n console.error(`Failed to make ${outputPath} executable:`, error);\r\n }\r\n }\r\n}\r\n\r\n/**\r\n * Resolves the collector binary and config, downloading if needed\r\n */\r\nasync function resolveCollector(allowDownload: boolean = true, logger?: Logger): Promise<{ binaryPath: string | null; configPath: string | null }> {\r\n const basePath = getCollectorStoragePath();\r\n const binaryName = getCollectorBinaryName();\r\n const configName = 'clickhouse-config.yaml';\r\n \r\n const binaryPath = join(basePath, binaryName);\r\n const configPath = join(basePath, configName);\r\n \r\n const itemsToDownload: Array<{ url: string; path: string; executable: boolean }> = [];\r\n \r\n if (!existsSync(binaryPath) && allowDownload) {\r\n const binaryUrl = `${COLLECTOR_DOWNLOAD_BASE}/${binaryName}.gz`;\r\n itemsToDownload.push({ url: binaryUrl, path: binaryPath, executable: true });\r\n logger?.info(`[Collector] Will download binary from: ${binaryUrl}`);\r\n } else if (existsSync(binaryPath)) {\r\n logger?.info(`[Collector] Binary found at: ${binaryPath}`);\r\n }\r\n \r\n if (!existsSync(configPath) && allowDownload) {\r\n const configUrl = `${COLLECTOR_DOWNLOAD_BASE}/${configName}.gz`;\r\n itemsToDownload.push({ url: configUrl, path: configPath, executable: false });\r\n logger?.info(`[Collector] Will download config from: ${configUrl}`);\r\n } else if (existsSync(configPath)) {\r\n logger?.info(`[Collector] Config found at: ${configPath}`);\r\n }\r\n \r\n for (const item of itemsToDownload) {\r\n try {\r\n logger?.info(`[Collector] Downloading ${item.url}...`);\r\n await downloadAndDecompressGzip(item.url, item.path, item.executable);\r\n logger?.info(`[Collector] Downloaded to ${item.path}`);\r\n } catch (error) {\r\n const errorMsg = error instanceof Error ? error.message : String(error);\r\n logger?.error(`[Collector] Failed to download ${item.url}: ${errorMsg}`);\r\n throw new Error(`Failed to download collector ${item.executable ? 'binary' : 'config'} from ${item.url}: ${errorMsg}`);\r\n }\r\n }\r\n \r\n return {\r\n binaryPath: existsSync(binaryPath) ? binaryPath : null,\r\n configPath: existsSync(configPath) ? configPath : null,\r\n };\r\n}\r\n\r\n/**\r\n * Discovers collector via UDP broadcast\r\n */\r\nfunction discoverCollectorViaUDP(timeoutMs: number = 5000): Promise<CollectorDiscoveryEntry | null> {\r\n return new Promise((resolve) => {\r\n const socket = dgram.createSocket('udp4');\r\n const discovered: CollectorDiscoveryEntry[] = [];\r\n let timeout: NodeJS.Timeout;\r\n \r\n socket.on('message', (msg) => {\r\n try {\r\n const message = JSON.parse(msg.toString()) as CollectorDiscoveryEntry;\r\n // Check if version matches\r\n if (message.version === COLLECTOR_VERSION && message.status === 'READY') {\r\n discovered.push(message);\r\n }\r\n } catch (error) {\r\n // Ignore parse errors\r\n }\r\n });\r\n \r\n socket.on('error', () => {\r\n clearTimeout(timeout);\r\n socket.close();\r\n resolve(null);\r\n });\r\n \r\n socket.bind(() => {\r\n socket.setBroadcast(true);\r\n \r\n timeout = setTimeout(() => {\r\n socket.close();\r\n // Return the first discovered collector\r\n resolve(discovered.length > 0 ? discovered[0] : null);\r\n }, timeoutMs);\r\n });\r\n });\r\n}\r\n\r\n/**\r\n * Gets the current status of ClickHouse credentials (from env or API)\r\n */\r\nexport function getClickHouseCredentialsStatus(): {\r\n hasEndpoint: boolean;\r\n hasUsername: boolean;\r\n hasPassword: boolean;\r\n source: 'environment' | 'api' | 'missing';\r\n} {\r\n const hasEndpoint = !!process.env.BEAM_CLICKHOUSE_ENDPOINT;\r\n const hasUsername = !!process.env.BEAM_CLICKHOUSE_USERNAME;\r\n const hasPassword = !!process.env.BEAM_CLICKHOUSE_PASSWORD;\r\n \r\n if (hasEndpoint && hasUsername && hasPassword) {\r\n return {\r\n hasEndpoint: true,\r\n hasUsername: true,\r\n hasPassword: true,\r\n source: 'environment',\r\n };\r\n }\r\n \r\n return {\r\n hasEndpoint,\r\n hasUsername,\r\n hasPassword,\r\n source: 'missing',\r\n };\r\n}\r\n\r\n/**\r\n * Checks if collector is already running via UDP discovery\r\n */\r\nexport async function isCollectorRunning(): Promise<CollectorStatus> {\r\n try {\r\n const discovered = await discoverCollectorViaUDP(2000);\r\n if (discovered) {\r\n return {\r\n isRunning: true,\r\n isReady: discovered.status === 'READY',\r\n pid: discovered.pid,\r\n otlpEndpoint: discovered.otlpEndpoint,\r\n version: discovered.version,\r\n };\r\n }\r\n } catch (error) {\r\n // Discovery failed, collector probably not running\r\n }\r\n \r\n return {\r\n isRunning: false,\r\n isReady: false,\r\n pid: 0,\r\n };\r\n}\r\n\r\n/**\r\n * Sets default collector configuration environment variables in process.env\r\n * Matches C# CollectorManager.AddCollectorConfigurationToEnvironment()\r\n */\r\nfunction addCollectorConfigurationToEnvironment(): void {\r\n const defaults: Record<string, string> = {\r\n BEAM_CLICKHOUSE_PROCESSOR_TIMEOUT: '5s',\r\n BEAM_CLICKHOUSE_PROCESSOR_BATCH_SIZE: '5000',\r\n BEAM_CLICKHOUSE_EXPORTER_TIMEOUT: '5s',\r\n BEAM_CLICKHOUSE_EXPORTER_QUEUE_SIZE: '1000',\r\n BEAM_CLICKHOUSE_EXPORTER_RETRY_ENABLED: 'true',\r\n BEAM_CLICKHOUSE_EXPORTER_RETRY_INITIAL_INTERVAL: '5s',\r\n BEAM_CLICKHOUSE_EXPORTER_RETRY_MAX_INTERVAL: '30s',\r\n BEAM_CLICKHOUSE_EXPORTER_RETRY_MAX_ELAPSED_TIME: '300s',\r\n };\r\n \r\n // Only set if not already set (matching C# behavior)\r\n for (const [key, defaultValue] of Object.entries(defaults)) {\r\n if (!process.env[key]) {\r\n process.env[key] = defaultValue;\r\n }\r\n }\r\n}\r\n\r\n/**\r\n * Sets ClickHouse credentials in process.env (matching C# CollectorManager.AddAuthEnvironmentVars)\r\n */\r\nexport function addAuthEnvironmentVars(endpoint: string, username: string, password: string): void {\r\n // Always set the values when explicitly provided (matching C# behavior)\r\n // This ensures credentials fetched from API override any existing (potentially empty) values\r\n // In Node.js, process.env is a writable object - setting properties here persists for the lifetime of the process\r\n process.env.BEAM_CLICKHOUSE_ENDPOINT = endpoint;\r\n process.env.BEAM_CLICKHOUSE_USERNAME = username;\r\n process.env.BEAM_CLICKHOUSE_PASSWORD = password;\r\n \r\n // Verify they were actually set (for debugging)\r\n if (!process.env.BEAM_CLICKHOUSE_ENDPOINT || !process.env.BEAM_CLICKHOUSE_USERNAME || !process.env.BEAM_CLICKHOUSE_PASSWORD) {\r\n throw new Error(`Failed to set ClickHouse credentials in process.env - this should never happen in Node.js`);\r\n }\r\n}\r\n\r\n/**\r\n * Starts the OpenTelemetry collector process\r\n */\r\nexport async function startCollector(\r\n logger: Logger,\r\n otlpEndpoint?: string,\r\n env?: EnvironmentConfig\r\n): Promise<{ process: ChildProcess; endpoint: string }> {\r\n // Clear any previous errors\r\n globalCollectorStartError = null;\r\n globalCollectorInitError = null;\r\n \r\n // CRITICAL: Fetch credentials FIRST, before resolving collector binary/config\r\n // This ensures credentials are in process.env even if collector download fails\r\n // The credentials will persist in process.env for the lifetime of the process.\r\n // Per Gabriel: \"these OTEL related ones you need to do it yourself at the beginning of the microservice startup\"\r\n let clickhouseEndpoint = process.env.BEAM_CLICKHOUSE_ENDPOINT;\r\n let clickhouseUsername = process.env.BEAM_CLICKHOUSE_USERNAME;\r\n let clickhousePassword = process.env.BEAM_CLICKHOUSE_PASSWORD;\r\n \r\n if ((!clickhouseEndpoint || !clickhouseUsername || !clickhousePassword) && env) {\r\n try {\r\n logger.info('[Collector] Fetching ClickHouse credentials from Beamable API...');\r\n const credentials = await fetchClickHouseCredentials(env);\r\n clickhouseEndpoint = credentials.endpoint;\r\n clickhouseUsername = credentials.username;\r\n clickhousePassword = credentials.password;\r\n \r\n // CRITICAL: Set them in process.env IMMEDIATELY after fetching (matching C# CollectorManager.AddAuthEnvironmentVars)\r\n // In Node.js, process.env is a writable object - setting properties here persists for the lifetime of the process\r\n // This ensures they're available to both the Node.js process and inherited by the collector child process\r\n addAuthEnvironmentVars(clickhouseEndpoint, clickhouseUsername, clickhousePassword);\r\n \r\n // Verify they were set (defensive check - should never fail in Node.js)\r\n const verifyEndpoint = process.env.BEAM_CLICKHOUSE_ENDPOINT;\r\n const verifyUsername = process.env.BEAM_CLICKHOUSE_USERNAME;\r\n const verifyPassword = process.env.BEAM_CLICKHOUSE_PASSWORD;\r\n \r\n if (!verifyEndpoint || !verifyUsername || !verifyPassword) {\r\n logger.error(`[Collector] CRITICAL: Credentials were set but are missing from process.env! This should never happen.`);\r\n throw new Error('Failed to persist ClickHouse credentials in process.env');\r\n }\r\n \r\n logger.info('[Collector] ClickHouse credentials fetched from API and verified in process.env');\r\n } catch (error) {\r\n const errorMsg = `[Collector] Failed to fetch ClickHouse credentials from API: ${error instanceof Error ? error.message : String(error)}`;\r\n logger.error(errorMsg);\r\n throw new Error(errorMsg);\r\n }\r\n }\r\n \r\n // Validate required environment variables (matching C# behavior)\r\n // These must be set before starting the collector, otherwise it will fail\r\n if (!clickhouseEndpoint || !clickhouseUsername || !clickhousePassword) {\r\n const errorMsg = `[Collector] Required ClickHouse credentials are missing. Set BEAM_CLICKHOUSE_ENDPOINT, BEAM_CLICKHOUSE_USERNAME, and BEAM_CLICKHOUSE_PASSWORD, or ensure the API endpoint is accessible.`;\r\n logger.error(errorMsg);\r\n throw new Error(errorMsg);\r\n }\r\n \r\n // Now resolve collector binary and config (after credentials are fetched and set)\r\n logger.info('[Collector] Resolving collector binary and config...');\r\n const collectorInfo = await resolveCollector(true, logger);\r\n \r\n if (!collectorInfo.binaryPath) {\r\n logger.error('[Collector] Binary not found and download failed');\r\n throw new Error('Collector binary not found and download failed');\r\n }\r\n \r\n if (!collectorInfo.configPath) {\r\n logger.error('[Collector] Config not found and download failed');\r\n throw new Error('Collector config not found and download failed');\r\n }\r\n \r\n logger.info(`[Collector] Using binary: ${collectorInfo.binaryPath}`);\r\n logger.info(`[Collector] Using config: ${collectorInfo.configPath}`);\r\n \r\n // Set default collector configuration environment variables in process.env\r\n // This ensures they're available to both the Node.js process and the collector child process\r\n // Matches C# CollectorManager.AddCollectorConfigurationToEnvironment()\r\n addCollectorConfigurationToEnvironment();\r\n \r\n // Determine OTLP endpoint\r\n // Use a free port if not specified (like C# does with PortUtil.FreeEndpoint())\r\n let localEndpoint = otlpEndpoint;\r\n if (!localEndpoint) {\r\n // For now, use default OTLP HTTP port\r\n // In production, this would be discovered or set by Beamable\r\n localEndpoint = '0.0.0.0:4318';\r\n }\r\n localEndpoint = localEndpoint.replace(/^http:\\/\\//, '').replace(/^https:\\/\\//, '');\r\n \r\n // Set environment variables for collector\r\n // Note: BEAM_CLICKHOUSE_ENDPOINT is for collector → ClickHouse communication\r\n // This is different from the OTLP endpoint (microservice → collector)\r\n // The collector process will inherit all environment variables from process.env automatically\r\n // We only need to set the OTLP endpoint and discovery port explicitly here\r\n const collectorEnv: NodeJS.ProcessEnv = {\r\n ...process.env, // Inherit all process.env (including credentials and config we just set)\r\n BEAM_OTLP_HTTP_ENDPOINT: localEndpoint,\r\n BEAM_COLLECTOR_DISCOVERY_PORT: String(DISCOVERY_PORT),\r\n };\r\n \r\n // Start collector process\r\n const collectorProcess = spawn(collectorInfo.binaryPath, ['--config', collectorInfo.configPath], {\r\n env: collectorEnv,\r\n stdio: ['ignore', 'pipe', 'pipe'],\r\n detached: false,\r\n });\r\n \r\n // Store globally to prevent GC and track status\r\n globalCollectorProcess = collectorProcess;\r\n globalCollectorStartError = null;\r\n globalCollectorExitCode = null;\r\n globalCollectorStderr = [];\r\n \r\n collectorProcess.stdout?.on('data', (data) => {\r\n logger.debug(`[Collector] ${data.toString().trim()}`);\r\n });\r\n \r\n collectorProcess.stderr?.on('data', (data) => {\r\n const errorLine = data.toString().trim();\r\n globalCollectorStderr.push(errorLine);\r\n // Keep only last 50 lines to prevent memory issues\r\n if (globalCollectorStderr.length > 50) {\r\n globalCollectorStderr.shift();\r\n }\r\n logger.debug(`[Collector ERR] ${errorLine}`);\r\n });\r\n \r\n collectorProcess.on('error', (err) => {\r\n globalCollectorStartError = err.message;\r\n logger.error(`[Collector] Failed to start: ${err.message}`);\r\n });\r\n \r\n collectorProcess.on('exit', (code) => {\r\n globalCollectorExitCode = code;\r\n globalCollectorProcess = null; // Clear reference when process exits\r\n logger.warn(`[Collector] Process exited with code ${code}`);\r\n });\r\n \r\n logger.info(`[Collector] Started with PID ${collectorProcess.pid}, endpoint: ${localEndpoint}`);\r\n \r\n return {\r\n process: collectorProcess,\r\n endpoint: `http://${localEndpoint}`,\r\n };\r\n}\r\n\r\n/**\r\n * Gets detailed collector status including process info and errors\r\n */\r\nexport function getCollectorProcessStatus(): {\r\n hasProcess: boolean;\r\n pid: number | null;\r\n exitCode: number | null;\r\n startError: string | null;\r\n initError: string | null;\r\n stderr: string[];\r\n} {\r\n return {\r\n hasProcess: globalCollectorProcess !== null,\r\n pid: globalCollectorProcess?.pid ?? null,\r\n exitCode: globalCollectorExitCode,\r\n startError: globalCollectorStartError,\r\n initError: globalCollectorInitError,\r\n stderr: [...globalCollectorStderr], // Return a copy\r\n };\r\n}\r\n\r\n/**\r\n * Discovers or starts the collector and returns the OTLP endpoint\r\n */\r\nexport async function discoverOrStartCollector(\r\n logger: Logger,\r\n standardOtelEnabled: boolean,\r\n env?: EnvironmentConfig\r\n): Promise<string | null> {\r\n if (!standardOtelEnabled) {\r\n return null;\r\n }\r\n \r\n // First, check if collector is already running\r\n const status = await isCollectorRunning();\r\n if (status.isRunning && status.isReady && status.otlpEndpoint) {\r\n logger.info(`[Collector] Found running collector at ${status.otlpEndpoint}`);\r\n return `http://${status.otlpEndpoint}`;\r\n }\r\n \r\n // Collector not running - start it\r\n try {\r\n // Clear any previous init error\r\n globalCollectorInitError = null;\r\n \r\n logger.info('[Collector] Starting OpenTelemetry collector...');\r\n const { endpoint } = await startCollector(logger, undefined, env);\r\n \r\n // Wait a bit for collector to start and become ready\r\n // Try to discover it\r\n for (let i = 0; i < DISCOVERY_ATTEMPTS; i++) {\r\n await new Promise(resolve => setTimeout(resolve, DISCOVERY_DELAY));\r\n const newStatus = await isCollectorRunning();\r\n if (newStatus.isRunning && newStatus.isReady) {\r\n logger.info(`[Collector] Collector is ready at ${newStatus.otlpEndpoint || endpoint}`);\r\n return newStatus.otlpEndpoint ? `http://${newStatus.otlpEndpoint}` : endpoint;\r\n }\r\n }\r\n \r\n // Return the endpoint we started with, even if discovery didn't find it yet\r\n logger.warn('[Collector] Collector started but not yet ready, using configured endpoint');\r\n return endpoint;\r\n } catch (err) {\r\n const errorMsg = err instanceof Error ? err.message : String(err);\r\n globalCollectorInitError = errorMsg;\r\n logger.error(`[Collector] Failed to start collector: ${errorMsg}`);\r\n return null;\r\n }\r\n}\r\n\r\n"]}
|
|
1
|
+
{"version":3,"file":"collector-manager.js","sourceRoot":"","sources":["../src/collector-manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAgB,MAAM,eAAe,CAAC;AACpD,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AACzE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AACrC,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,MAAM,MAAM,CAAC;AACpC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAmBhD,MAAM,iBAAiB,GAAG,OAAO,CAAC,CAAC,2DAA2D;AAC9F,MAAM,uBAAuB,GAAG,2CAA2C,iBAAiB,EAAE,CAAC;AAC/F,MAAM,cAAc,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,6BAA6B,IAAI,MAAM,EAAE,EAAE,CAAC,CAAC;AAEzF,mFAAmF;AACnF,IAAI,sBAAsB,GAAwB,IAAI,CAAC;AACvD,IAAI,yBAAyB,GAAkB,IAAI,CAAC;AACpD,IAAI,uBAAuB,GAAkB,IAAI,CAAC;AAClD,IAAI,qBAAqB,GAAa,EAAE,CAAC;AACzC,IAAI,wBAAwB,GAAkB,IAAI,CAAC,CAAC,8CAA8C;AAClG,MAAM,eAAe,GAAG,GAAG,CAAC,CAAC,eAAe;AAC5C,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAY9B;;;;GAIG;AACH,SAAS,kBAAkB,CACzB,GAAW,EACX,MAAc,EACd,eAAuB,EACvB,OAAsB,IAAI,EAC1B,UAAkB,GAAG;IAErB,IAAI,UAAU,GAAG,GAAG,MAAM,GAAG,GAAG,GAAG,OAAO,GAAG,eAAe,EAAE,CAAC;IAC/D,IAAI,IAAI,EAAE,CAAC;QACT,UAAU,IAAI,IAAI,CAAC;IACrB,CAAC;IAED,kDAAkD;IAClD,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC3E,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,GAAW;IAClC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,OAAO,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC;IACzC,CAAC;IAAC,MAAM,CAAC;QACP,qCAAqC;QACrC,OAAO,GAAG,CAAC;IACb,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC9C,GAAsB;IAEtB,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,OAAO,GAAG,oCAAoC,CAAC;IACrD,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC;IAEtD,6DAA6D;IAC7D,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC;IAClC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CAAC,yEAAyE,CAAC,CAAC;IAC7F,CAAC;IAED,2EAA2E;IAC3E,MAAM,YAAY,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IAC9C,MAAM,SAAS,GAAG,kBAAkB,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,YAAY,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;IAE/E,6EAA6E;IAC7E,MAAM,OAAO,GAA2B;QACtC,cAAc,EAAE,kBAAkB;QAClC,MAAM,EAAE,kBAAkB;QAC1B,cAAc,EAAE,GAAG,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,EAAE;QACvC,kBAAkB,EAAE,SAAS;KAC9B,CAAC;IAEF,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE;QACtC,MAAM,EAAE,KAAK;QACb,OAAO;KACR,CAAC,CAAC;IAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,eAAe,CAAC,CAAC;QACrE,MAAM,IAAI,KAAK,CAAC,+CAA+C,SAAS,KAAK,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,MAAM,SAAS,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IAC1J,CAAC;IAED,MAAM,WAAW,GAAG,MAAM,QAAQ,CAAC,IAAI,EAA2B,CAAC;IAEnE,IAAI,CAAC,WAAW,CAAC,QAAQ,IAAI,CAAC,WAAW,CAAC,QAAQ,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC;QAC5E,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;IACtF,CAAC;IAED,OAAO,WAAW,CAAC;AACrB,CAAC;AAED;;GAEG;AACH,SAAS,uBAAuB;IAC9B,qEAAqE;IACrE,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,MAAM,CAAC;IAChE,OAAO,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,iBAAiB,CAAC,CAAC;AAChE,CAAC;AAED;;GAEG;AACH,SAAS,sBAAsB;IAC7B,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;IAClC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAE1B,4CAA4C;IAC5C,2EAA2E;IAC3E,IAAI,QAAQ,KAAK,OAAO,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;QAC3C,OAAO,uBAAuB,CAAC;IACjC,CAAC;SAAM,IAAI,QAAQ,KAAK,OAAO,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;QACpD,OAAO,uBAAuB,CAAC;IACjC,CAAC;SAAM,IAAI,QAAQ,KAAK,QAAQ,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;QACnD,OAAO,wBAAwB,CAAC;IAClC,CAAC;SAAM,IAAI,QAAQ,KAAK,QAAQ,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;QACrD,OAAO,wBAAwB,CAAC;IAClC,CAAC;SAAM,IAAI,QAAQ,KAAK,OAAO,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;QAClD,OAAO,6BAA6B,CAAC;IACvC,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,yBAAyB,QAAQ,IAAI,IAAI,EAAE,CAAC,CAAC;AAC/D,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,yBAAyB,CAAC,GAAW,EAAE,UAAkB,EAAE,iBAA0B,KAAK;IACvG,IAAI,QAAkB,CAAC;IACvB,IAAI,CAAC;QACH,wCAAwC;QACxC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC,oBAAoB;QAEnF,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAC1B,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,OAAO,EAAE;oBACP,YAAY,EAAE,wCAAwC;iBACvD;aACF,CAAC,CAAC;YACH,YAAY,CAAC,SAAS,CAAC,CAAC;QAC1B,CAAC;QAAC,OAAO,UAAU,EAAE,CAAC;YACpB,YAAY,CAAC,SAAS,CAAC,CAAC;YACxB,IAAI,UAAU,YAAY,KAAK,EAAE,CAAC;gBAChC,8CAA8C;gBAC9C,IAAI,eAAe,GAAG,EAAE,CAAC;gBACzB,IAAK,UAAkB,CAAC,KAAK,EAAE,CAAC;oBAC9B,MAAM,KAAK,GAAI,UAAkB,CAAC,KAAK,CAAC;oBACxC,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;wBAC3B,MAAM,SAAS,GAAI,KAAa,CAAC,IAAI,CAAC;wBACtC,eAAe,GAAG,YAAY,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,WAAW,SAAS,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC;oBAC5G,CAAC;yBAAM,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;wBACnD,eAAe,GAAG,iBAAiB,KAAK,CAAC,IAAI,GAAG,CAAC;oBACnD,CAAC;gBACH,CAAC;gBAED,uCAAuC;gBACvC,IAAI,UAAU,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;oBACrC,MAAM,IAAI,KAAK,CAAC,uBAAuB,GAAG,wBAAwB,eAAe,EAAE,CAAC,CAAC;gBACvF,CAAC;gBAED,wDAAwD;gBACxD,MAAM,SAAS,GAAI,UAAkB,CAAC,IAAI,IAAI,CAAE,UAAkB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;gBAChF,IAAI,SAAS,EAAE,CAAC;oBACd,IAAI,SAAS,KAAK,WAAW,IAAI,SAAS,KAAK,WAAW,EAAE,CAAC;wBAC3D,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,WAAW,SAAS,iDAAiD,CAAC,CAAC;oBACzH,CAAC;oBACD,IAAI,SAAS,KAAK,cAAc,EAAE,CAAC;wBACjC,MAAM,IAAI,KAAK,CAAC,yBAAyB,GAAG,WAAW,SAAS,6CAA6C,CAAC,CAAC;oBACjH,CAAC;oBACD,IAAI,SAAS,KAAK,WAAW,IAAI,SAAS,KAAK,YAAY,EAAE,CAAC;wBAC5D,MAAM,IAAI,KAAK,CAAC,+BAA+B,GAAG,WAAW,SAAS,wCAAwC,CAAC,CAAC;oBAClH,CAAC;oBACD,IAAI,SAAS,KAAK,kBAAkB,IAAI,SAAS,KAAK,iCAAiC,EAAE,CAAC;wBACxF,MAAM,IAAI,KAAK,CAAC,2CAA2C,GAAG,WAAW,SAAS,MAAM,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC;oBAChH,CAAC;gBACH,CAAC;gBAED,6CAA6C;gBAC7C,MAAM,QAAQ,GAAG,UAAU,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;gBAClD,IAAI,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC/D,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,iDAAiD,eAAe,EAAE,CAAC,CAAC;gBACtH,CAAC;gBACD,IAAI,QAAQ,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,oBAAoB,CAAC,EAAE,CAAC;oBACjF,MAAM,IAAI,KAAK,CAAC,yBAAyB,GAAG,6CAA6C,eAAe,EAAE,CAAC,CAAC;gBAC9G,CAAC;gBACD,IAAI,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;oBACnE,MAAM,IAAI,KAAK,CAAC,yBAAyB,GAAG,wCAAwC,eAAe,EAAE,CAAC,CAAC;gBACzG,CAAC;gBACD,IAAI,QAAQ,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC7F,MAAM,IAAI,KAAK,CAAC,+BAA+B,GAAG,KAAK,UAAU,CAAC,OAAO,GAAG,eAAe,EAAE,CAAC,CAAC;gBACjG,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,KAAK,UAAU,CAAC,IAAI,MAAM,UAAU,CAAC,OAAO,GAAG,eAAe,EAAE,CAAC,CAAC;YACpH,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,KAAK,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC7E,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,6BAA6B;QAC7B,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;gBAC/G,MAAM,KAAK,CAAC,CAAC,2BAA2B;YAC1C,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACxE,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,KAAK,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACxE,CAAC;IAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QACxD,MAAM,IAAI,KAAK,CAAC,QAAQ,QAAQ,CAAC,MAAM,gBAAgB,GAAG,KAAK,QAAQ,CAAC,UAAU,GAAG,SAAS,CAAC,CAAC,CAAC,MAAM,SAAS,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC/I,CAAC;IAED,MAAM,GAAG,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IAChC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACrB,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtC,CAAC;IAED,MAAM,MAAM,GAAG,YAAY,EAAE,CAAC;IAC9B,MAAM,WAAW,GAAG,iBAAiB,CAAC,UAAU,CAAC,CAAC;IAElD,MAAM,QAAQ,CAAC,QAAQ,CAAC,IAAW,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;IAE1D,IAAI,cAAc,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;QACnD,IAAI,CAAC;YACH,SAAS,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QAC/B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,kBAAkB,UAAU,cAAc,EAAE,KAAK,CAAC,CAAC;QACnE,CAAC;IACH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,gBAAgB,CAAC,gBAAyB,IAAI,EAAE,MAAe;IAC5E,MAAM,QAAQ,GAAG,uBAAuB,EAAE,CAAC;IAC3C,MAAM,UAAU,GAAG,sBAAsB,EAAE,CAAC;IAC5C,MAAM,UAAU,GAAG,wBAAwB,CAAC;IAE5C,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAC9C,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAE9C,MAAM,eAAe,GAA8D,EAAE,CAAC;IAEtF,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,aAAa,EAAE,CAAC;QAC7C,MAAM,SAAS,GAAG,GAAG,uBAAuB,IAAI,UAAU,KAAK,CAAC;QAChE,eAAe,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7E,MAAM,EAAE,IAAI,CAAC,0CAA0C,SAAS,EAAE,CAAC,CAAC;IACtE,CAAC;SAAM,IAAI,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAClC,MAAM,EAAE,IAAI,CAAC,gCAAgC,UAAU,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,aAAa,EAAE,CAAC;QAC7C,MAAM,SAAS,GAAG,GAAG,uBAAuB,IAAI,UAAU,KAAK,CAAC;QAChE,eAAe,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC;QAC9E,MAAM,EAAE,IAAI,CAAC,0CAA0C,SAAS,EAAE,CAAC,CAAC;IACtE,CAAC;SAAM,IAAI,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAClC,MAAM,EAAE,IAAI,CAAC,gCAAgC,UAAU,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,eAAe,EAAE,CAAC;QACnC,IAAI,CAAC;YACH,MAAM,EAAE,IAAI,CAAC,2BAA2B,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC;YACvD,MAAM,yBAAyB,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YACtE,MAAM,EAAE,IAAI,CAAC,6BAA6B,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QACzD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,wEAAwE;YACxE,MAAM,QAAQ,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACxE,MAAM,EAAE,KAAK,CAAC,kCAAkC,IAAI,CAAC,GAAG,KAAK,QAAQ,EAAE,CAAC,CAAC;YACzE,yDAAyD;YACzD,IAAI,KAAK,YAAY,KAAK,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC1D,MAAM,KAAK,CAAC,CAAC,oCAAoC;YACnD,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,gCAAgC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,SAAS,IAAI,CAAC,GAAG,KAAK,QAAQ,EAAE,CAAC,CAAC;QACzH,CAAC;IACH,CAAC;IAED,OAAO;QACL,UAAU,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI;QACtD,UAAU,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI;KACvD,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,uBAAuB,CAAC,YAAoB,IAAI;IACvD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,MAAM,MAAM,GAAG,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,UAAU,GAA8B,EAAE,CAAC;QACjD,IAAI,OAAuB,CAAC;QAE5B,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,EAAE;YAC3B,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,CAA4B,CAAC;gBACtE,2BAA2B;gBAC3B,IAAI,OAAO,CAAC,OAAO,KAAK,iBAAiB,IAAI,OAAO,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;oBACxE,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAC3B,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,sBAAsB;YACxB,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YACtB,YAAY,CAAC,OAAO,CAAC,CAAC;YACtB,MAAM,CAAC,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,CAAC;QAChB,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE;YACf,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAE1B,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;gBACxB,MAAM,CAAC,KAAK,EAAE,CAAC;gBACf,wCAAwC;gBACxC,OAAO,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACxD,CAAC,EAAE,SAAS,CAAC,CAAC;QAChB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,8BAA8B;IAM5C,MAAM,WAAW,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;IAC3D,MAAM,WAAW,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;IAC3D,MAAM,WAAW,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;IAE3D,IAAI,WAAW,IAAI,WAAW,IAAI,WAAW,EAAE,CAAC;QAC9C,OAAO;YACL,WAAW,EAAE,IAAI;YACjB,WAAW,EAAE,IAAI;YACjB,WAAW,EAAE,IAAI;YACjB,MAAM,EAAE,aAAa;SACtB,CAAC;IACJ,CAAC;IAED,OAAO;QACL,WAAW;QACX,WAAW;QACX,WAAW;QACX,MAAM,EAAE,SAAS;KAClB,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB;IACtC,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,MAAM,uBAAuB,CAAC,IAAI,CAAC,CAAC;QACvD,IAAI,UAAU,EAAE,CAAC;YACf,OAAO;gBACL,SAAS,EAAE,IAAI;gBACf,OAAO,EAAE,UAAU,CAAC,MAAM,KAAK,OAAO;gBACtC,GAAG,EAAE,UAAU,CAAC,GAAG;gBACnB,YAAY,EAAE,UAAU,CAAC,YAAY;gBACrC,OAAO,EAAE,UAAU,CAAC,OAAO;aAC5B,CAAC;QACJ,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,mDAAmD;IACrD,CAAC;IAED,OAAO;QACL,SAAS,EAAE,KAAK;QAChB,OAAO,EAAE,KAAK;QACd,GAAG,EAAE,CAAC;KACP,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,SAAS,sCAAsC;IAC7C,MAAM,QAAQ,GAA2B;QACvC,iCAAiC,EAAE,IAAI;QACvC,oCAAoC,EAAE,MAAM;QAC5C,gCAAgC,EAAE,IAAI;QACtC,mCAAmC,EAAE,MAAM;QAC3C,sCAAsC,EAAE,MAAM;QAC9C,+CAA+C,EAAE,IAAI;QACrD,2CAA2C,EAAE,KAAK;QAClD,+CAA+C,EAAE,MAAM;KACxD,CAAC;IAEF,qDAAqD;IACrD,KAAK,MAAM,CAAC,GAAG,EAAE,YAAY,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC3D,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,YAAY,CAAC;QAClC,CAAC;IACH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,sBAAsB,CAAC,QAAgB,EAAE,QAAgB,EAAE,QAAgB;IACzF,wEAAwE;IACxE,6FAA6F;IAC7F,kHAAkH;IAClH,OAAO,CAAC,GAAG,CAAC,wBAAwB,GAAG,QAAQ,CAAC;IAChD,OAAO,CAAC,GAAG,CAAC,wBAAwB,GAAG,QAAQ,CAAC;IAChD,OAAO,CAAC,GAAG,CAAC,wBAAwB,GAAG,QAAQ,CAAC;IAEhD,gDAAgD;IAChD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,EAAE,CAAC;QAC5H,MAAM,IAAI,KAAK,CAAC,2FAA2F,CAAC,CAAC;IAC/G,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAAc,EACd,YAAqB,EACrB,GAAuB;IAEvB,4BAA4B;IAC5B,yBAAyB,GAAG,IAAI,CAAC;IACjC,wBAAwB,GAAG,IAAI,CAAC;IAEhC,8EAA8E;IAC9E,+EAA+E;IAC/E,+EAA+E;IAC/E,iHAAiH;IACjH,IAAI,kBAAkB,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;IAC9D,IAAI,kBAAkB,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;IAC9D,IAAI,kBAAkB,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;IAE9D,IAAI,CAAC,CAAC,kBAAkB,IAAI,CAAC,kBAAkB,IAAI,CAAC,kBAAkB,CAAC,IAAI,GAAG,EAAE,CAAC;QAC/E,IAAI,CAAC;YACH,MAAM,CAAC,IAAI,CAAC,kEAAkE,CAAC,CAAC;YAChF,MAAM,WAAW,GAAG,MAAM,0BAA0B,CAAC,GAAG,CAAC,CAAC;YAC1D,kBAAkB,GAAG,WAAW,CAAC,QAAQ,CAAC;YAC1C,kBAAkB,GAAG,WAAW,CAAC,QAAQ,CAAC;YAC1C,kBAAkB,GAAG,WAAW,CAAC,QAAQ,CAAC;YAE1C,qHAAqH;YACrH,kHAAkH;YAClH,0GAA0G;YAC1G,sBAAsB,CAAC,kBAAkB,EAAE,kBAAkB,EAAE,kBAAkB,CAAC,CAAC;YAEnF,wEAAwE;YACxE,MAAM,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;YAC5D,MAAM,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;YAC5D,MAAM,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;YAE5D,IAAI,CAAC,cAAc,IAAI,CAAC,cAAc,IAAI,CAAC,cAAc,EAAE,CAAC;gBAC1D,MAAM,CAAC,KAAK,CAAC,wGAAwG,CAAC,CAAC;gBACvH,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;YAC7E,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,iFAAiF,CAAC,CAAC;QACjG,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,QAAQ,GAAG,gEAAgE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YAC1I,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,iEAAiE;IACjE,0EAA0E;IAC1E,IAAI,CAAC,kBAAkB,IAAI,CAAC,kBAAkB,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACtE,MAAM,QAAQ,GAAG,0LAA0L,CAAC;QAC5M,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC5B,CAAC;IAED,kFAAkF;IAClF,MAAM,CAAC,IAAI,CAAC,sDAAsD,CAAC,CAAC;IACpE,MAAM,aAAa,GAAG,MAAM,gBAAgB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAE3D,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,CAAC;QAC9B,MAAM,CAAC,KAAK,CAAC,kDAAkD,CAAC,CAAC;QACjE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;IACpE,CAAC;IAED,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,CAAC;QAC9B,MAAM,CAAC,KAAK,CAAC,kDAAkD,CAAC,CAAC;QACjE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;IACpE,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,6BAA6B,aAAa,CAAC,UAAU,EAAE,CAAC,CAAC;IACrE,MAAM,CAAC,IAAI,CAAC,6BAA6B,aAAa,CAAC,UAAU,EAAE,CAAC,CAAC;IAErE,2EAA2E;IAC3E,6FAA6F;IAC7F,uEAAuE;IACvE,sCAAsC,EAAE,CAAC;IAEzC,sHAAsH;IACtH,iDAAiD;IACjD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,6BAA6B,EAAE,CAAC;QAC/C,OAAO,CAAC,GAAG,CAAC,6BAA6B,GAAG,MAAM,CAAC,cAAc,CAAC,CAAC;QACnE,MAAM,CAAC,IAAI,CAAC,iDAAiD,cAAc,YAAY,CAAC,CAAC;IAC3F,CAAC;IAED,0BAA0B;IAC1B,+EAA+E;IAC/E,IAAI,aAAa,GAAG,YAAY,CAAC;IACjC,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,sCAAsC;QACtC,6DAA6D;QAC7D,aAAa,GAAG,cAAc,CAAC;IACjC,CAAC;IACD,aAAa,GAAG,aAAa,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;IAEnF,0CAA0C;IAC1C,6EAA6E;IAC7E,sEAAsE;IACtE,8FAA8F;IAC9F,2EAA2E;IAC3E,MAAM,YAAY,GAAsB;QACtC,GAAG,OAAO,CAAC,GAAG,EAAE,yEAAyE;QACzF,uBAAuB,EAAE,aAAa;QACtC,6BAA6B,EAAE,MAAM,CAAC,cAAc,CAAC;KACtD,CAAC;IAEF,0BAA0B;IAC1B,MAAM,gBAAgB,GAAG,KAAK,CAAC,aAAa,CAAC,UAAU,EAAE,CAAC,UAAU,EAAE,aAAa,CAAC,UAAU,CAAC,EAAE;QAC/F,GAAG,EAAE,YAAY;QACjB,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;QACjC,QAAQ,EAAE,KAAK;KAChB,CAAC,CAAC;IAEH,gDAAgD;IAChD,sBAAsB,GAAG,gBAAgB,CAAC;IAC1C,yBAAyB,GAAG,IAAI,CAAC;IACjC,uBAAuB,GAAG,IAAI,CAAC;IAC/B,qBAAqB,GAAG,EAAE,CAAC;IAE3B,gBAAgB,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;QAC3C,MAAM,CAAC,KAAK,CAAC,eAAe,IAAI,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IACxD,CAAC,CAAC,CAAC;IAEH,gBAAgB,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;QAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC;QACzC,qBAAqB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtC,mDAAmD;QACnD,IAAI,qBAAqB,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YACtC,qBAAqB,CAAC,KAAK,EAAE,CAAC;QAChC,CAAC;QACD,+DAA+D;QAC/D,MAAM,CAAC,IAAI,CAAC,mBAAmB,SAAS,EAAE,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;IAEH,gBAAgB,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;QACnC,yBAAyB,GAAG,GAAG,CAAC,OAAO,CAAC;QACxC,MAAM,CAAC,KAAK,CAAC,gCAAgC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;IAEH,gBAAgB,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;QACnC,uBAAuB,GAAG,IAAI,CAAC;QAC/B,sBAAsB,GAAG,IAAI,CAAC,CAAC,qCAAqC;QACpE,MAAM,CAAC,IAAI,CAAC,wCAAwC,IAAI,EAAE,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,IAAI,CAAC,gCAAgC,gBAAgB,CAAC,GAAG,eAAe,aAAa,EAAE,CAAC,CAAC;IAEhG,OAAO;QACL,OAAO,EAAE,gBAAgB;QACzB,QAAQ,EAAE,UAAU,aAAa,EAAE;KACpC,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,yBAAyB;IAQvC,OAAO;QACL,UAAU,EAAE,sBAAsB,KAAK,IAAI;QAC3C,GAAG,EAAE,sBAAsB,EAAE,GAAG,IAAI,IAAI;QACxC,QAAQ,EAAE,uBAAuB;QACjC,UAAU,EAAE,yBAAyB;QACrC,SAAS,EAAE,wBAAwB;QACnC,MAAM,EAAE,CAAC,GAAG,qBAAqB,CAAC,EAAE,gBAAgB;KACrD,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,MAAc,EACd,mBAA4B,EAC5B,GAAuB;IAEvB,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,+CAA+C;IAC/C,MAAM,MAAM,GAAG,MAAM,kBAAkB,EAAE,CAAC;IAC1C,IAAI,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;QAC9D,MAAM,CAAC,IAAI,CAAC,0CAA0C,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC;QAC7E,OAAO,UAAU,MAAM,CAAC,YAAY,EAAE,CAAC;IACzC,CAAC;IAED,mCAAmC;IACnC,IAAI,CAAC;QACH,gCAAgC;QAChC,wBAAwB,GAAG,IAAI,CAAC;QAEhC,MAAM,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;QAC/D,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC;QAElE,qDAAqD;QACrD,qBAAqB;QACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,kBAAkB,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC,CAAC;YACnE,MAAM,SAAS,GAAG,MAAM,kBAAkB,EAAE,CAAC;YAC7C,IAAI,SAAS,CAAC,SAAS,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;gBAC7C,MAAM,CAAC,IAAI,CAAC,qCAAqC,SAAS,CAAC,YAAY,IAAI,QAAQ,EAAE,CAAC,CAAC;gBACvF,OAAO,SAAS,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,SAAS,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC;YAChF,CAAC;QACH,CAAC;QAED,4EAA4E;QAC5E,MAAM,CAAC,IAAI,CAAC,4EAA4E,CAAC,CAAC;QAC1F,OAAO,QAAQ,CAAC;IAClB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,QAAQ,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAClE,wBAAwB,GAAG,QAAQ,CAAC;QACpC,MAAM,CAAC,KAAK,CAAC,0CAA0C,QAAQ,EAAE,CAAC,CAAC;QACnE,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC","sourcesContent":["import { spawn, ChildProcess } from 'child_process';\r\nimport { createWriteStream, existsSync, chmodSync, mkdirSync } from 'fs';\r\nimport { join, dirname } from 'path';\r\nimport { pipeline } from 'stream/promises';\r\nimport { createGunzip } from 'zlib';\r\nimport { createHash } from 'node:crypto';\r\nimport type { Logger } from 'pino';\r\nimport dgram from 'dgram';\r\nimport type { EnvironmentConfig } from './types.js';\r\nimport { hostToHttpUrl } from './utils/urls.js';\r\n\r\n// Protocol is httpprotobuf \r\n\r\ninterface CollectorDiscoveryEntry {\r\n version: string;\r\n status: string;\r\n pid: number;\r\n otlpEndpoint: string;\r\n}\r\n\r\nexport interface CollectorStatus {\r\n isRunning: boolean;\r\n isReady: boolean;\r\n pid: number;\r\n otlpEndpoint?: string;\r\n version?: string;\r\n}\r\n\r\nconst COLLECTOR_VERSION = '1.0.1'; // Match C# collector version (from collector-version.json)\r\nconst COLLECTOR_DOWNLOAD_BASE = `https://collectors.beamable.com/version/${COLLECTOR_VERSION}`;\r\nconst DISCOVERY_PORT = parseInt(process.env.BEAM_COLLECTOR_DISCOVERY_PORT || '8688', 10);\r\n\r\n// Global storage for collector process and errors (to prevent GC and track status)\r\nlet globalCollectorProcess: ChildProcess | null = null;\r\nlet globalCollectorStartError: string | null = null;\r\nlet globalCollectorExitCode: number | null = null;\r\nlet globalCollectorStderr: string[] = [];\r\nlet globalCollectorInitError: string | null = null; // Tracks errors from discoverOrStartCollector\r\nconst DISCOVERY_DELAY = 100; // milliseconds\r\nconst DISCOVERY_ATTEMPTS = 10;\r\n\r\n/**\r\n * ClickHouse credentials response from Beamable API\r\n */\r\ninterface ClickHouseCredentials {\r\n endpoint: string;\r\n expiresAt?: string;\r\n password: string;\r\n username: string;\r\n}\r\n\r\n/**\r\n * Calculates Beamable signature for signed requests\r\n * Signature format: MD5(secret + pid + version + uriPathAndQuery + body) as Base64\r\n * Matches C# SignedRequesterHelper.CalculateSignature\r\n */\r\nfunction calculateSignature(\r\n pid: string,\r\n secret: string,\r\n uriPathAndQuery: string,\r\n body: string | null = null,\r\n version: string = '1'\r\n): string {\r\n let dataToSign = `${secret}${pid}${version}${uriPathAndQuery}`;\r\n if (body) {\r\n dataToSign += body;\r\n }\r\n \r\n // MD5 hash to Base64 (matching C# implementation)\r\n const hash = createHash('md5').update(dataToSign, 'utf8').digest('base64');\r\n return hash;\r\n}\r\n\r\n/**\r\n * Extracts path and query from a URL\r\n */\r\nfunction getPathAndQuery(url: string): string {\r\n try {\r\n const urlObj = new URL(url);\r\n return urlObj.pathname + urlObj.search;\r\n } catch {\r\n // If URL parsing fails, return as-is\r\n return url;\r\n }\r\n}\r\n\r\n/**\r\n * Fetches ClickHouse credentials from Beamable API\r\n * GET /api/beamo/otel/auth/writer/config\r\n * Requires signed request authentication (X-BEAM-SIGNATURE header)\r\n */\r\nexport async function fetchClickHouseCredentials(\r\n env: EnvironmentConfig\r\n): Promise<ClickHouseCredentials> {\r\n const apiUrl = hostToHttpUrl(env.host);\r\n const uriPath = '/api/beamo/otel/auth/writer/config';\r\n const configUrl = new URL(uriPath, apiUrl).toString();\r\n \r\n // Get secret from environment (required for signed requests)\r\n const secret = process.env.SECRET;\r\n if (!secret) {\r\n throw new Error('SECRET environment variable is required to fetch ClickHouse credentials');\r\n }\r\n \r\n // Calculate signature for signed request (matching C# HttpSignedRequester)\r\n const pathAndQuery = getPathAndQuery(uriPath);\r\n const signature = calculateSignature(env.pid, secret, pathAndQuery, null, '1');\r\n \r\n // Build headers with signed authentication (matching C# HttpSignedRequester)\r\n const headers: Record<string, string> = {\r\n 'Content-Type': 'application/json',\r\n Accept: 'application/json',\r\n 'X-BEAM-SCOPE': `${env.cid}.${env.pid}`,\r\n 'X-BEAM-SIGNATURE': signature,\r\n };\r\n \r\n const response = await fetch(configUrl, {\r\n method: 'GET',\r\n headers,\r\n });\r\n \r\n if (!response.ok) {\r\n const errorText = await response.text().catch(() => 'Unknown error');\r\n throw new Error(`Failed to fetch ClickHouse credentials from ${configUrl}: ${response.status} ${response.statusText} - ${errorText.substring(0, 200)}`);\r\n }\r\n \r\n const credentials = await response.json() as ClickHouseCredentials;\r\n \r\n if (!credentials.endpoint || !credentials.username || !credentials.password) {\r\n throw new Error('Invalid ClickHouse credentials response: missing required fields');\r\n }\r\n \r\n return credentials;\r\n}\r\n\r\n/**\r\n * Gets the collector storage directory (similar to C# LocalApplicationData/beam/collectors/version)\r\n */\r\nfunction getCollectorStoragePath(): string {\r\n // Use temp directory for now - in containers this should be writable\r\n const tempDir = process.env.TMPDIR || process.env.TMP || '/tmp';\r\n return join(tempDir, 'beam', 'collectors', COLLECTOR_VERSION);\r\n}\r\n\r\n/**\r\n * Gets the collector binary name for the current platform\r\n */\r\nfunction getCollectorBinaryName(): string {\r\n const platform = process.platform;\r\n const arch = process.arch;\r\n \r\n // Match C# naming: collector-{osArchSuffix}\r\n // C# returns \"collector-linux-amd64\", not \"beamable-collector-linux-amd64\"\r\n if (platform === 'linux' && arch === 'x64') {\r\n return 'collector-linux-amd64';\r\n } else if (platform === 'linux' && arch === 'arm64') {\r\n return 'collector-linux-arm64';\r\n } else if (platform === 'darwin' && arch === 'x64') {\r\n return 'collector-darwin-amd64';\r\n } else if (platform === 'darwin' && arch === 'arm64') {\r\n return 'collector-darwin-arm64';\r\n } else if (platform === 'win32' && arch === 'x64') {\r\n return 'collector-windows-amd64.exe';\r\n }\r\n \r\n throw new Error(`Unsupported platform: ${platform} ${arch}`);\r\n}\r\n\r\n/**\r\n * Downloads and decompresses a gzipped file\r\n */\r\nasync function downloadAndDecompressGzip(url: string, outputPath: string, makeExecutable: boolean = false): Promise<void> {\r\n let response: Response;\r\n try {\r\n // Add timeout and better error handling\r\n const controller = new AbortController();\r\n const timeoutId = setTimeout(() => controller.abort(), 30000); // 30 second timeout\r\n \r\n try {\r\n response = await fetch(url, {\r\n signal: controller.signal,\r\n headers: {\r\n 'User-Agent': 'Beamable-Node-Microservice-Runtime/1.0',\r\n },\r\n });\r\n clearTimeout(timeoutId);\r\n } catch (fetchError) {\r\n clearTimeout(timeoutId);\r\n if (fetchError instanceof Error) {\r\n // Extract underlying error details from cause\r\n let underlyingError = '';\r\n if ((fetchError as any).cause) {\r\n const cause = (fetchError as any).cause;\r\n if (cause instanceof Error) {\r\n const causeCode = (cause as any).code;\r\n underlyingError = ` (cause: ${cause.name} - ${cause.message}${causeCode ? ` [code: ${causeCode}]` : ''})`;\r\n } else if (typeof cause === 'object' && cause.code) {\r\n underlyingError = ` (cause code: ${cause.code})`;\r\n }\r\n }\r\n \r\n // Provide more context about the error\r\n if (fetchError.name === 'AbortError') {\r\n throw new Error(`Timeout downloading ${url} (30s limit exceeded)${underlyingError}`);\r\n }\r\n \r\n // Check error code if available (Node.js system errors)\r\n const errorCode = (fetchError as any).code || ((fetchError as any).cause?.code);\r\n if (errorCode) {\r\n if (errorCode === 'ENOTFOUND' || errorCode === 'EAI_AGAIN') {\r\n throw new Error(`DNS resolution failed for ${url} (code: ${errorCode}). Check network connectivity and DNS settings.`);\r\n }\r\n if (errorCode === 'ECONNREFUSED') {\r\n throw new Error(`Connection refused to ${url} (code: ${errorCode}). Server may be down or firewall blocking.`);\r\n }\r\n if (errorCode === 'ETIMEDOUT' || errorCode === 'ECONNRESET') {\r\n throw new Error(`Connection timeout/reset to ${url} (code: ${errorCode}). Network may be slow or unreachable.`);\r\n }\r\n if (errorCode === 'CERT_HAS_EXPIRED' || errorCode === 'UNABLE_TO_VERIFY_LEAF_SIGNATURE') {\r\n throw new Error(`SSL/TLS certificate error connecting to ${url} (code: ${errorCode}): ${fetchError.message}`);\r\n }\r\n }\r\n \r\n // Check for common network errors in message\r\n const errorMsg = fetchError.message.toLowerCase();\r\n if (errorMsg.includes('enotfound') || errorMsg.includes('dns')) {\r\n throw new Error(`DNS resolution failed for ${url}. Check network connectivity and DNS settings.${underlyingError}`);\r\n }\r\n if (errorMsg.includes('econnrefused') || errorMsg.includes('connection refused')) {\r\n throw new Error(`Connection refused to ${url}. Server may be down or firewall blocking.${underlyingError}`);\r\n }\r\n if (errorMsg.includes('etimedout') || errorMsg.includes('timeout')) {\r\n throw new Error(`Connection timeout to ${url}. Network may be slow or unreachable.${underlyingError}`);\r\n }\r\n if (errorMsg.includes('certificate') || errorMsg.includes('ssl') || errorMsg.includes('tls')) {\r\n throw new Error(`SSL/TLS error connecting to ${url}: ${fetchError.message}${underlyingError}`);\r\n }\r\n throw new Error(`Network error downloading ${url}: ${fetchError.name} - ${fetchError.message}${underlyingError}`);\r\n }\r\n throw new Error(`Network error downloading ${url}: ${String(fetchError)}`);\r\n }\r\n } catch (error) {\r\n // Re-throw with more context\r\n if (error instanceof Error) {\r\n if (error.message.includes('Timeout') || error.message.includes('DNS') || error.message.includes('Connection')) {\r\n throw error; // Already has good context\r\n }\r\n throw new Error(`Network error downloading ${url}: ${error.message}`);\r\n }\r\n throw new Error(`Network error downloading ${url}: ${String(error)}`);\r\n }\r\n \r\n if (!response.ok) {\r\n const errorText = await response.text().catch(() => '');\r\n throw new Error(`HTTP ${response.status} downloading ${url}: ${response.statusText}${errorText ? ` - ${errorText.substring(0, 200)}` : ''}`);\r\n }\r\n \r\n const dir = dirname(outputPath);\r\n if (!existsSync(dir)) {\r\n mkdirSync(dir, { recursive: true });\r\n }\r\n \r\n const gunzip = createGunzip();\r\n const writeStream = createWriteStream(outputPath);\r\n \r\n await pipeline(response.body as any, gunzip, writeStream);\r\n \r\n if (makeExecutable && process.platform !== 'win32') {\r\n try {\r\n chmodSync(outputPath, 0o755);\r\n } catch (error) {\r\n console.error(`Failed to make ${outputPath} executable:`, error);\r\n }\r\n }\r\n}\r\n\r\n/**\r\n * Resolves the collector binary and config, downloading if needed\r\n */\r\nasync function resolveCollector(allowDownload: boolean = true, logger?: Logger): Promise<{ binaryPath: string | null; configPath: string | null }> {\r\n const basePath = getCollectorStoragePath();\r\n const binaryName = getCollectorBinaryName();\r\n const configName = 'clickhouse-config.yaml';\r\n \r\n const binaryPath = join(basePath, binaryName);\r\n const configPath = join(basePath, configName);\r\n \r\n const itemsToDownload: Array<{ url: string; path: string; executable: boolean }> = [];\r\n \r\n if (!existsSync(binaryPath) && allowDownload) {\r\n const binaryUrl = `${COLLECTOR_DOWNLOAD_BASE}/${binaryName}.gz`;\r\n itemsToDownload.push({ url: binaryUrl, path: binaryPath, executable: true });\r\n logger?.info(`[Collector] Will download binary from: ${binaryUrl}`);\r\n } else if (existsSync(binaryPath)) {\r\n logger?.info(`[Collector] Binary found at: ${binaryPath}`);\r\n }\r\n \r\n if (!existsSync(configPath) && allowDownload) {\r\n const configUrl = `${COLLECTOR_DOWNLOAD_BASE}/${configName}.gz`;\r\n itemsToDownload.push({ url: configUrl, path: configPath, executable: false });\r\n logger?.info(`[Collector] Will download config from: ${configUrl}`);\r\n } else if (existsSync(configPath)) {\r\n logger?.info(`[Collector] Config found at: ${configPath}`);\r\n }\r\n \r\n for (const item of itemsToDownload) {\r\n try {\r\n logger?.info(`[Collector] Downloading ${item.url}...`);\r\n await downloadAndDecompressGzip(item.url, item.path, item.executable);\r\n logger?.info(`[Collector] Downloaded to ${item.path}`);\r\n } catch (error) {\r\n // Don't wrap the error again if it already contains the URL and details\r\n const errorMsg = error instanceof Error ? error.message : String(error);\r\n logger?.error(`[Collector] Failed to download ${item.url}: ${errorMsg}`);\r\n // Only wrap if the error doesn't already contain the URL\r\n if (error instanceof Error && errorMsg.includes(item.url)) {\r\n throw error; // Error already has URL and context\r\n }\r\n throw new Error(`Failed to download collector ${item.executable ? 'binary' : 'config'} from ${item.url}: ${errorMsg}`);\r\n }\r\n }\r\n \r\n return {\r\n binaryPath: existsSync(binaryPath) ? binaryPath : null,\r\n configPath: existsSync(configPath) ? configPath : null,\r\n };\r\n}\r\n\r\n/**\r\n * Discovers collector via UDP broadcast\r\n */\r\nfunction discoverCollectorViaUDP(timeoutMs: number = 5000): Promise<CollectorDiscoveryEntry | null> {\r\n return new Promise((resolve) => {\r\n const socket = dgram.createSocket('udp4');\r\n const discovered: CollectorDiscoveryEntry[] = [];\r\n let timeout: NodeJS.Timeout;\r\n \r\n socket.on('message', (msg) => {\r\n try {\r\n const message = JSON.parse(msg.toString()) as CollectorDiscoveryEntry;\r\n // Check if version matches\r\n if (message.version === COLLECTOR_VERSION && message.status === 'READY') {\r\n discovered.push(message);\r\n }\r\n } catch (error) {\r\n // Ignore parse errors\r\n }\r\n });\r\n \r\n socket.on('error', () => {\r\n clearTimeout(timeout);\r\n socket.close();\r\n resolve(null);\r\n });\r\n \r\n socket.bind(() => {\r\n socket.setBroadcast(true);\r\n \r\n timeout = setTimeout(() => {\r\n socket.close();\r\n // Return the first discovered collector\r\n resolve(discovered.length > 0 ? discovered[0] : null);\r\n }, timeoutMs);\r\n });\r\n });\r\n}\r\n\r\n/**\r\n * Gets the current status of ClickHouse credentials (from env or API)\r\n */\r\nexport function getClickHouseCredentialsStatus(): {\r\n hasEndpoint: boolean;\r\n hasUsername: boolean;\r\n hasPassword: boolean;\r\n source: 'environment' | 'api' | 'missing';\r\n} {\r\n const hasEndpoint = !!process.env.BEAM_CLICKHOUSE_ENDPOINT;\r\n const hasUsername = !!process.env.BEAM_CLICKHOUSE_USERNAME;\r\n const hasPassword = !!process.env.BEAM_CLICKHOUSE_PASSWORD;\r\n \r\n if (hasEndpoint && hasUsername && hasPassword) {\r\n return {\r\n hasEndpoint: true,\r\n hasUsername: true,\r\n hasPassword: true,\r\n source: 'environment',\r\n };\r\n }\r\n \r\n return {\r\n hasEndpoint,\r\n hasUsername,\r\n hasPassword,\r\n source: 'missing',\r\n };\r\n}\r\n\r\n/**\r\n * Checks if collector is already running via UDP discovery\r\n */\r\nexport async function isCollectorRunning(): Promise<CollectorStatus> {\r\n try {\r\n const discovered = await discoverCollectorViaUDP(2000);\r\n if (discovered) {\r\n return {\r\n isRunning: true,\r\n isReady: discovered.status === 'READY',\r\n pid: discovered.pid,\r\n otlpEndpoint: discovered.otlpEndpoint,\r\n version: discovered.version,\r\n };\r\n }\r\n } catch (error) {\r\n // Discovery failed, collector probably not running\r\n }\r\n \r\n return {\r\n isRunning: false,\r\n isReady: false,\r\n pid: 0,\r\n };\r\n}\r\n\r\n/**\r\n * Sets default collector configuration environment variables in process.env\r\n * Matches C# CollectorManager.AddCollectorConfigurationToEnvironment()\r\n */\r\nfunction addCollectorConfigurationToEnvironment(): void {\r\n const defaults: Record<string, string> = {\r\n BEAM_CLICKHOUSE_PROCESSOR_TIMEOUT: '5s',\r\n BEAM_CLICKHOUSE_PROCESSOR_BATCH_SIZE: '5000',\r\n BEAM_CLICKHOUSE_EXPORTER_TIMEOUT: '5s',\r\n BEAM_CLICKHOUSE_EXPORTER_QUEUE_SIZE: '1000',\r\n BEAM_CLICKHOUSE_EXPORTER_RETRY_ENABLED: 'true',\r\n BEAM_CLICKHOUSE_EXPORTER_RETRY_INITIAL_INTERVAL: '5s',\r\n BEAM_CLICKHOUSE_EXPORTER_RETRY_MAX_INTERVAL: '30s',\r\n BEAM_CLICKHOUSE_EXPORTER_RETRY_MAX_ELAPSED_TIME: '300s',\r\n };\r\n \r\n // Only set if not already set (matching C# behavior)\r\n for (const [key, defaultValue] of Object.entries(defaults)) {\r\n if (!process.env[key]) {\r\n process.env[key] = defaultValue;\r\n }\r\n }\r\n}\r\n\r\n/**\r\n * Sets ClickHouse credentials in process.env (matching C# CollectorManager.AddAuthEnvironmentVars)\r\n */\r\nexport function addAuthEnvironmentVars(endpoint: string, username: string, password: string): void {\r\n // Always set the values when explicitly provided (matching C# behavior)\r\n // This ensures credentials fetched from API override any existing (potentially empty) values\r\n // In Node.js, process.env is a writable object - setting properties here persists for the lifetime of the process\r\n process.env.BEAM_CLICKHOUSE_ENDPOINT = endpoint;\r\n process.env.BEAM_CLICKHOUSE_USERNAME = username;\r\n process.env.BEAM_CLICKHOUSE_PASSWORD = password;\r\n \r\n // Verify they were actually set (for debugging)\r\n if (!process.env.BEAM_CLICKHOUSE_ENDPOINT || !process.env.BEAM_CLICKHOUSE_USERNAME || !process.env.BEAM_CLICKHOUSE_PASSWORD) {\r\n throw new Error(`Failed to set ClickHouse credentials in process.env - this should never happen in Node.js`);\r\n }\r\n}\r\n\r\n/**\r\n * Starts the OpenTelemetry collector process\r\n */\r\nexport async function startCollector(\r\n logger: Logger,\r\n otlpEndpoint?: string,\r\n env?: EnvironmentConfig\r\n): Promise<{ process: ChildProcess; endpoint: string }> {\r\n // Clear any previous errors\r\n globalCollectorStartError = null;\r\n globalCollectorInitError = null;\r\n \r\n // CRITICAL: Fetch credentials FIRST, before resolving collector binary/config\r\n // This ensures credentials are in process.env even if collector download fails\r\n // The credentials will persist in process.env for the lifetime of the process.\r\n // Per Gabriel: \"these OTEL related ones you need to do it yourself at the beginning of the microservice startup\"\r\n let clickhouseEndpoint = process.env.BEAM_CLICKHOUSE_ENDPOINT;\r\n let clickhouseUsername = process.env.BEAM_CLICKHOUSE_USERNAME;\r\n let clickhousePassword = process.env.BEAM_CLICKHOUSE_PASSWORD;\r\n \r\n if ((!clickhouseEndpoint || !clickhouseUsername || !clickhousePassword) && env) {\r\n try {\r\n logger.info('[Collector] Fetching ClickHouse credentials from Beamable API...');\r\n const credentials = await fetchClickHouseCredentials(env);\r\n clickhouseEndpoint = credentials.endpoint;\r\n clickhouseUsername = credentials.username;\r\n clickhousePassword = credentials.password;\r\n \r\n // CRITICAL: Set them in process.env IMMEDIATELY after fetching (matching C# CollectorManager.AddAuthEnvironmentVars)\r\n // In Node.js, process.env is a writable object - setting properties here persists for the lifetime of the process\r\n // This ensures they're available to both the Node.js process and inherited by the collector child process\r\n addAuthEnvironmentVars(clickhouseEndpoint, clickhouseUsername, clickhousePassword);\r\n \r\n // Verify they were set (defensive check - should never fail in Node.js)\r\n const verifyEndpoint = process.env.BEAM_CLICKHOUSE_ENDPOINT;\r\n const verifyUsername = process.env.BEAM_CLICKHOUSE_USERNAME;\r\n const verifyPassword = process.env.BEAM_CLICKHOUSE_PASSWORD;\r\n \r\n if (!verifyEndpoint || !verifyUsername || !verifyPassword) {\r\n logger.error(`[Collector] CRITICAL: Credentials were set but are missing from process.env! This should never happen.`);\r\n throw new Error('Failed to persist ClickHouse credentials in process.env');\r\n }\r\n \r\n logger.info('[Collector] ClickHouse credentials fetched from API and verified in process.env');\r\n } catch (error) {\r\n const errorMsg = `[Collector] Failed to fetch ClickHouse credentials from API: ${error instanceof Error ? error.message : String(error)}`;\r\n logger.error(errorMsg);\r\n throw new Error(errorMsg);\r\n }\r\n }\r\n \r\n // Validate required environment variables (matching C# behavior)\r\n // These must be set before starting the collector, otherwise it will fail\r\n if (!clickhouseEndpoint || !clickhouseUsername || !clickhousePassword) {\r\n const errorMsg = `[Collector] Required ClickHouse credentials are missing. Set BEAM_CLICKHOUSE_ENDPOINT, BEAM_CLICKHOUSE_USERNAME, and BEAM_CLICKHOUSE_PASSWORD, or ensure the API endpoint is accessible.`;\r\n logger.error(errorMsg);\r\n throw new Error(errorMsg);\r\n }\r\n \r\n // Now resolve collector binary and config (after credentials are fetched and set)\r\n logger.info('[Collector] Resolving collector binary and config...');\r\n const collectorInfo = await resolveCollector(true, logger);\r\n \r\n if (!collectorInfo.binaryPath) {\r\n logger.error('[Collector] Binary not found and download failed');\r\n throw new Error('Collector binary not found and download failed');\r\n }\r\n \r\n if (!collectorInfo.configPath) {\r\n logger.error('[Collector] Config not found and download failed');\r\n throw new Error('Collector config not found and download failed');\r\n }\r\n \r\n logger.info(`[Collector] Using binary: ${collectorInfo.binaryPath}`);\r\n logger.info(`[Collector] Using config: ${collectorInfo.configPath}`);\r\n \r\n // Set default collector configuration environment variables in process.env\r\n // This ensures they're available to both the Node.js process and the collector child process\r\n // Matches C# CollectorManager.AddCollectorConfigurationToEnvironment()\r\n addCollectorConfigurationToEnvironment();\r\n \r\n // CRITICAL: Set BEAM_COLLECTOR_DISCOVERY_PORT if not already set (matching C# AddDefaultCollectorHostAndPortFallback)\r\n // This MUST be set before starting the collector\r\n if (!process.env.BEAM_COLLECTOR_DISCOVERY_PORT) {\r\n process.env.BEAM_COLLECTOR_DISCOVERY_PORT = String(DISCOVERY_PORT);\r\n logger.info(`[Collector] Set BEAM_COLLECTOR_DISCOVERY_PORT=${DISCOVERY_PORT} (default)`);\r\n }\r\n \r\n // Determine OTLP endpoint\r\n // Use a free port if not specified (like C# does with PortUtil.FreeEndpoint())\r\n let localEndpoint = otlpEndpoint;\r\n if (!localEndpoint) {\r\n // For now, use default OTLP HTTP port\r\n // In production, this would be discovered or set by Beamable\r\n localEndpoint = '0.0.0.0:4318';\r\n }\r\n localEndpoint = localEndpoint.replace(/^http:\\/\\//, '').replace(/^https:\\/\\//, '');\r\n \r\n // Set environment variables for collector\r\n // Note: BEAM_CLICKHOUSE_ENDPOINT is for collector → ClickHouse communication\r\n // This is different from the OTLP endpoint (microservice → collector)\r\n // The collector process will inherit all environment variables from process.env automatically\r\n // We only need to set the OTLP endpoint and discovery port explicitly here\r\n const collectorEnv: NodeJS.ProcessEnv = {\r\n ...process.env, // Inherit all process.env (including credentials and config we just set)\r\n BEAM_OTLP_HTTP_ENDPOINT: localEndpoint,\r\n BEAM_COLLECTOR_DISCOVERY_PORT: String(DISCOVERY_PORT),\r\n };\r\n \r\n // Start collector process\r\n const collectorProcess = spawn(collectorInfo.binaryPath, ['--config', collectorInfo.configPath], {\r\n env: collectorEnv,\r\n stdio: ['ignore', 'pipe', 'pipe'],\r\n detached: false,\r\n });\r\n \r\n // Store globally to prevent GC and track status\r\n globalCollectorProcess = collectorProcess;\r\n globalCollectorStartError = null;\r\n globalCollectorExitCode = null;\r\n globalCollectorStderr = [];\r\n \r\n collectorProcess.stdout?.on('data', (data) => {\r\n logger.debug(`[Collector] ${data.toString().trim()}`);\r\n });\r\n \r\n collectorProcess.stderr?.on('data', (data) => {\r\n const errorLine = data.toString().trim();\r\n globalCollectorStderr.push(errorLine);\r\n // Keep only last 50 lines to prevent memory issues\r\n if (globalCollectorStderr.length > 50) {\r\n globalCollectorStderr.shift();\r\n }\r\n // Log stderr at info level so we can see why collector crashes\r\n logger.info(`[Collector ERR] ${errorLine}`);\r\n });\r\n \r\n collectorProcess.on('error', (err) => {\r\n globalCollectorStartError = err.message;\r\n logger.error(`[Collector] Failed to start: ${err.message}`);\r\n });\r\n \r\n collectorProcess.on('exit', (code) => {\r\n globalCollectorExitCode = code;\r\n globalCollectorProcess = null; // Clear reference when process exits\r\n logger.warn(`[Collector] Process exited with code ${code}`);\r\n });\r\n \r\n logger.info(`[Collector] Started with PID ${collectorProcess.pid}, endpoint: ${localEndpoint}`);\r\n \r\n return {\r\n process: collectorProcess,\r\n endpoint: `http://${localEndpoint}`,\r\n };\r\n}\r\n\r\n/**\r\n * Gets detailed collector status including process info and errors\r\n */\r\nexport function getCollectorProcessStatus(): {\r\n hasProcess: boolean;\r\n pid: number | null;\r\n exitCode: number | null;\r\n startError: string | null;\r\n initError: string | null;\r\n stderr: string[];\r\n} {\r\n return {\r\n hasProcess: globalCollectorProcess !== null,\r\n pid: globalCollectorProcess?.pid ?? null,\r\n exitCode: globalCollectorExitCode,\r\n startError: globalCollectorStartError,\r\n initError: globalCollectorInitError,\r\n stderr: [...globalCollectorStderr], // Return a copy\r\n };\r\n}\r\n\r\n/**\r\n * Discovers or starts the collector and returns the OTLP endpoint\r\n */\r\nexport async function discoverOrStartCollector(\r\n logger: Logger,\r\n standardOtelEnabled: boolean,\r\n env?: EnvironmentConfig\r\n): Promise<string | null> {\r\n if (!standardOtelEnabled) {\r\n return null;\r\n }\r\n \r\n // First, check if collector is already running\r\n const status = await isCollectorRunning();\r\n if (status.isRunning && status.isReady && status.otlpEndpoint) {\r\n logger.info(`[Collector] Found running collector at ${status.otlpEndpoint}`);\r\n return `http://${status.otlpEndpoint}`;\r\n }\r\n \r\n // Collector not running - start it\r\n try {\r\n // Clear any previous init error\r\n globalCollectorInitError = null;\r\n \r\n logger.info('[Collector] Starting OpenTelemetry collector...');\r\n const { endpoint } = await startCollector(logger, undefined, env);\r\n \r\n // Wait a bit for collector to start and become ready\r\n // Try to discover it\r\n for (let i = 0; i < DISCOVERY_ATTEMPTS; i++) {\r\n await new Promise(resolve => setTimeout(resolve, DISCOVERY_DELAY));\r\n const newStatus = await isCollectorRunning();\r\n if (newStatus.isRunning && newStatus.isReady) {\r\n logger.info(`[Collector] Collector is ready at ${newStatus.otlpEndpoint || endpoint}`);\r\n return newStatus.otlpEndpoint ? `http://${newStatus.otlpEndpoint}` : endpoint;\r\n }\r\n }\r\n \r\n // Return the endpoint we started with, even if discovery didn't find it yet\r\n logger.warn('[Collector] Collector started but not yet ready, using configured endpoint');\r\n return endpoint;\r\n } catch (err) {\r\n const errorMsg = err instanceof Error ? err.message : String(err);\r\n globalCollectorInitError = errorMsg;\r\n logger.error(`[Collector] Failed to start collector: ${errorMsg}`);\r\n return null;\r\n }\r\n}\r\n\r\n"]}
|
package/package.json
CHANGED
package/src/collector-manager.ts
CHANGED
|
@@ -187,25 +187,55 @@ async function downloadAndDecompressGzip(url: string, outputPath: string, makeEx
|
|
|
187
187
|
} catch (fetchError) {
|
|
188
188
|
clearTimeout(timeoutId);
|
|
189
189
|
if (fetchError instanceof Error) {
|
|
190
|
+
// Extract underlying error details from cause
|
|
191
|
+
let underlyingError = '';
|
|
192
|
+
if ((fetchError as any).cause) {
|
|
193
|
+
const cause = (fetchError as any).cause;
|
|
194
|
+
if (cause instanceof Error) {
|
|
195
|
+
const causeCode = (cause as any).code;
|
|
196
|
+
underlyingError = ` (cause: ${cause.name} - ${cause.message}${causeCode ? ` [code: ${causeCode}]` : ''})`;
|
|
197
|
+
} else if (typeof cause === 'object' && cause.code) {
|
|
198
|
+
underlyingError = ` (cause code: ${cause.code})`;
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
|
|
190
202
|
// Provide more context about the error
|
|
191
203
|
if (fetchError.name === 'AbortError') {
|
|
192
|
-
throw new Error(`Timeout downloading ${url} (30s limit exceeded)`);
|
|
204
|
+
throw new Error(`Timeout downloading ${url} (30s limit exceeded)${underlyingError}`);
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
// Check error code if available (Node.js system errors)
|
|
208
|
+
const errorCode = (fetchError as any).code || ((fetchError as any).cause?.code);
|
|
209
|
+
if (errorCode) {
|
|
210
|
+
if (errorCode === 'ENOTFOUND' || errorCode === 'EAI_AGAIN') {
|
|
211
|
+
throw new Error(`DNS resolution failed for ${url} (code: ${errorCode}). Check network connectivity and DNS settings.`);
|
|
212
|
+
}
|
|
213
|
+
if (errorCode === 'ECONNREFUSED') {
|
|
214
|
+
throw new Error(`Connection refused to ${url} (code: ${errorCode}). Server may be down or firewall blocking.`);
|
|
215
|
+
}
|
|
216
|
+
if (errorCode === 'ETIMEDOUT' || errorCode === 'ECONNRESET') {
|
|
217
|
+
throw new Error(`Connection timeout/reset to ${url} (code: ${errorCode}). Network may be slow or unreachable.`);
|
|
218
|
+
}
|
|
219
|
+
if (errorCode === 'CERT_HAS_EXPIRED' || errorCode === 'UNABLE_TO_VERIFY_LEAF_SIGNATURE') {
|
|
220
|
+
throw new Error(`SSL/TLS certificate error connecting to ${url} (code: ${errorCode}): ${fetchError.message}`);
|
|
221
|
+
}
|
|
193
222
|
}
|
|
194
|
-
|
|
223
|
+
|
|
224
|
+
// Check for common network errors in message
|
|
195
225
|
const errorMsg = fetchError.message.toLowerCase();
|
|
196
226
|
if (errorMsg.includes('enotfound') || errorMsg.includes('dns')) {
|
|
197
|
-
throw new Error(`DNS resolution failed for ${url}. Check network connectivity and DNS settings
|
|
227
|
+
throw new Error(`DNS resolution failed for ${url}. Check network connectivity and DNS settings.${underlyingError}`);
|
|
198
228
|
}
|
|
199
229
|
if (errorMsg.includes('econnrefused') || errorMsg.includes('connection refused')) {
|
|
200
|
-
throw new Error(`Connection refused to ${url}. Server may be down or firewall blocking
|
|
230
|
+
throw new Error(`Connection refused to ${url}. Server may be down or firewall blocking.${underlyingError}`);
|
|
201
231
|
}
|
|
202
232
|
if (errorMsg.includes('etimedout') || errorMsg.includes('timeout')) {
|
|
203
|
-
throw new Error(`Connection timeout to ${url}. Network may be slow or unreachable
|
|
233
|
+
throw new Error(`Connection timeout to ${url}. Network may be slow or unreachable.${underlyingError}`);
|
|
204
234
|
}
|
|
205
235
|
if (errorMsg.includes('certificate') || errorMsg.includes('ssl') || errorMsg.includes('tls')) {
|
|
206
|
-
throw new Error(`SSL/TLS error connecting to ${url}: ${fetchError.message}`);
|
|
236
|
+
throw new Error(`SSL/TLS error connecting to ${url}: ${fetchError.message}${underlyingError}`);
|
|
207
237
|
}
|
|
208
|
-
throw new Error(`Network error downloading ${url}: ${fetchError.name} - ${fetchError.message}`);
|
|
238
|
+
throw new Error(`Network error downloading ${url}: ${fetchError.name} - ${fetchError.message}${underlyingError}`);
|
|
209
239
|
}
|
|
210
240
|
throw new Error(`Network error downloading ${url}: ${String(fetchError)}`);
|
|
211
241
|
}
|
|
@@ -279,8 +309,13 @@ async function resolveCollector(allowDownload: boolean = true, logger?: Logger):
|
|
|
279
309
|
await downloadAndDecompressGzip(item.url, item.path, item.executable);
|
|
280
310
|
logger?.info(`[Collector] Downloaded to ${item.path}`);
|
|
281
311
|
} catch (error) {
|
|
312
|
+
// Don't wrap the error again if it already contains the URL and details
|
|
282
313
|
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
283
314
|
logger?.error(`[Collector] Failed to download ${item.url}: ${errorMsg}`);
|
|
315
|
+
// Only wrap if the error doesn't already contain the URL
|
|
316
|
+
if (error instanceof Error && errorMsg.includes(item.url)) {
|
|
317
|
+
throw error; // Error already has URL and context
|
|
318
|
+
}
|
|
284
319
|
throw new Error(`Failed to download collector ${item.executable ? 'binary' : 'config'} from ${item.url}: ${errorMsg}`);
|
|
285
320
|
}
|
|
286
321
|
}
|
|
@@ -508,6 +543,13 @@ export async function startCollector(
|
|
|
508
543
|
// Matches C# CollectorManager.AddCollectorConfigurationToEnvironment()
|
|
509
544
|
addCollectorConfigurationToEnvironment();
|
|
510
545
|
|
|
546
|
+
// CRITICAL: Set BEAM_COLLECTOR_DISCOVERY_PORT if not already set (matching C# AddDefaultCollectorHostAndPortFallback)
|
|
547
|
+
// This MUST be set before starting the collector
|
|
548
|
+
if (!process.env.BEAM_COLLECTOR_DISCOVERY_PORT) {
|
|
549
|
+
process.env.BEAM_COLLECTOR_DISCOVERY_PORT = String(DISCOVERY_PORT);
|
|
550
|
+
logger.info(`[Collector] Set BEAM_COLLECTOR_DISCOVERY_PORT=${DISCOVERY_PORT} (default)`);
|
|
551
|
+
}
|
|
552
|
+
|
|
511
553
|
// Determine OTLP endpoint
|
|
512
554
|
// Use a free port if not specified (like C# does with PortUtil.FreeEndpoint())
|
|
513
555
|
let localEndpoint = otlpEndpoint;
|
|
@@ -553,7 +595,8 @@ export async function startCollector(
|
|
|
553
595
|
if (globalCollectorStderr.length > 50) {
|
|
554
596
|
globalCollectorStderr.shift();
|
|
555
597
|
}
|
|
556
|
-
|
|
598
|
+
// Log stderr at info level so we can see why collector crashes
|
|
599
|
+
logger.info(`[Collector ERR] ${errorLine}`);
|
|
557
600
|
});
|
|
558
601
|
|
|
559
602
|
collectorProcess.on('error', (err) => {
|
|
Binary file
|
|
Binary file
|