@adalo/metrics 0.0.0-staging.11 → 0.0.0-staging.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -0
- package/lib/health/healthCheckClient.js +1 -1
- package/lib/health/healthCheckClient.js.map +1 -1
- package/lib/health/healthCheckWorker.d.ts.map +1 -1
- package/lib/health/healthCheckWorker.js +15 -1
- package/lib/health/healthCheckWorker.js.map +1 -1
- package/lib/metrics/baseMetricsClient.d.ts +19 -1
- package/lib/metrics/baseMetricsClient.d.ts.map +1 -1
- package/lib/metrics/baseMetricsClient.js +55 -43
- package/lib/metrics/baseMetricsClient.js.map +1 -1
- package/lib/metrics/httpMetricsRedisStore.d.ts +39 -0
- package/lib/metrics/httpMetricsRedisStore.d.ts.map +1 -0
- package/lib/metrics/httpMetricsRedisStore.js +176 -0
- package/lib/metrics/httpMetricsRedisStore.js.map +1 -0
- package/lib/metrics/metricsClient.d.ts +39 -53
- package/lib/metrics/metricsClient.d.ts.map +1 -1
- package/lib/metrics/metricsClient.js +172 -88
- package/lib/metrics/metricsClient.js.map +1 -1
- package/lib/metrics/metricsDatabaseClient.d.ts +1 -1
- package/lib/metrics/metricsDatabaseClient.d.ts.map +1 -1
- package/lib/metrics/metricsDatabaseClient.js +9 -10
- package/lib/metrics/metricsDatabaseClient.js.map +1 -1
- package/lib/metrics/metricsQueueRedisClient.d.ts +1 -1
- package/lib/metrics/metricsQueueRedisClient.d.ts.map +1 -1
- package/lib/metrics/metricsQueueRedisClient.js +3 -3
- package/lib/metrics/metricsQueueRedisClient.js.map +1 -1
- package/lib/metrics/shouldRunMetricsPush.d.ts +20 -0
- package/lib/metrics/shouldRunMetricsPush.d.ts.map +1 -0
- package/lib/metrics/shouldRunMetricsPush.js +39 -0
- package/lib/metrics/shouldRunMetricsPush.js.map +1 -0
- package/package.json +1 -1
- package/src/health/healthCheckClient.js +1 -1
- package/src/health/healthCheckWorker.js +18 -1
- package/src/metrics/baseMetricsClient.js +64 -49
- package/src/metrics/httpMetricsRedisStore.js +190 -0
- package/src/metrics/metricsClient.js +236 -81
- package/src/metrics/metricsDatabaseClient.js +9 -10
- package/src/metrics/metricsQueueRedisClient.js +3 -3
- package/src/metrics/shouldRunMetricsPush.js +43 -0
package/README.md
CHANGED
|
@@ -14,6 +14,16 @@ Utility library for Prometheus metrics and HTTP request counting in Node.js appl
|
|
|
14
14
|
- In `package.json`: `"@adalo/metrics": "staging"` (resolves to latest `@adalo/metrics@staging`), or pin a specific version: `"@adalo/metrics": "0.0.0-staging.123"`.
|
|
15
15
|
- Deploy that app to staging and verify metrics/behavior. When satisfied, merge **metrics-js** into **main** and release a normal version; then update the app back to `"@adalo/metrics": "^x.y.z"` or `"latest"`.
|
|
16
16
|
|
|
17
|
+
## Optional peer: `redis` (multi-dyno HTTP metrics)
|
|
18
|
+
|
|
19
|
+
`@adalo/metrics` does **not** bundle the [`redis`](https://www.npmjs.com/package/redis) (node-redis v3) package. If you enable HTTP metrics with Redis aggregation (`METRICS_HTTP_ENABLED` and `REDIS_URL` / `METRICS_HTTP_REDIS_URL`), install it in **your** app:
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
yarn add redis@3
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
Without it, the client logs a warning and keeps per-process HTTP counters only. Queue/health code can keep using **ioredis** or node-redis v4 via injected clients; the optional peer is only for the HTTP metrics Redis store’s `createClient` API.
|
|
26
|
+
|
|
17
27
|
## Constructor
|
|
18
28
|
```ts
|
|
19
29
|
new MetricsClient({
|
|
@@ -31,7 +31,7 @@ const DEFAULT_HEALTH_CONFIG = {
|
|
|
31
31
|
checkIntervalMs: 30_000,
|
|
32
32
|
staleThresholdMs: 180_000,
|
|
33
33
|
checkTimeoutMs: 15_000,
|
|
34
|
-
maxDbConnectLatencyMs: readNumberEnv('HEALTH_DB_MAX_CONNECT_LATENCY_MS') ??
|
|
34
|
+
maxDbConnectLatencyMs: readNumberEnv('HEALTH_DB_MAX_CONNECT_LATENCY_MS') ?? 30_000
|
|
35
35
|
};
|
|
36
36
|
const SENSITIVE_PATTERNS = [{
|
|
37
37
|
pattern: /(postgres(?:ql)?|mysql|mongodb|redis|amqp):\/\/([^:]+):([^@]+)@([^:/]+)(:\d+)?\/([^\s?]+)/gi,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"healthCheckClient.js","names":["createDatabasePool","runHealthCheck","closePool","require","getRedisClientType","REDIS_V4","IOREDIS","REDIS_V3","HealthCheckCache","readNumberEnv","name","raw","process","env","undefined","num","Number","isFinite","DEFAULT_HEALTH_CONFIG","checkIntervalMs","staleThresholdMs","checkTimeoutMs","maxDbConnectLatencyMs","SENSITIVE_PATTERNS","pattern","replacement","maskSensitiveData","text","masked","replace","HealthCheckClient","constructor","options","healthConfig","config","appName","BUILD_APP_NAME","prefixLogs","_refreshPromise","_databasePools","Map","_resources","resources","redisClient","_getRedisClientForCache","_redisClientType","_cache","cacheKey","_getEnv","resource","redisResource","find","r","client","_getPool","url","has","pool","type","set","get","_checkDatabase","status","error","timings","maxConnect","connectMs","err","healthCheckTimings","message","_checkRedis","pong","Promise","resolve","reject","ping","result","_performHealthCheckInternal","sortedResources","Object","keys","sort","reduce","acc","key","hasError","values","some","lastCheckAt","Date","now","isStale","_formatResult","cached","performHealthCheck","then","catch","getCachedResult","console","refreshCache","clearCache","_log503Failure","req","payload","requestId","headers","id","line","msg","request_id","resourceErrors","entries","filter","map","warn","JSON","stringify","healthHandler","res","body","json","statusCode","cleanup","clear","module","exports"],"sources":["../../src/health/healthCheckClient.js"],"sourcesContent":["const {\n createDatabasePool,\n runHealthCheck,\n closePool,\n} = require('./databaseChecker')\nconst {\n getRedisClientType,\n REDIS_V4,\n IOREDIS,\n REDIS_V3,\n} = require('../redisUtils')\nconst { HealthCheckCache } = require('./healthCheckCache')\n\n/**\n * @param {string} name\n * @returns {number | undefined}\n */\nfunction readNumberEnv(name) {\n const raw = process.env[name]\n if (raw == null || raw === '') return undefined\n const num = Number(raw)\n return Number.isFinite(num) ? num : undefined\n}\n\n/** @type {{ checkIntervalMs: number, staleThresholdMs: number, checkTimeoutMs: number, maxDbConnectLatencyMs: number }} */\nconst DEFAULT_HEALTH_CONFIG = {\n checkIntervalMs: 30_000,\n staleThresholdMs: 180_000,\n checkTimeoutMs: 15_000,\n maxDbConnectLatencyMs:\n readNumberEnv('HEALTH_DB_MAX_CONNECT_LATENCY_MS') ?? 1000,\n}\n\nconst SENSITIVE_PATTERNS = [\n {\n pattern:\n /(postgres(?:ql)?|mysql|mongodb|redis|amqp):\\/\\/([^:]+):([^@]+)@([^:/]+)(:\\d+)?\\/([^\\s?]+)/gi,\n replacement: '$1://***:***@***$5/***',\n },\n {\n pattern: /(\\w+):\\/\\/([^:]+):([^@]+)@([^\\s/]+)/gi,\n replacement: '$1://***:***@***',\n },\n {\n pattern:\n /(password|passwd|pwd|secret|token|api[_-]?key|auth[_-]?token)[\"\\s]*[:=][\"\\s]*([^\\s,}\"]+)/gi,\n replacement: '$1=***',\n },\n {\n pattern:\n /(database|table|schema|role|user|relation|column|index)\\s*[\"']([^\"']+)[\"']/gi,\n replacement: '$1 \"***\"',\n },\n {\n pattern: /\\b(\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3})(:\\d+)?\\b/g,\n replacement: '***$2',\n },\n {\n pattern: /\\b(host|hostname|server)[\"\\s]*[:=][\"\\s]*([^\\s,}\"]+)/gi,\n replacement: '$1=***',\n },\n]\n\n/**\n * @param {string} text\n * @returns {string}\n */\nfunction maskSensitiveData(text) {\n if (!text || typeof text !== 'string') return text\n let masked = text\n for (const { pattern, replacement } of SENSITIVE_PATTERNS) {\n masked = masked.replace(pattern, replacement)\n }\n return masked\n}\n\n/**\n * @typedef {{ env: string, url?: string } | { env: string, client?: any }} HealthResource\n */\nclass HealthCheckClient {\n /**\n * @param {Object} options\n * @param {HealthResource[]} options.resources - Must include Redis resource with client\n * @param {Object} [options.config]\n * @param {string} [options.appName] - For cache key: healthcheck:${appName}\n * @param {string} [options.cacheKey] - Redis key (overrides appName)\n */\n constructor(options = {}) {\n this.healthConfig = { ...DEFAULT_HEALTH_CONFIG, ...options.config }\n this.appName =\n options.appName || process.env.BUILD_APP_NAME || 'unknown-app'\n\n this.prefixLogs = `[${this.appName}] [HealthCheck]`\n\n this._refreshPromise = null\n /** @type {Map<string, { pool: any, type: string }>} */\n this._databasePools = new Map()\n\n /** @type {HealthResource[]} */\n this._resources = options.resources || []\n\n const redisClient = this._getRedisClientForCache()\n if (redisClient) {\n this._redisClientType = getRedisClientType(redisClient)\n }\n\n this._cache = new HealthCheckCache({\n redisClient: redisClient || null,\n cacheKey: options.cacheKey,\n appName: this.appName,\n staleThresholdMs: this.healthConfig.staleThresholdMs,\n })\n }\n\n _getEnv(resource) {\n return resource.env ?? resource.name\n }\n\n _getRedisClientForCache() {\n const redisResource = this._resources.find(r => 'client' in r && r.client)\n return redisResource?.client || null\n }\n\n _getPool(env, url) {\n if (!this._databasePools.has(env)) {\n const { pool, type } = createDatabasePool(\n env,\n url,\n this.healthConfig.checkTimeoutMs\n )\n this._databasePools.set(env, { pool, type })\n }\n return this._databasePools.get(env)\n }\n\n async _checkDatabase(resource) {\n const env = this._getEnv(resource)\n const url = 'url' in resource ? resource.url : process.env[env]\n if (!url) {\n return { status: 'error', error: `Env ${env} not set` }\n }\n\n try {\n const { pool, type } = this._getPool(env, url)\n const timings = await runHealthCheck(pool, type)\n\n const maxConnect = this.healthConfig.maxDbConnectLatencyMs\n if (\n typeof maxConnect === 'number' &&\n Number.isFinite(maxConnect) &&\n timings?.connectMs != null &&\n timings.connectMs > maxConnect\n ) {\n return {\n status: 'error',\n error: `DB connect latency ${timings.connectMs}ms exceeds ${maxConnect}ms`,\n connectMs: timings.connectMs,\n }\n }\n\n return { status: 'ok', connectMs: timings.connectMs }\n } catch (err) {\n const timings = err?.healthCheckTimings\n return {\n status: 'error',\n error: maskSensitiveData(err.message),\n ...(timings && typeof timings === 'object'\n ? { connectMs: timings.connectMs }\n : {}),\n }\n }\n }\n\n async _checkRedis(resource) {\n const { client } = resource\n if (!client) return { status: 'ok' }\n\n try {\n let pong\n if (this._redisClientType === REDIS_V3) {\n pong = await new Promise((resolve, reject) => {\n client.ping((err, result) => {\n if (err) reject(err)\n else resolve(result)\n })\n })\n } else if (\n this._redisClientType === REDIS_V4 ||\n this._redisClientType === IOREDIS\n ) {\n pong = await client.ping()\n } else {\n return { status: 'error', error: 'Unknown Redis client type' }\n }\n\n return pong === 'PONG'\n ? { status: 'ok' }\n : { status: 'error', error: `Unexpected: ${pong}` }\n } catch (err) {\n return { status: 'error', error: maskSensitiveData(err.message) }\n }\n }\n\n async _performHealthCheckInternal() {\n const resources = {}\n\n for (const resource of this._resources) {\n const env = this._getEnv(resource)\n\n if ('client' in resource && resource.client) {\n resources[env] = await this._checkRedis(resource)\n } else {\n resources[env] = await this._checkDatabase(resource)\n }\n }\n\n const sortedResources = Object.keys(resources)\n .sort()\n .reduce((acc, key) => {\n acc[key] = resources[key]\n return acc\n }, {})\n\n const hasError = Object.values(resources).some(r => r.status === 'error')\n const lastCheckAt = Date.now()\n\n return {\n status: hasError ? 'error' : 'ok',\n lastCheckAt,\n resources: sortedResources,\n isStale: false,\n config: this.healthConfig,\n }\n }\n\n _formatResult(result, cached = false) {\n const isStale =\n !result.lastCheckAt ||\n Date.now() - result.lastCheckAt > this.healthConfig.staleThresholdMs\n\n return {\n ...result,\n isStale,\n status: isStale ? 'stale' : result.status,\n ...(isStale && {\n error:\n 'Health check data is stale, health-check worker may not be running. Resource statuses are unknown.',\n }),\n ...(cached && { cached: true }),\n }\n }\n\n async performHealthCheck() {\n if (this._refreshPromise) {\n return this._refreshPromise\n }\n\n this._refreshPromise = this._performHealthCheckInternal()\n .then(result => {\n this._refreshPromise = null\n return this._formatResult(result)\n })\n .catch(err => {\n this._refreshPromise = null\n throw err\n })\n\n return this._refreshPromise\n }\n\n async getCachedResult() {\n try {\n const cached = await this._cache.get()\n if (cached) return this._formatResult(cached)\n return null\n } catch (err) {\n console.error(`${this.prefixLogs} Failed to read from cache:`, err)\n return {\n status: 'error',\n lastCheckAt: null,\n resources: {},\n isStale: true,\n error:\n 'Redis unavailable, unable to read health status of other resources',\n config: this.healthConfig,\n }\n }\n }\n\n async refreshCache() {\n const result = await this._performHealthCheckInternal()\n await this._cache.set(result)\n return result\n }\n\n clearCache() {\n this._refreshPromise = null\n }\n\n /**\n * Logs a structured line when health returns 503 so log drains / Loki can store\n * and query failed health data. Includes request_id when present for correlation\n * with router logs.\n * @param {object} req - Express request (for request_id header)\n * @param {object} payload - Safe summary of the 503 response body\n */\n _log503Failure(req, payload) {\n const requestId =\n (req && req.headers && req.headers['x-request-id']) ||\n (req && req.id) ||\n null\n const line = {\n msg: 'HealthCheck 503',\n appName: this.appName,\n request_id: requestId,\n error: payload.error,\n isStale: payload.isStale,\n status: payload.status,\n resourceErrors:\n payload.resources &&\n Object.entries(payload.resources)\n .filter(([, r]) => r && r.status === 'error')\n .map(([env, r]) => ({ env, error: r.error })),\n }\n console.warn(JSON.stringify(line))\n }\n\n healthHandler() {\n return async (req, res) => {\n try {\n const result = await this.getCachedResult()\n\n if (!result) {\n const body = {\n status: 'error',\n lastCheckAt: null,\n resources: {},\n isStale: true,\n error:\n 'No health check data yet, health-check worker may not be running',\n config: this.healthConfig,\n }\n this._log503Failure(req, body)\n res.status(503).json(body)\n return\n }\n\n const statusCode = result.status === 'ok' ? 200 : 503\n if (statusCode === 503) {\n this._log503Failure(req, result)\n }\n res.status(statusCode).json(result)\n } catch (err) {\n console.error(`${this.prefixLogs} Health check failed:`, err)\n const body = {\n status: 'error',\n lastCheckAt: null,\n resources: {},\n isStale: true,\n error:\n 'Redis unavailable, unable to read health status of other resources',\n config: this.healthConfig,\n }\n this._log503Failure(req, body)\n res.status(503).json(body)\n }\n }\n }\n\n async cleanup() {\n for (const [, { pool }] of this._databasePools) {\n try {\n await closePool(pool)\n } catch (err) {\n console.error(`${this.prefixLogs} Error closing database pool:`, err)\n }\n }\n this._databasePools.clear()\n }\n}\n\nmodule.exports = { HealthCheckClient, DEFAULT_HEALTH_CONFIG }\n"],"mappings":";;AAAA,MAAM;EACJA,kBAAkB;EAClBC,cAAc;EACdC;AACF,CAAC,GAAGC,OAAO,CAAC,mBAAmB,CAAC;AAChC,MAAM;EACJC,kBAAkB;EAClBC,QAAQ;EACRC,OAAO;EACPC;AACF,CAAC,GAAGJ,OAAO,CAAC,eAAe,CAAC;AAC5B,MAAM;EAAEK;AAAiB,CAAC,GAAGL,OAAO,CAAC,oBAAoB,CAAC;;AAE1D;AACA;AACA;AACA;AACA,SAASM,aAAaA,CAACC,IAAI,EAAE;EAC3B,MAAMC,GAAG,GAAGC,OAAO,CAACC,GAAG,CAACH,IAAI,CAAC;EAC7B,IAAIC,GAAG,IAAI,IAAI,IAAIA,GAAG,KAAK,EAAE,EAAE,OAAOG,SAAS;EAC/C,MAAMC,GAAG,GAAGC,MAAM,CAACL,GAAG,CAAC;EACvB,OAAOK,MAAM,CAACC,QAAQ,CAACF,GAAG,CAAC,GAAGA,GAAG,GAAGD,SAAS;AAC/C;;AAEA;AACA,MAAMI,qBAAqB,GAAG;EAC5BC,eAAe,EAAE,MAAM;EACvBC,gBAAgB,EAAE,OAAO;EACzBC,cAAc,EAAE,MAAM;EACtBC,qBAAqB,EACnBb,aAAa,CAAC,kCAAkC,CAAC,IAAI;AACzD,CAAC;AAED,MAAMc,kBAAkB,GAAG,CACzB;EACEC,OAAO,EACL,6FAA6F;EAC/FC,WAAW,EAAE;AACf,CAAC,EACD;EACED,OAAO,EAAE,uCAAuC;EAChDC,WAAW,EAAE;AACf,CAAC,EACD;EACED,OAAO,EACL,4FAA4F;EAC9FC,WAAW,EAAE;AACf,CAAC,EACD;EACED,OAAO,EACL,8EAA8E;EAChFC,WAAW,EAAE;AACf,CAAC,EACD;EACED,OAAO,EAAE,kDAAkD;EAC3DC,WAAW,EAAE;AACf,CAAC,EACD;EACED,OAAO,EAAE,uDAAuD;EAChEC,WAAW,EAAE;AACf,CAAC,CACF;;AAED;AACA;AACA;AACA;AACA,SAASC,iBAAiBA,CAACC,IAAI,EAAE;EAC/B,IAAI,CAACA,IAAI,IAAI,OAAOA,IAAI,KAAK,QAAQ,EAAE,OAAOA,IAAI;EAClD,IAAIC,MAAM,GAAGD,IAAI;EACjB,KAAK,MAAM;IAAEH,OAAO;IAAEC;EAAY,CAAC,IAAIF,kBAAkB,EAAE;IACzDK,MAAM,GAAGA,MAAM,CAACC,OAAO,CAACL,OAAO,EAAEC,WAAW,CAAC;EAC/C;EACA,OAAOG,MAAM;AACf;;AAEA;AACA;AACA;AACA,MAAME,iBAAiB,CAAC;EACtB;AACF;AACA;AACA;AACA;AACA;AACA;EACEC,WAAWA,CAACC,OAAO,GAAG,CAAC,CAAC,EAAE;IACxB,IAAI,CAACC,YAAY,GAAG;MAAE,GAAGf,qBAAqB;MAAE,GAAGc,OAAO,CAACE;IAAO,CAAC;IACnE,IAAI,CAACC,OAAO,GACVH,OAAO,CAACG,OAAO,IAAIvB,OAAO,CAACC,GAAG,CAACuB,cAAc,IAAI,aAAa;IAEhE,IAAI,CAACC,UAAU,GAAG,IAAI,IAAI,CAACF,OAAO,iBAAiB;IAEnD,IAAI,CAACG,eAAe,GAAG,IAAI;IAC3B;IACA,IAAI,CAACC,cAAc,GAAG,IAAIC,GAAG,CAAC,CAAC;;IAE/B;IACA,IAAI,CAACC,UAAU,GAAGT,OAAO,CAACU,SAAS,IAAI,EAAE;IAEzC,MAAMC,WAAW,GAAG,IAAI,CAACC,uBAAuB,CAAC,CAAC;IAClD,IAAID,WAAW,EAAE;MACf,IAAI,CAACE,gBAAgB,GAAGzC,kBAAkB,CAACuC,WAAW,CAAC;IACzD;IAEA,IAAI,CAACG,MAAM,GAAG,IAAItC,gBAAgB,CAAC;MACjCmC,WAAW,EAAEA,WAAW,IAAI,IAAI;MAChCI,QAAQ,EAAEf,OAAO,CAACe,QAAQ;MAC1BZ,OAAO,EAAE,IAAI,CAACA,OAAO;MACrBf,gBAAgB,EAAE,IAAI,CAACa,YAAY,CAACb;IACtC,CAAC,CAAC;EACJ;EAEA4B,OAAOA,CAACC,QAAQ,EAAE;IAChB,OAAOA,QAAQ,CAACpC,GAAG,IAAIoC,QAAQ,CAACvC,IAAI;EACtC;EAEAkC,uBAAuBA,CAAA,EAAG;IACxB,MAAMM,aAAa,GAAG,IAAI,CAACT,UAAU,CAACU,IAAI,CAACC,CAAC,IAAI,QAAQ,IAAIA,CAAC,IAAIA,CAAC,CAACC,MAAM,CAAC;IAC1E,OAAOH,aAAa,EAAEG,MAAM,IAAI,IAAI;EACtC;EAEAC,QAAQA,CAACzC,GAAG,EAAE0C,GAAG,EAAE;IACjB,IAAI,CAAC,IAAI,CAAChB,cAAc,CAACiB,GAAG,CAAC3C,GAAG,CAAC,EAAE;MACjC,MAAM;QAAE4C,IAAI;QAAEC;MAAK,CAAC,GAAG1D,kBAAkB,CACvCa,GAAG,EACH0C,GAAG,EACH,IAAI,CAACtB,YAAY,CAACZ,cACpB,CAAC;MACD,IAAI,CAACkB,cAAc,CAACoB,GAAG,CAAC9C,GAAG,EAAE;QAAE4C,IAAI;QAAEC;MAAK,CAAC,CAAC;IAC9C;IACA,OAAO,IAAI,CAACnB,cAAc,CAACqB,GAAG,CAAC/C,GAAG,CAAC;EACrC;EAEA,MAAMgD,cAAcA,CAACZ,QAAQ,EAAE;IAC7B,MAAMpC,GAAG,GAAG,IAAI,CAACmC,OAAO,CAACC,QAAQ,CAAC;IAClC,MAAMM,GAAG,GAAG,KAAK,IAAIN,QAAQ,GAAGA,QAAQ,CAACM,GAAG,GAAG3C,OAAO,CAACC,GAAG,CAACA,GAAG,CAAC;IAC/D,IAAI,CAAC0C,GAAG,EAAE;MACR,OAAO;QAAEO,MAAM,EAAE,OAAO;QAAEC,KAAK,EAAE,OAAOlD,GAAG;MAAW,CAAC;IACzD;IAEA,IAAI;MACF,MAAM;QAAE4C,IAAI;QAAEC;MAAK,CAAC,GAAG,IAAI,CAACJ,QAAQ,CAACzC,GAAG,EAAE0C,GAAG,CAAC;MAC9C,MAAMS,OAAO,GAAG,MAAM/D,cAAc,CAACwD,IAAI,EAAEC,IAAI,CAAC;MAEhD,MAAMO,UAAU,GAAG,IAAI,CAAChC,YAAY,CAACX,qBAAqB;MAC1D,IACE,OAAO2C,UAAU,KAAK,QAAQ,IAC9BjD,MAAM,CAACC,QAAQ,CAACgD,UAAU,CAAC,IAC3BD,OAAO,EAAEE,SAAS,IAAI,IAAI,IAC1BF,OAAO,CAACE,SAAS,GAAGD,UAAU,EAC9B;QACA,OAAO;UACLH,MAAM,EAAE,OAAO;UACfC,KAAK,EAAE,sBAAsBC,OAAO,CAACE,SAAS,cAAcD,UAAU,IAAI;UAC1EC,SAAS,EAAEF,OAAO,CAACE;QACrB,CAAC;MACH;MAEA,OAAO;QAAEJ,MAAM,EAAE,IAAI;QAAEI,SAAS,EAAEF,OAAO,CAACE;MAAU,CAAC;IACvD,CAAC,CAAC,OAAOC,GAAG,EAAE;MACZ,MAAMH,OAAO,GAAGG,GAAG,EAAEC,kBAAkB;MACvC,OAAO;QACLN,MAAM,EAAE,OAAO;QACfC,KAAK,EAAErC,iBAAiB,CAACyC,GAAG,CAACE,OAAO,CAAC;QACrC,IAAIL,OAAO,IAAI,OAAOA,OAAO,KAAK,QAAQ,GACtC;UAAEE,SAAS,EAAEF,OAAO,CAACE;QAAU,CAAC,GAChC,CAAC,CAAC;MACR,CAAC;IACH;EACF;EAEA,MAAMI,WAAWA,CAACrB,QAAQ,EAAE;IAC1B,MAAM;MAAEI;IAAO,CAAC,GAAGJ,QAAQ;IAC3B,IAAI,CAACI,MAAM,EAAE,OAAO;MAAES,MAAM,EAAE;IAAK,CAAC;IAEpC,IAAI;MACF,IAAIS,IAAI;MACR,IAAI,IAAI,CAAC1B,gBAAgB,KAAKtC,QAAQ,EAAE;QACtCgE,IAAI,GAAG,MAAM,IAAIC,OAAO,CAAC,CAACC,OAAO,EAAEC,MAAM,KAAK;UAC5CrB,MAAM,CAACsB,IAAI,CAAC,CAACR,GAAG,EAAES,MAAM,KAAK;YAC3B,IAAIT,GAAG,EAAEO,MAAM,CAACP,GAAG,CAAC,MACfM,OAAO,CAACG,MAAM,CAAC;UACtB,CAAC,CAAC;QACJ,CAAC,CAAC;MACJ,CAAC,MAAM,IACL,IAAI,CAAC/B,gBAAgB,KAAKxC,QAAQ,IAClC,IAAI,CAACwC,gBAAgB,KAAKvC,OAAO,EACjC;QACAiE,IAAI,GAAG,MAAMlB,MAAM,CAACsB,IAAI,CAAC,CAAC;MAC5B,CAAC,MAAM;QACL,OAAO;UAAEb,MAAM,EAAE,OAAO;UAAEC,KAAK,EAAE;QAA4B,CAAC;MAChE;MAEA,OAAOQ,IAAI,KAAK,MAAM,GAClB;QAAET,MAAM,EAAE;MAAK,CAAC,GAChB;QAAEA,MAAM,EAAE,OAAO;QAAEC,KAAK,EAAE,eAAeQ,IAAI;MAAG,CAAC;IACvD,CAAC,CAAC,OAAOJ,GAAG,EAAE;MACZ,OAAO;QAAEL,MAAM,EAAE,OAAO;QAAEC,KAAK,EAAErC,iBAAiB,CAACyC,GAAG,CAACE,OAAO;MAAE,CAAC;IACnE;EACF;EAEA,MAAMQ,2BAA2BA,CAAA,EAAG;IAClC,MAAMnC,SAAS,GAAG,CAAC,CAAC;IAEpB,KAAK,MAAMO,QAAQ,IAAI,IAAI,CAACR,UAAU,EAAE;MACtC,MAAM5B,GAAG,GAAG,IAAI,CAACmC,OAAO,CAACC,QAAQ,CAAC;MAElC,IAAI,QAAQ,IAAIA,QAAQ,IAAIA,QAAQ,CAACI,MAAM,EAAE;QAC3CX,SAAS,CAAC7B,GAAG,CAAC,GAAG,MAAM,IAAI,CAACyD,WAAW,CAACrB,QAAQ,CAAC;MACnD,CAAC,MAAM;QACLP,SAAS,CAAC7B,GAAG,CAAC,GAAG,MAAM,IAAI,CAACgD,cAAc,CAACZ,QAAQ,CAAC;MACtD;IACF;IAEA,MAAM6B,eAAe,GAAGC,MAAM,CAACC,IAAI,CAACtC,SAAS,CAAC,CAC3CuC,IAAI,CAAC,CAAC,CACNC,MAAM,CAAC,CAACC,GAAG,EAAEC,GAAG,KAAK;MACpBD,GAAG,CAACC,GAAG,CAAC,GAAG1C,SAAS,CAAC0C,GAAG,CAAC;MACzB,OAAOD,GAAG;IACZ,CAAC,EAAE,CAAC,CAAC,CAAC;IAER,MAAME,QAAQ,GAAGN,MAAM,CAACO,MAAM,CAAC5C,SAAS,CAAC,CAAC6C,IAAI,CAACnC,CAAC,IAAIA,CAAC,CAACU,MAAM,KAAK,OAAO,CAAC;IACzE,MAAM0B,WAAW,GAAGC,IAAI,CAACC,GAAG,CAAC,CAAC;IAE9B,OAAO;MACL5B,MAAM,EAAEuB,QAAQ,GAAG,OAAO,GAAG,IAAI;MACjCG,WAAW;MACX9C,SAAS,EAAEoC,eAAe;MAC1Ba,OAAO,EAAE,KAAK;MACdzD,MAAM,EAAE,IAAI,CAACD;IACf,CAAC;EACH;EAEA2D,aAAaA,CAAChB,MAAM,EAAEiB,MAAM,GAAG,KAAK,EAAE;IACpC,MAAMF,OAAO,GACX,CAACf,MAAM,CAACY,WAAW,IACnBC,IAAI,CAACC,GAAG,CAAC,CAAC,GAAGd,MAAM,CAACY,WAAW,GAAG,IAAI,CAACvD,YAAY,CAACb,gBAAgB;IAEtE,OAAO;MACL,GAAGwD,MAAM;MACTe,OAAO;MACP7B,MAAM,EAAE6B,OAAO,GAAG,OAAO,GAAGf,MAAM,CAACd,MAAM;MACzC,IAAI6B,OAAO,IAAI;QACb5B,KAAK,EACH;MACJ,CAAC,CAAC;MACF,IAAI8B,MAAM,IAAI;QAAEA,MAAM,EAAE;MAAK,CAAC;IAChC,CAAC;EACH;EAEA,MAAMC,kBAAkBA,CAAA,EAAG;IACzB,IAAI,IAAI,CAACxD,eAAe,EAAE;MACxB,OAAO,IAAI,CAACA,eAAe;IAC7B;IAEA,IAAI,CAACA,eAAe,GAAG,IAAI,CAACuC,2BAA2B,CAAC,CAAC,CACtDkB,IAAI,CAACnB,MAAM,IAAI;MACd,IAAI,CAACtC,eAAe,GAAG,IAAI;MAC3B,OAAO,IAAI,CAACsD,aAAa,CAAChB,MAAM,CAAC;IACnC,CAAC,CAAC,CACDoB,KAAK,CAAC7B,GAAG,IAAI;MACZ,IAAI,CAAC7B,eAAe,GAAG,IAAI;MAC3B,MAAM6B,GAAG;IACX,CAAC,CAAC;IAEJ,OAAO,IAAI,CAAC7B,eAAe;EAC7B;EAEA,MAAM2D,eAAeA,CAAA,EAAG;IACtB,IAAI;MACF,MAAMJ,MAAM,GAAG,MAAM,IAAI,CAAC/C,MAAM,CAACc,GAAG,CAAC,CAAC;MACtC,IAAIiC,MAAM,EAAE,OAAO,IAAI,CAACD,aAAa,CAACC,MAAM,CAAC;MAC7C,OAAO,IAAI;IACb,CAAC,CAAC,OAAO1B,GAAG,EAAE;MACZ+B,OAAO,CAACnC,KAAK,CAAC,GAAG,IAAI,CAAC1B,UAAU,6BAA6B,EAAE8B,GAAG,CAAC;MACnE,OAAO;QACLL,MAAM,EAAE,OAAO;QACf0B,WAAW,EAAE,IAAI;QACjB9C,SAAS,EAAE,CAAC,CAAC;QACbiD,OAAO,EAAE,IAAI;QACb5B,KAAK,EACH,oEAAoE;QACtE7B,MAAM,EAAE,IAAI,CAACD;MACf,CAAC;IACH;EACF;EAEA,MAAMkE,YAAYA,CAAA,EAAG;IACnB,MAAMvB,MAAM,GAAG,MAAM,IAAI,CAACC,2BAA2B,CAAC,CAAC;IACvD,MAAM,IAAI,CAAC/B,MAAM,CAACa,GAAG,CAACiB,MAAM,CAAC;IAC7B,OAAOA,MAAM;EACf;EAEAwB,UAAUA,CAAA,EAAG;IACX,IAAI,CAAC9D,eAAe,GAAG,IAAI;EAC7B;;EAEA;AACF;AACA;AACA;AACA;AACA;AACA;EACE+D,cAAcA,CAACC,GAAG,EAAEC,OAAO,EAAE;IAC3B,MAAMC,SAAS,GACZF,GAAG,IAAIA,GAAG,CAACG,OAAO,IAAIH,GAAG,CAACG,OAAO,CAAC,cAAc,CAAC,IACjDH,GAAG,IAAIA,GAAG,CAACI,EAAG,IACf,IAAI;IACN,MAAMC,IAAI,GAAG;MACXC,GAAG,EAAE,iBAAiB;MACtBzE,OAAO,EAAE,IAAI,CAACA,OAAO;MACrB0E,UAAU,EAAEL,SAAS;MACrBzC,KAAK,EAAEwC,OAAO,CAACxC,KAAK;MACpB4B,OAAO,EAAEY,OAAO,CAACZ,OAAO;MACxB7B,MAAM,EAAEyC,OAAO,CAACzC,MAAM;MACtBgD,cAAc,EACZP,OAAO,CAAC7D,SAAS,IACjBqC,MAAM,CAACgC,OAAO,CAACR,OAAO,CAAC7D,SAAS,CAAC,CAC9BsE,MAAM,CAAC,CAAC,GAAG5D,CAAC,CAAC,KAAKA,CAAC,IAAIA,CAAC,CAACU,MAAM,KAAK,OAAO,CAAC,CAC5CmD,GAAG,CAAC,CAAC,CAACpG,GAAG,EAAEuC,CAAC,CAAC,MAAM;QAAEvC,GAAG;QAAEkD,KAAK,EAAEX,CAAC,CAACW;MAAM,CAAC,CAAC;IAClD,CAAC;IACDmC,OAAO,CAACgB,IAAI,CAACC,IAAI,CAACC,SAAS,CAACT,IAAI,CAAC,CAAC;EACpC;EAEAU,aAAaA,CAAA,EAAG;IACd,OAAO,OAAOf,GAAG,EAAEgB,GAAG,KAAK;MACzB,IAAI;QACF,MAAM1C,MAAM,GAAG,MAAM,IAAI,CAACqB,eAAe,CAAC,CAAC;QAE3C,IAAI,CAACrB,MAAM,EAAE;UACX,MAAM2C,IAAI,GAAG;YACXzD,MAAM,EAAE,OAAO;YACf0B,WAAW,EAAE,IAAI;YACjB9C,SAAS,EAAE,CAAC,CAAC;YACbiD,OAAO,EAAE,IAAI;YACb5B,KAAK,EACH,kEAAkE;YACpE7B,MAAM,EAAE,IAAI,CAACD;UACf,CAAC;UACD,IAAI,CAACoE,cAAc,CAACC,GAAG,EAAEiB,IAAI,CAAC;UAC9BD,GAAG,CAACxD,MAAM,CAAC,GAAG,CAAC,CAAC0D,IAAI,CAACD,IAAI,CAAC;UAC1B;QACF;QAEA,MAAME,UAAU,GAAG7C,MAAM,CAACd,MAAM,KAAK,IAAI,GAAG,GAAG,GAAG,GAAG;QACrD,IAAI2D,UAAU,KAAK,GAAG,EAAE;UACtB,IAAI,CAACpB,cAAc,CAACC,GAAG,EAAE1B,MAAM,CAAC;QAClC;QACA0C,GAAG,CAACxD,MAAM,CAAC2D,UAAU,CAAC,CAACD,IAAI,CAAC5C,MAAM,CAAC;MACrC,CAAC,CAAC,OAAOT,GAAG,EAAE;QACZ+B,OAAO,CAACnC,KAAK,CAAC,GAAG,IAAI,CAAC1B,UAAU,uBAAuB,EAAE8B,GAAG,CAAC;QAC7D,MAAMoD,IAAI,GAAG;UACXzD,MAAM,EAAE,OAAO;UACf0B,WAAW,EAAE,IAAI;UACjB9C,SAAS,EAAE,CAAC,CAAC;UACbiD,OAAO,EAAE,IAAI;UACb5B,KAAK,EACH,oEAAoE;UACtE7B,MAAM,EAAE,IAAI,CAACD;QACf,CAAC;QACD,IAAI,CAACoE,cAAc,CAACC,GAAG,EAAEiB,IAAI,CAAC;QAC9BD,GAAG,CAACxD,MAAM,CAAC,GAAG,CAAC,CAAC0D,IAAI,CAACD,IAAI,CAAC;MAC5B;IACF,CAAC;EACH;EAEA,MAAMG,OAAOA,CAAA,EAAG;IACd,KAAK,MAAM,GAAG;MAAEjE;IAAK,CAAC,CAAC,IAAI,IAAI,CAAClB,cAAc,EAAE;MAC9C,IAAI;QACF,MAAMrC,SAAS,CAACuD,IAAI,CAAC;MACvB,CAAC,CAAC,OAAOU,GAAG,EAAE;QACZ+B,OAAO,CAACnC,KAAK,CAAC,GAAG,IAAI,CAAC1B,UAAU,+BAA+B,EAAE8B,GAAG,CAAC;MACvE;IACF;IACA,IAAI,CAAC5B,cAAc,CAACoF,KAAK,CAAC,CAAC;EAC7B;AACF;AAEAC,MAAM,CAACC,OAAO,GAAG;EAAE/F,iBAAiB;EAAEZ;AAAsB,CAAC","ignoreList":[]}
|
|
1
|
+
{"version":3,"file":"healthCheckClient.js","names":["createDatabasePool","runHealthCheck","closePool","require","getRedisClientType","REDIS_V4","IOREDIS","REDIS_V3","HealthCheckCache","readNumberEnv","name","raw","process","env","undefined","num","Number","isFinite","DEFAULT_HEALTH_CONFIG","checkIntervalMs","staleThresholdMs","checkTimeoutMs","maxDbConnectLatencyMs","SENSITIVE_PATTERNS","pattern","replacement","maskSensitiveData","text","masked","replace","HealthCheckClient","constructor","options","healthConfig","config","appName","BUILD_APP_NAME","prefixLogs","_refreshPromise","_databasePools","Map","_resources","resources","redisClient","_getRedisClientForCache","_redisClientType","_cache","cacheKey","_getEnv","resource","redisResource","find","r","client","_getPool","url","has","pool","type","set","get","_checkDatabase","status","error","timings","maxConnect","connectMs","err","healthCheckTimings","message","_checkRedis","pong","Promise","resolve","reject","ping","result","_performHealthCheckInternal","sortedResources","Object","keys","sort","reduce","acc","key","hasError","values","some","lastCheckAt","Date","now","isStale","_formatResult","cached","performHealthCheck","then","catch","getCachedResult","console","refreshCache","clearCache","_log503Failure","req","payload","requestId","headers","id","line","msg","request_id","resourceErrors","entries","filter","map","warn","JSON","stringify","healthHandler","res","body","json","statusCode","cleanup","clear","module","exports"],"sources":["../../src/health/healthCheckClient.js"],"sourcesContent":["const {\n createDatabasePool,\n runHealthCheck,\n closePool,\n} = require('./databaseChecker')\nconst {\n getRedisClientType,\n REDIS_V4,\n IOREDIS,\n REDIS_V3,\n} = require('../redisUtils')\nconst { HealthCheckCache } = require('./healthCheckCache')\n\n/**\n * @param {string} name\n * @returns {number | undefined}\n */\nfunction readNumberEnv(name) {\n const raw = process.env[name]\n if (raw == null || raw === '') return undefined\n const num = Number(raw)\n return Number.isFinite(num) ? num : undefined\n}\n\n/** @type {{ checkIntervalMs: number, staleThresholdMs: number, checkTimeoutMs: number, maxDbConnectLatencyMs: number }} */\nconst DEFAULT_HEALTH_CONFIG = {\n checkIntervalMs: 30_000,\n staleThresholdMs: 180_000,\n checkTimeoutMs: 15_000,\n maxDbConnectLatencyMs:\n readNumberEnv('HEALTH_DB_MAX_CONNECT_LATENCY_MS') ?? 30_000,\n}\n\nconst SENSITIVE_PATTERNS = [\n {\n pattern:\n /(postgres(?:ql)?|mysql|mongodb|redis|amqp):\\/\\/([^:]+):([^@]+)@([^:/]+)(:\\d+)?\\/([^\\s?]+)/gi,\n replacement: '$1://***:***@***$5/***',\n },\n {\n pattern: /(\\w+):\\/\\/([^:]+):([^@]+)@([^\\s/]+)/gi,\n replacement: '$1://***:***@***',\n },\n {\n pattern:\n /(password|passwd|pwd|secret|token|api[_-]?key|auth[_-]?token)[\"\\s]*[:=][\"\\s]*([^\\s,}\"]+)/gi,\n replacement: '$1=***',\n },\n {\n pattern:\n /(database|table|schema|role|user|relation|column|index)\\s*[\"']([^\"']+)[\"']/gi,\n replacement: '$1 \"***\"',\n },\n {\n pattern: /\\b(\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3})(:\\d+)?\\b/g,\n replacement: '***$2',\n },\n {\n pattern: /\\b(host|hostname|server)[\"\\s]*[:=][\"\\s]*([^\\s,}\"]+)/gi,\n replacement: '$1=***',\n },\n]\n\n/**\n * @param {string} text\n * @returns {string}\n */\nfunction maskSensitiveData(text) {\n if (!text || typeof text !== 'string') return text\n let masked = text\n for (const { pattern, replacement } of SENSITIVE_PATTERNS) {\n masked = masked.replace(pattern, replacement)\n }\n return masked\n}\n\n/**\n * @typedef {{ env: string, url?: string } | { env: string, client?: any }} HealthResource\n */\nclass HealthCheckClient {\n /**\n * @param {Object} options\n * @param {HealthResource[]} options.resources - Must include Redis resource with client\n * @param {Object} [options.config]\n * @param {string} [options.appName] - For cache key: healthcheck:${appName}\n * @param {string} [options.cacheKey] - Redis key (overrides appName)\n */\n constructor(options = {}) {\n this.healthConfig = { ...DEFAULT_HEALTH_CONFIG, ...options.config }\n this.appName =\n options.appName || process.env.BUILD_APP_NAME || 'unknown-app'\n\n this.prefixLogs = `[${this.appName}] [HealthCheck]`\n\n this._refreshPromise = null\n /** @type {Map<string, { pool: any, type: string }>} */\n this._databasePools = new Map()\n\n /** @type {HealthResource[]} */\n this._resources = options.resources || []\n\n const redisClient = this._getRedisClientForCache()\n if (redisClient) {\n this._redisClientType = getRedisClientType(redisClient)\n }\n\n this._cache = new HealthCheckCache({\n redisClient: redisClient || null,\n cacheKey: options.cacheKey,\n appName: this.appName,\n staleThresholdMs: this.healthConfig.staleThresholdMs,\n })\n }\n\n _getEnv(resource) {\n return resource.env ?? resource.name\n }\n\n _getRedisClientForCache() {\n const redisResource = this._resources.find(r => 'client' in r && r.client)\n return redisResource?.client || null\n }\n\n _getPool(env, url) {\n if (!this._databasePools.has(env)) {\n const { pool, type } = createDatabasePool(\n env,\n url,\n this.healthConfig.checkTimeoutMs\n )\n this._databasePools.set(env, { pool, type })\n }\n return this._databasePools.get(env)\n }\n\n async _checkDatabase(resource) {\n const env = this._getEnv(resource)\n const url = 'url' in resource ? resource.url : process.env[env]\n if (!url) {\n return { status: 'error', error: `Env ${env} not set` }\n }\n\n try {\n const { pool, type } = this._getPool(env, url)\n const timings = await runHealthCheck(pool, type)\n\n const maxConnect = this.healthConfig.maxDbConnectLatencyMs\n if (\n typeof maxConnect === 'number' &&\n Number.isFinite(maxConnect) &&\n timings?.connectMs != null &&\n timings.connectMs > maxConnect\n ) {\n return {\n status: 'error',\n error: `DB connect latency ${timings.connectMs}ms exceeds ${maxConnect}ms`,\n connectMs: timings.connectMs,\n }\n }\n\n return { status: 'ok', connectMs: timings.connectMs }\n } catch (err) {\n const timings = err?.healthCheckTimings\n return {\n status: 'error',\n error: maskSensitiveData(err.message),\n ...(timings && typeof timings === 'object'\n ? { connectMs: timings.connectMs }\n : {}),\n }\n }\n }\n\n async _checkRedis(resource) {\n const { client } = resource\n if (!client) return { status: 'ok' }\n\n try {\n let pong\n if (this._redisClientType === REDIS_V3) {\n pong = await new Promise((resolve, reject) => {\n client.ping((err, result) => {\n if (err) reject(err)\n else resolve(result)\n })\n })\n } else if (\n this._redisClientType === REDIS_V4 ||\n this._redisClientType === IOREDIS\n ) {\n pong = await client.ping()\n } else {\n return { status: 'error', error: 'Unknown Redis client type' }\n }\n\n return pong === 'PONG'\n ? { status: 'ok' }\n : { status: 'error', error: `Unexpected: ${pong}` }\n } catch (err) {\n return { status: 'error', error: maskSensitiveData(err.message) }\n }\n }\n\n async _performHealthCheckInternal() {\n const resources = {}\n\n for (const resource of this._resources) {\n const env = this._getEnv(resource)\n\n if ('client' in resource && resource.client) {\n resources[env] = await this._checkRedis(resource)\n } else {\n resources[env] = await this._checkDatabase(resource)\n }\n }\n\n const sortedResources = Object.keys(resources)\n .sort()\n .reduce((acc, key) => {\n acc[key] = resources[key]\n return acc\n }, {})\n\n const hasError = Object.values(resources).some(r => r.status === 'error')\n const lastCheckAt = Date.now()\n\n return {\n status: hasError ? 'error' : 'ok',\n lastCheckAt,\n resources: sortedResources,\n isStale: false,\n config: this.healthConfig,\n }\n }\n\n _formatResult(result, cached = false) {\n const isStale =\n !result.lastCheckAt ||\n Date.now() - result.lastCheckAt > this.healthConfig.staleThresholdMs\n\n return {\n ...result,\n isStale,\n status: isStale ? 'stale' : result.status,\n ...(isStale && {\n error:\n 'Health check data is stale, health-check worker may not be running. Resource statuses are unknown.',\n }),\n ...(cached && { cached: true }),\n }\n }\n\n async performHealthCheck() {\n if (this._refreshPromise) {\n return this._refreshPromise\n }\n\n this._refreshPromise = this._performHealthCheckInternal()\n .then(result => {\n this._refreshPromise = null\n return this._formatResult(result)\n })\n .catch(err => {\n this._refreshPromise = null\n throw err\n })\n\n return this._refreshPromise\n }\n\n async getCachedResult() {\n try {\n const cached = await this._cache.get()\n if (cached) return this._formatResult(cached)\n return null\n } catch (err) {\n console.error(`${this.prefixLogs} Failed to read from cache:`, err)\n return {\n status: 'error',\n lastCheckAt: null,\n resources: {},\n isStale: true,\n error:\n 'Redis unavailable, unable to read health status of other resources',\n config: this.healthConfig,\n }\n }\n }\n\n async refreshCache() {\n const result = await this._performHealthCheckInternal()\n await this._cache.set(result)\n return result\n }\n\n clearCache() {\n this._refreshPromise = null\n }\n\n /**\n * Logs a structured line when health returns 503 so log drains / Loki can store\n * and query failed health data. Includes request_id when present for correlation\n * with router logs.\n * @param {object} req - Express request (for request_id header)\n * @param {object} payload - Safe summary of the 503 response body\n */\n _log503Failure(req, payload) {\n const requestId =\n (req && req.headers && req.headers['x-request-id']) ||\n (req && req.id) ||\n null\n const line = {\n msg: 'HealthCheck 503',\n appName: this.appName,\n request_id: requestId,\n error: payload.error,\n isStale: payload.isStale,\n status: payload.status,\n resourceErrors:\n payload.resources &&\n Object.entries(payload.resources)\n .filter(([, r]) => r && r.status === 'error')\n .map(([env, r]) => ({ env, error: r.error })),\n }\n console.warn(JSON.stringify(line))\n }\n\n healthHandler() {\n return async (req, res) => {\n try {\n const result = await this.getCachedResult()\n\n if (!result) {\n const body = {\n status: 'error',\n lastCheckAt: null,\n resources: {},\n isStale: true,\n error:\n 'No health check data yet, health-check worker may not be running',\n config: this.healthConfig,\n }\n this._log503Failure(req, body)\n res.status(503).json(body)\n return\n }\n\n const statusCode = result.status === 'ok' ? 200 : 503\n if (statusCode === 503) {\n this._log503Failure(req, result)\n }\n res.status(statusCode).json(result)\n } catch (err) {\n console.error(`${this.prefixLogs} Health check failed:`, err)\n const body = {\n status: 'error',\n lastCheckAt: null,\n resources: {},\n isStale: true,\n error:\n 'Redis unavailable, unable to read health status of other resources',\n config: this.healthConfig,\n }\n this._log503Failure(req, body)\n res.status(503).json(body)\n }\n }\n }\n\n async cleanup() {\n for (const [, { pool }] of this._databasePools) {\n try {\n await closePool(pool)\n } catch (err) {\n console.error(`${this.prefixLogs} Error closing database pool:`, err)\n }\n }\n this._databasePools.clear()\n }\n}\n\nmodule.exports = { HealthCheckClient, DEFAULT_HEALTH_CONFIG }\n"],"mappings":";;AAAA,MAAM;EACJA,kBAAkB;EAClBC,cAAc;EACdC;AACF,CAAC,GAAGC,OAAO,CAAC,mBAAmB,CAAC;AAChC,MAAM;EACJC,kBAAkB;EAClBC,QAAQ;EACRC,OAAO;EACPC;AACF,CAAC,GAAGJ,OAAO,CAAC,eAAe,CAAC;AAC5B,MAAM;EAAEK;AAAiB,CAAC,GAAGL,OAAO,CAAC,oBAAoB,CAAC;;AAE1D;AACA;AACA;AACA;AACA,SAASM,aAAaA,CAACC,IAAI,EAAE;EAC3B,MAAMC,GAAG,GAAGC,OAAO,CAACC,GAAG,CAACH,IAAI,CAAC;EAC7B,IAAIC,GAAG,IAAI,IAAI,IAAIA,GAAG,KAAK,EAAE,EAAE,OAAOG,SAAS;EAC/C,MAAMC,GAAG,GAAGC,MAAM,CAACL,GAAG,CAAC;EACvB,OAAOK,MAAM,CAACC,QAAQ,CAACF,GAAG,CAAC,GAAGA,GAAG,GAAGD,SAAS;AAC/C;;AAEA;AACA,MAAMI,qBAAqB,GAAG;EAC5BC,eAAe,EAAE,MAAM;EACvBC,gBAAgB,EAAE,OAAO;EACzBC,cAAc,EAAE,MAAM;EACtBC,qBAAqB,EACnBb,aAAa,CAAC,kCAAkC,CAAC,IAAI;AACzD,CAAC;AAED,MAAMc,kBAAkB,GAAG,CACzB;EACEC,OAAO,EACL,6FAA6F;EAC/FC,WAAW,EAAE;AACf,CAAC,EACD;EACED,OAAO,EAAE,uCAAuC;EAChDC,WAAW,EAAE;AACf,CAAC,EACD;EACED,OAAO,EACL,4FAA4F;EAC9FC,WAAW,EAAE;AACf,CAAC,EACD;EACED,OAAO,EACL,8EAA8E;EAChFC,WAAW,EAAE;AACf,CAAC,EACD;EACED,OAAO,EAAE,kDAAkD;EAC3DC,WAAW,EAAE;AACf,CAAC,EACD;EACED,OAAO,EAAE,uDAAuD;EAChEC,WAAW,EAAE;AACf,CAAC,CACF;;AAED;AACA;AACA;AACA;AACA,SAASC,iBAAiBA,CAACC,IAAI,EAAE;EAC/B,IAAI,CAACA,IAAI,IAAI,OAAOA,IAAI,KAAK,QAAQ,EAAE,OAAOA,IAAI;EAClD,IAAIC,MAAM,GAAGD,IAAI;EACjB,KAAK,MAAM;IAAEH,OAAO;IAAEC;EAAY,CAAC,IAAIF,kBAAkB,EAAE;IACzDK,MAAM,GAAGA,MAAM,CAACC,OAAO,CAACL,OAAO,EAAEC,WAAW,CAAC;EAC/C;EACA,OAAOG,MAAM;AACf;;AAEA;AACA;AACA;AACA,MAAME,iBAAiB,CAAC;EACtB;AACF;AACA;AACA;AACA;AACA;AACA;EACEC,WAAWA,CAACC,OAAO,GAAG,CAAC,CAAC,EAAE;IACxB,IAAI,CAACC,YAAY,GAAG;MAAE,GAAGf,qBAAqB;MAAE,GAAGc,OAAO,CAACE;IAAO,CAAC;IACnE,IAAI,CAACC,OAAO,GACVH,OAAO,CAACG,OAAO,IAAIvB,OAAO,CAACC,GAAG,CAACuB,cAAc,IAAI,aAAa;IAEhE,IAAI,CAACC,UAAU,GAAG,IAAI,IAAI,CAACF,OAAO,iBAAiB;IAEnD,IAAI,CAACG,eAAe,GAAG,IAAI;IAC3B;IACA,IAAI,CAACC,cAAc,GAAG,IAAIC,GAAG,CAAC,CAAC;;IAE/B;IACA,IAAI,CAACC,UAAU,GAAGT,OAAO,CAACU,SAAS,IAAI,EAAE;IAEzC,MAAMC,WAAW,GAAG,IAAI,CAACC,uBAAuB,CAAC,CAAC;IAClD,IAAID,WAAW,EAAE;MACf,IAAI,CAACE,gBAAgB,GAAGzC,kBAAkB,CAACuC,WAAW,CAAC;IACzD;IAEA,IAAI,CAACG,MAAM,GAAG,IAAItC,gBAAgB,CAAC;MACjCmC,WAAW,EAAEA,WAAW,IAAI,IAAI;MAChCI,QAAQ,EAAEf,OAAO,CAACe,QAAQ;MAC1BZ,OAAO,EAAE,IAAI,CAACA,OAAO;MACrBf,gBAAgB,EAAE,IAAI,CAACa,YAAY,CAACb;IACtC,CAAC,CAAC;EACJ;EAEA4B,OAAOA,CAACC,QAAQ,EAAE;IAChB,OAAOA,QAAQ,CAACpC,GAAG,IAAIoC,QAAQ,CAACvC,IAAI;EACtC;EAEAkC,uBAAuBA,CAAA,EAAG;IACxB,MAAMM,aAAa,GAAG,IAAI,CAACT,UAAU,CAACU,IAAI,CAACC,CAAC,IAAI,QAAQ,IAAIA,CAAC,IAAIA,CAAC,CAACC,MAAM,CAAC;IAC1E,OAAOH,aAAa,EAAEG,MAAM,IAAI,IAAI;EACtC;EAEAC,QAAQA,CAACzC,GAAG,EAAE0C,GAAG,EAAE;IACjB,IAAI,CAAC,IAAI,CAAChB,cAAc,CAACiB,GAAG,CAAC3C,GAAG,CAAC,EAAE;MACjC,MAAM;QAAE4C,IAAI;QAAEC;MAAK,CAAC,GAAG1D,kBAAkB,CACvCa,GAAG,EACH0C,GAAG,EACH,IAAI,CAACtB,YAAY,CAACZ,cACpB,CAAC;MACD,IAAI,CAACkB,cAAc,CAACoB,GAAG,CAAC9C,GAAG,EAAE;QAAE4C,IAAI;QAAEC;MAAK,CAAC,CAAC;IAC9C;IACA,OAAO,IAAI,CAACnB,cAAc,CAACqB,GAAG,CAAC/C,GAAG,CAAC;EACrC;EAEA,MAAMgD,cAAcA,CAACZ,QAAQ,EAAE;IAC7B,MAAMpC,GAAG,GAAG,IAAI,CAACmC,OAAO,CAACC,QAAQ,CAAC;IAClC,MAAMM,GAAG,GAAG,KAAK,IAAIN,QAAQ,GAAGA,QAAQ,CAACM,GAAG,GAAG3C,OAAO,CAACC,GAAG,CAACA,GAAG,CAAC;IAC/D,IAAI,CAAC0C,GAAG,EAAE;MACR,OAAO;QAAEO,MAAM,EAAE,OAAO;QAAEC,KAAK,EAAE,OAAOlD,GAAG;MAAW,CAAC;IACzD;IAEA,IAAI;MACF,MAAM;QAAE4C,IAAI;QAAEC;MAAK,CAAC,GAAG,IAAI,CAACJ,QAAQ,CAACzC,GAAG,EAAE0C,GAAG,CAAC;MAC9C,MAAMS,OAAO,GAAG,MAAM/D,cAAc,CAACwD,IAAI,EAAEC,IAAI,CAAC;MAEhD,MAAMO,UAAU,GAAG,IAAI,CAAChC,YAAY,CAACX,qBAAqB;MAC1D,IACE,OAAO2C,UAAU,KAAK,QAAQ,IAC9BjD,MAAM,CAACC,QAAQ,CAACgD,UAAU,CAAC,IAC3BD,OAAO,EAAEE,SAAS,IAAI,IAAI,IAC1BF,OAAO,CAACE,SAAS,GAAGD,UAAU,EAC9B;QACA,OAAO;UACLH,MAAM,EAAE,OAAO;UACfC,KAAK,EAAE,sBAAsBC,OAAO,CAACE,SAAS,cAAcD,UAAU,IAAI;UAC1EC,SAAS,EAAEF,OAAO,CAACE;QACrB,CAAC;MACH;MAEA,OAAO;QAAEJ,MAAM,EAAE,IAAI;QAAEI,SAAS,EAAEF,OAAO,CAACE;MAAU,CAAC;IACvD,CAAC,CAAC,OAAOC,GAAG,EAAE;MACZ,MAAMH,OAAO,GAAGG,GAAG,EAAEC,kBAAkB;MACvC,OAAO;QACLN,MAAM,EAAE,OAAO;QACfC,KAAK,EAAErC,iBAAiB,CAACyC,GAAG,CAACE,OAAO,CAAC;QACrC,IAAIL,OAAO,IAAI,OAAOA,OAAO,KAAK,QAAQ,GACtC;UAAEE,SAAS,EAAEF,OAAO,CAACE;QAAU,CAAC,GAChC,CAAC,CAAC;MACR,CAAC;IACH;EACF;EAEA,MAAMI,WAAWA,CAACrB,QAAQ,EAAE;IAC1B,MAAM;MAAEI;IAAO,CAAC,GAAGJ,QAAQ;IAC3B,IAAI,CAACI,MAAM,EAAE,OAAO;MAAES,MAAM,EAAE;IAAK,CAAC;IAEpC,IAAI;MACF,IAAIS,IAAI;MACR,IAAI,IAAI,CAAC1B,gBAAgB,KAAKtC,QAAQ,EAAE;QACtCgE,IAAI,GAAG,MAAM,IAAIC,OAAO,CAAC,CAACC,OAAO,EAAEC,MAAM,KAAK;UAC5CrB,MAAM,CAACsB,IAAI,CAAC,CAACR,GAAG,EAAES,MAAM,KAAK;YAC3B,IAAIT,GAAG,EAAEO,MAAM,CAACP,GAAG,CAAC,MACfM,OAAO,CAACG,MAAM,CAAC;UACtB,CAAC,CAAC;QACJ,CAAC,CAAC;MACJ,CAAC,MAAM,IACL,IAAI,CAAC/B,gBAAgB,KAAKxC,QAAQ,IAClC,IAAI,CAACwC,gBAAgB,KAAKvC,OAAO,EACjC;QACAiE,IAAI,GAAG,MAAMlB,MAAM,CAACsB,IAAI,CAAC,CAAC;MAC5B,CAAC,MAAM;QACL,OAAO;UAAEb,MAAM,EAAE,OAAO;UAAEC,KAAK,EAAE;QAA4B,CAAC;MAChE;MAEA,OAAOQ,IAAI,KAAK,MAAM,GAClB;QAAET,MAAM,EAAE;MAAK,CAAC,GAChB;QAAEA,MAAM,EAAE,OAAO;QAAEC,KAAK,EAAE,eAAeQ,IAAI;MAAG,CAAC;IACvD,CAAC,CAAC,OAAOJ,GAAG,EAAE;MACZ,OAAO;QAAEL,MAAM,EAAE,OAAO;QAAEC,KAAK,EAAErC,iBAAiB,CAACyC,GAAG,CAACE,OAAO;MAAE,CAAC;IACnE;EACF;EAEA,MAAMQ,2BAA2BA,CAAA,EAAG;IAClC,MAAMnC,SAAS,GAAG,CAAC,CAAC;IAEpB,KAAK,MAAMO,QAAQ,IAAI,IAAI,CAACR,UAAU,EAAE;MACtC,MAAM5B,GAAG,GAAG,IAAI,CAACmC,OAAO,CAACC,QAAQ,CAAC;MAElC,IAAI,QAAQ,IAAIA,QAAQ,IAAIA,QAAQ,CAACI,MAAM,EAAE;QAC3CX,SAAS,CAAC7B,GAAG,CAAC,GAAG,MAAM,IAAI,CAACyD,WAAW,CAACrB,QAAQ,CAAC;MACnD,CAAC,MAAM;QACLP,SAAS,CAAC7B,GAAG,CAAC,GAAG,MAAM,IAAI,CAACgD,cAAc,CAACZ,QAAQ,CAAC;MACtD;IACF;IAEA,MAAM6B,eAAe,GAAGC,MAAM,CAACC,IAAI,CAACtC,SAAS,CAAC,CAC3CuC,IAAI,CAAC,CAAC,CACNC,MAAM,CAAC,CAACC,GAAG,EAAEC,GAAG,KAAK;MACpBD,GAAG,CAACC,GAAG,CAAC,GAAG1C,SAAS,CAAC0C,GAAG,CAAC;MACzB,OAAOD,GAAG;IACZ,CAAC,EAAE,CAAC,CAAC,CAAC;IAER,MAAME,QAAQ,GAAGN,MAAM,CAACO,MAAM,CAAC5C,SAAS,CAAC,CAAC6C,IAAI,CAACnC,CAAC,IAAIA,CAAC,CAACU,MAAM,KAAK,OAAO,CAAC;IACzE,MAAM0B,WAAW,GAAGC,IAAI,CAACC,GAAG,CAAC,CAAC;IAE9B,OAAO;MACL5B,MAAM,EAAEuB,QAAQ,GAAG,OAAO,GAAG,IAAI;MACjCG,WAAW;MACX9C,SAAS,EAAEoC,eAAe;MAC1Ba,OAAO,EAAE,KAAK;MACdzD,MAAM,EAAE,IAAI,CAACD;IACf,CAAC;EACH;EAEA2D,aAAaA,CAAChB,MAAM,EAAEiB,MAAM,GAAG,KAAK,EAAE;IACpC,MAAMF,OAAO,GACX,CAACf,MAAM,CAACY,WAAW,IACnBC,IAAI,CAACC,GAAG,CAAC,CAAC,GAAGd,MAAM,CAACY,WAAW,GAAG,IAAI,CAACvD,YAAY,CAACb,gBAAgB;IAEtE,OAAO;MACL,GAAGwD,MAAM;MACTe,OAAO;MACP7B,MAAM,EAAE6B,OAAO,GAAG,OAAO,GAAGf,MAAM,CAACd,MAAM;MACzC,IAAI6B,OAAO,IAAI;QACb5B,KAAK,EACH;MACJ,CAAC,CAAC;MACF,IAAI8B,MAAM,IAAI;QAAEA,MAAM,EAAE;MAAK,CAAC;IAChC,CAAC;EACH;EAEA,MAAMC,kBAAkBA,CAAA,EAAG;IACzB,IAAI,IAAI,CAACxD,eAAe,EAAE;MACxB,OAAO,IAAI,CAACA,eAAe;IAC7B;IAEA,IAAI,CAACA,eAAe,GAAG,IAAI,CAACuC,2BAA2B,CAAC,CAAC,CACtDkB,IAAI,CAACnB,MAAM,IAAI;MACd,IAAI,CAACtC,eAAe,GAAG,IAAI;MAC3B,OAAO,IAAI,CAACsD,aAAa,CAAChB,MAAM,CAAC;IACnC,CAAC,CAAC,CACDoB,KAAK,CAAC7B,GAAG,IAAI;MACZ,IAAI,CAAC7B,eAAe,GAAG,IAAI;MAC3B,MAAM6B,GAAG;IACX,CAAC,CAAC;IAEJ,OAAO,IAAI,CAAC7B,eAAe;EAC7B;EAEA,MAAM2D,eAAeA,CAAA,EAAG;IACtB,IAAI;MACF,MAAMJ,MAAM,GAAG,MAAM,IAAI,CAAC/C,MAAM,CAACc,GAAG,CAAC,CAAC;MACtC,IAAIiC,MAAM,EAAE,OAAO,IAAI,CAACD,aAAa,CAACC,MAAM,CAAC;MAC7C,OAAO,IAAI;IACb,CAAC,CAAC,OAAO1B,GAAG,EAAE;MACZ+B,OAAO,CAACnC,KAAK,CAAC,GAAG,IAAI,CAAC1B,UAAU,6BAA6B,EAAE8B,GAAG,CAAC;MACnE,OAAO;QACLL,MAAM,EAAE,OAAO;QACf0B,WAAW,EAAE,IAAI;QACjB9C,SAAS,EAAE,CAAC,CAAC;QACbiD,OAAO,EAAE,IAAI;QACb5B,KAAK,EACH,oEAAoE;QACtE7B,MAAM,EAAE,IAAI,CAACD;MACf,CAAC;IACH;EACF;EAEA,MAAMkE,YAAYA,CAAA,EAAG;IACnB,MAAMvB,MAAM,GAAG,MAAM,IAAI,CAACC,2BAA2B,CAAC,CAAC;IACvD,MAAM,IAAI,CAAC/B,MAAM,CAACa,GAAG,CAACiB,MAAM,CAAC;IAC7B,OAAOA,MAAM;EACf;EAEAwB,UAAUA,CAAA,EAAG;IACX,IAAI,CAAC9D,eAAe,GAAG,IAAI;EAC7B;;EAEA;AACF;AACA;AACA;AACA;AACA;AACA;EACE+D,cAAcA,CAACC,GAAG,EAAEC,OAAO,EAAE;IAC3B,MAAMC,SAAS,GACZF,GAAG,IAAIA,GAAG,CAACG,OAAO,IAAIH,GAAG,CAACG,OAAO,CAAC,cAAc,CAAC,IACjDH,GAAG,IAAIA,GAAG,CAACI,EAAG,IACf,IAAI;IACN,MAAMC,IAAI,GAAG;MACXC,GAAG,EAAE,iBAAiB;MACtBzE,OAAO,EAAE,IAAI,CAACA,OAAO;MACrB0E,UAAU,EAAEL,SAAS;MACrBzC,KAAK,EAAEwC,OAAO,CAACxC,KAAK;MACpB4B,OAAO,EAAEY,OAAO,CAACZ,OAAO;MACxB7B,MAAM,EAAEyC,OAAO,CAACzC,MAAM;MACtBgD,cAAc,EACZP,OAAO,CAAC7D,SAAS,IACjBqC,MAAM,CAACgC,OAAO,CAACR,OAAO,CAAC7D,SAAS,CAAC,CAC9BsE,MAAM,CAAC,CAAC,GAAG5D,CAAC,CAAC,KAAKA,CAAC,IAAIA,CAAC,CAACU,MAAM,KAAK,OAAO,CAAC,CAC5CmD,GAAG,CAAC,CAAC,CAACpG,GAAG,EAAEuC,CAAC,CAAC,MAAM;QAAEvC,GAAG;QAAEkD,KAAK,EAAEX,CAAC,CAACW;MAAM,CAAC,CAAC;IAClD,CAAC;IACDmC,OAAO,CAACgB,IAAI,CAACC,IAAI,CAACC,SAAS,CAACT,IAAI,CAAC,CAAC;EACpC;EAEAU,aAAaA,CAAA,EAAG;IACd,OAAO,OAAOf,GAAG,EAAEgB,GAAG,KAAK;MACzB,IAAI;QACF,MAAM1C,MAAM,GAAG,MAAM,IAAI,CAACqB,eAAe,CAAC,CAAC;QAE3C,IAAI,CAACrB,MAAM,EAAE;UACX,MAAM2C,IAAI,GAAG;YACXzD,MAAM,EAAE,OAAO;YACf0B,WAAW,EAAE,IAAI;YACjB9C,SAAS,EAAE,CAAC,CAAC;YACbiD,OAAO,EAAE,IAAI;YACb5B,KAAK,EACH,kEAAkE;YACpE7B,MAAM,EAAE,IAAI,CAACD;UACf,CAAC;UACD,IAAI,CAACoE,cAAc,CAACC,GAAG,EAAEiB,IAAI,CAAC;UAC9BD,GAAG,CAACxD,MAAM,CAAC,GAAG,CAAC,CAAC0D,IAAI,CAACD,IAAI,CAAC;UAC1B;QACF;QAEA,MAAME,UAAU,GAAG7C,MAAM,CAACd,MAAM,KAAK,IAAI,GAAG,GAAG,GAAG,GAAG;QACrD,IAAI2D,UAAU,KAAK,GAAG,EAAE;UACtB,IAAI,CAACpB,cAAc,CAACC,GAAG,EAAE1B,MAAM,CAAC;QAClC;QACA0C,GAAG,CAACxD,MAAM,CAAC2D,UAAU,CAAC,CAACD,IAAI,CAAC5C,MAAM,CAAC;MACrC,CAAC,CAAC,OAAOT,GAAG,EAAE;QACZ+B,OAAO,CAACnC,KAAK,CAAC,GAAG,IAAI,CAAC1B,UAAU,uBAAuB,EAAE8B,GAAG,CAAC;QAC7D,MAAMoD,IAAI,GAAG;UACXzD,MAAM,EAAE,OAAO;UACf0B,WAAW,EAAE,IAAI;UACjB9C,SAAS,EAAE,CAAC,CAAC;UACbiD,OAAO,EAAE,IAAI;UACb5B,KAAK,EACH,oEAAoE;UACtE7B,MAAM,EAAE,IAAI,CAACD;QACf,CAAC;QACD,IAAI,CAACoE,cAAc,CAACC,GAAG,EAAEiB,IAAI,CAAC;QAC9BD,GAAG,CAACxD,MAAM,CAAC,GAAG,CAAC,CAAC0D,IAAI,CAACD,IAAI,CAAC;MAC5B;IACF,CAAC;EACH;EAEA,MAAMG,OAAOA,CAAA,EAAG;IACd,KAAK,MAAM,GAAG;MAAEjE;IAAK,CAAC,CAAC,IAAI,IAAI,CAAClB,cAAc,EAAE;MAC9C,IAAI;QACF,MAAMrC,SAAS,CAACuD,IAAI,CAAC;MACvB,CAAC,CAAC,OAAOU,GAAG,EAAE;QACZ+B,OAAO,CAACnC,KAAK,CAAC,GAAG,IAAI,CAAC1B,UAAU,+BAA+B,EAAE8B,GAAG,CAAC;MACvE;IACF;IACA,IAAI,CAAC5B,cAAc,CAACoF,KAAK,CAAC,CAAC;EAC7B;AACF;AAEAC,MAAM,CAACC,OAAO,GAAG;EAAE/F,iBAAiB;EAAEZ;AAAsB,CAAC","ignoreList":[]}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"healthCheckWorker.d.ts","sourceRoot":"","sources":["../../src/health/healthCheckWorker.js"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"healthCheckWorker.d.ts","sourceRoot":"","sources":["../../src/health/healthCheckWorker.js"],"names":[],"mappings":"AA2BA,2EA4DC"}
|
|
@@ -19,6 +19,18 @@ const {
|
|
|
19
19
|
const {
|
|
20
20
|
DEFAULT_HEALTH_CONFIG
|
|
21
21
|
} = require('./healthCheckClient');
|
|
22
|
+
|
|
23
|
+
/** Latency above this (ms) triggers a warning log on the worker. 2000 = 2 seconds. */
|
|
24
|
+
const LATENCY_WARNING_MS = 2000;
|
|
25
|
+
function logLatencyWarnings(prefixLogs, result) {
|
|
26
|
+
const resources = result?.resources || {};
|
|
27
|
+
for (const [env, r] of Object.entries(resources)) {
|
|
28
|
+
const connectMs = r?.connectMs;
|
|
29
|
+
if (typeof connectMs === 'number' && connectMs > LATENCY_WARNING_MS) {
|
|
30
|
+
console.warn(`${prefixLogs} DB connect latency high: ${env} connectMs=${connectMs}ms (threshold ${LATENCY_WARNING_MS}ms)`);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|
|
22
34
|
function createHealthCheckWorker(options) {
|
|
23
35
|
const {
|
|
24
36
|
refreshIntervalMs = DEFAULT_HEALTH_CONFIG.checkIntervalMs,
|
|
@@ -34,7 +46,8 @@ function createHealthCheckWorker(options) {
|
|
|
34
46
|
console.log(`${prefixLogs} Starting health check worker...`);
|
|
35
47
|
console.log(`${prefixLogs} Refresh interval: ${refreshIntervalMs}ms`);
|
|
36
48
|
try {
|
|
37
|
-
await healthCheckClient.refreshCache();
|
|
49
|
+
const result = await healthCheckClient.refreshCache();
|
|
50
|
+
logLatencyWarnings(prefixLogs, result);
|
|
38
51
|
if (logValues) {
|
|
39
52
|
console.log(`${prefixLogs} Initial health check completed`);
|
|
40
53
|
}
|
|
@@ -44,6 +57,7 @@ function createHealthCheckWorker(options) {
|
|
|
44
57
|
const interval = setInterval(async () => {
|
|
45
58
|
try {
|
|
46
59
|
const result = await healthCheckClient.refreshCache();
|
|
60
|
+
logLatencyWarnings(prefixLogs, result);
|
|
47
61
|
if (logValues) {
|
|
48
62
|
console.log(`${prefixLogs} Health check refreshed at ${result.lastCheckAt}, status: ${result.status}`);
|
|
49
63
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"healthCheckWorker.js","names":["createHealthCheckWorkerClient","require","DEFAULT_HEALTH_CONFIG","createHealthCheckWorker","options","refreshIntervalMs","checkIntervalMs","workerClientOptions","appName","process","
|
|
1
|
+
{"version":3,"file":"healthCheckWorker.js","names":["createHealthCheckWorkerClient","require","DEFAULT_HEALTH_CONFIG","LATENCY_WARNING_MS","logLatencyWarnings","prefixLogs","result","resources","env","r","Object","entries","connectMs","console","warn","createHealthCheckWorker","options","refreshIntervalMs","checkIntervalMs","workerClientOptions","appName","process","BUILD_APP_NAME","dynoId","HOSTNAME","processType","BUILD_DYNO_PROCESS_TYPE","logValues","HEALTH_LOG_VALUES","healthCheckClient","runHealthCheckWorker","log","refreshCache","err","error","interval","setInterval","lastCheckAt","status","on","clearInterval","cleanup","finally","exit"],"sources":["../../src/health/healthCheckWorker.js"],"sourcesContent":["/**\n * @param {Object} options\n * @param {{ name: string } | { name: string, url?: string } | { name: string, client: any }}[] options.resources - Must include Redis resource with client\n * @param {string} [options.appName]\n * @param {number} [options.refreshIntervalMs]\n * @param {string} [options.databaseUrl]\n * @param {string} [options.databaseName]\n * @param {Object<string, string>} [options.additionalDatabaseUrls]\n */\nconst { createHealthCheckWorkerClient } = require('./healthCheckUtils')\nconst { DEFAULT_HEALTH_CONFIG } = require('./healthCheckClient')\n\n/** Latency above this (ms) triggers a warning log on the worker. 2000 = 2 seconds. */\nconst LATENCY_WARNING_MS = 2000\n\nfunction logLatencyWarnings(prefixLogs, result) {\n const resources = result?.resources || {}\n for (const [env, r] of Object.entries(resources)) {\n const connectMs = r?.connectMs\n if (typeof connectMs === 'number' && connectMs > LATENCY_WARNING_MS) {\n console.warn(\n `${prefixLogs} DB connect latency high: ${env} connectMs=${connectMs}ms (threshold ${LATENCY_WARNING_MS}ms)`\n )\n }\n }\n}\n\nexport function createHealthCheckWorker(options) {\n const {\n refreshIntervalMs = DEFAULT_HEALTH_CONFIG.checkIntervalMs,\n ...workerClientOptions\n } = options\n\n const appName =\n workerClientOptions.appName || process.env.BUILD_APP_NAME || 'unknown-app'\n const dynoId = process.env.HOSTNAME || 'unknown-dyno'\n const processType =\n process.env.BUILD_DYNO_PROCESS_TYPE || 'health-check-worker'\n const logValues = process.env.HEALTH_LOG_VALUES === 'true'\n const prefixLogs = `[${processType}] [${appName}] [${dynoId}] [HealthCheck]`\n\n const healthCheckClient = createHealthCheckWorkerClient(workerClientOptions)\n\n return async function runHealthCheckWorker() {\n console.log(`${prefixLogs} Starting health check worker...`)\n console.log(`${prefixLogs} Refresh interval: ${refreshIntervalMs}ms`)\n\n try {\n const result = await healthCheckClient.refreshCache()\n logLatencyWarnings(prefixLogs, result)\n if (logValues) {\n console.log(`${prefixLogs} Initial health check completed`)\n }\n } catch (err) {\n console.error(`${prefixLogs} Initial health check failed:`, err)\n }\n\n const interval = setInterval(async () => {\n try {\n const result = await healthCheckClient.refreshCache()\n logLatencyWarnings(prefixLogs, result)\n if (logValues) {\n console.log(\n `${prefixLogs} Health check refreshed at ${result.lastCheckAt}, status: ${result.status}`\n )\n }\n } catch (err) {\n console.error(`${prefixLogs} Health check refresh failed:`, err)\n }\n }, refreshIntervalMs)\n\n process.on('SIGTERM', () => {\n console.log(`${prefixLogs} Received SIGTERM, shutting down...`)\n clearInterval(interval)\n healthCheckClient.cleanup().finally(() => {\n process.exit(0)\n })\n })\n\n process.on('SIGINT', () => {\n console.log(`${prefixLogs} Received SIGINT, shutting down...`)\n clearInterval(interval)\n healthCheckClient.cleanup().finally(() => {\n process.exit(0)\n })\n })\n }\n}\n"],"mappings":";;;;;;AAAA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAM;EAAEA;AAA8B,CAAC,GAAGC,OAAO,CAAC,oBAAoB,CAAC;AACvE,MAAM;EAAEC;AAAsB,CAAC,GAAGD,OAAO,CAAC,qBAAqB,CAAC;;AAEhE;AACA,MAAME,kBAAkB,GAAG,IAAI;AAE/B,SAASC,kBAAkBA,CAACC,UAAU,EAAEC,MAAM,EAAE;EAC9C,MAAMC,SAAS,GAAGD,MAAM,EAAEC,SAAS,IAAI,CAAC,CAAC;EACzC,KAAK,MAAM,CAACC,GAAG,EAAEC,CAAC,CAAC,IAAIC,MAAM,CAACC,OAAO,CAACJ,SAAS,CAAC,EAAE;IAChD,MAAMK,SAAS,GAAGH,CAAC,EAAEG,SAAS;IAC9B,IAAI,OAAOA,SAAS,KAAK,QAAQ,IAAIA,SAAS,GAAGT,kBAAkB,EAAE;MACnEU,OAAO,CAACC,IAAI,CACV,GAAGT,UAAU,6BAA6BG,GAAG,cAAcI,SAAS,iBAAiBT,kBAAkB,KACzG,CAAC;IACH;EACF;AACF;AAEO,SAASY,uBAAuBA,CAACC,OAAO,EAAE;EAC/C,MAAM;IACJC,iBAAiB,GAAGf,qBAAqB,CAACgB,eAAe;IACzD,GAAGC;EACL,CAAC,GAAGH,OAAO;EAEX,MAAMI,OAAO,GACXD,mBAAmB,CAACC,OAAO,IAAIC,OAAO,CAACb,GAAG,CAACc,cAAc,IAAI,aAAa;EAC5E,MAAMC,MAAM,GAAGF,OAAO,CAACb,GAAG,CAACgB,QAAQ,IAAI,cAAc;EACrD,MAAMC,WAAW,GACfJ,OAAO,CAACb,GAAG,CAACkB,uBAAuB,IAAI,qBAAqB;EAC9D,MAAMC,SAAS,GAAGN,OAAO,CAACb,GAAG,CAACoB,iBAAiB,KAAK,MAAM;EAC1D,MAAMvB,UAAU,GAAG,IAAIoB,WAAW,MAAML,OAAO,MAAMG,MAAM,iBAAiB;EAE5E,MAAMM,iBAAiB,GAAG7B,6BAA6B,CAACmB,mBAAmB,CAAC;EAE5E,OAAO,eAAeW,oBAAoBA,CAAA,EAAG;IAC3CjB,OAAO,CAACkB,GAAG,CAAC,GAAG1B,UAAU,kCAAkC,CAAC;IAC5DQ,OAAO,CAACkB,GAAG,CAAC,GAAG1B,UAAU,sBAAsBY,iBAAiB,IAAI,CAAC;IAErE,IAAI;MACF,MAAMX,MAAM,GAAG,MAAMuB,iBAAiB,CAACG,YAAY,CAAC,CAAC;MACrD5B,kBAAkB,CAACC,UAAU,EAAEC,MAAM,CAAC;MACtC,IAAIqB,SAAS,EAAE;QACbd,OAAO,CAACkB,GAAG,CAAC,GAAG1B,UAAU,iCAAiC,CAAC;MAC7D;IACF,CAAC,CAAC,OAAO4B,GAAG,EAAE;MACZpB,OAAO,CAACqB,KAAK,CAAC,GAAG7B,UAAU,+BAA+B,EAAE4B,GAAG,CAAC;IAClE;IAEA,MAAME,QAAQ,GAAGC,WAAW,CAAC,YAAY;MACvC,IAAI;QACF,MAAM9B,MAAM,GAAG,MAAMuB,iBAAiB,CAACG,YAAY,CAAC,CAAC;QACrD5B,kBAAkB,CAACC,UAAU,EAAEC,MAAM,CAAC;QACtC,IAAIqB,SAAS,EAAE;UACbd,OAAO,CAACkB,GAAG,CACT,GAAG1B,UAAU,8BAA8BC,MAAM,CAAC+B,WAAW,aAAa/B,MAAM,CAACgC,MAAM,EACzF,CAAC;QACH;MACF,CAAC,CAAC,OAAOL,GAAG,EAAE;QACZpB,OAAO,CAACqB,KAAK,CAAC,GAAG7B,UAAU,+BAA+B,EAAE4B,GAAG,CAAC;MAClE;IACF,CAAC,EAAEhB,iBAAiB,CAAC;IAErBI,OAAO,CAACkB,EAAE,CAAC,SAAS,EAAE,MAAM;MAC1B1B,OAAO,CAACkB,GAAG,CAAC,GAAG1B,UAAU,qCAAqC,CAAC;MAC/DmC,aAAa,CAACL,QAAQ,CAAC;MACvBN,iBAAiB,CAACY,OAAO,CAAC,CAAC,CAACC,OAAO,CAAC,MAAM;QACxCrB,OAAO,CAACsB,IAAI,CAAC,CAAC,CAAC;MACjB,CAAC,CAAC;IACJ,CAAC,CAAC;IAEFtB,OAAO,CAACkB,EAAE,CAAC,QAAQ,EAAE,MAAM;MACzB1B,OAAO,CAACkB,GAAG,CAAC,GAAG1B,UAAU,oCAAoC,CAAC;MAC9DmC,aAAa,CAACL,QAAQ,CAAC;MACvBN,iBAAiB,CAACY,OAAO,CAAC,CAAC,CAACC,OAAO,CAAC,MAAM;QACxCrB,OAAO,CAACsB,IAAI,CAAC,CAAC,CAAC;MACjB,CAAC,CAAC;IACJ,CAAC,CAAC;EACJ,CAAC;AACH","ignoreList":[]}
|
|
@@ -17,6 +17,7 @@ export class BaseMetricsClient {
|
|
|
17
17
|
* @param {boolean} [config.removeOldMetrics] Enable to clear metrics by service name
|
|
18
18
|
* @param {function} [config.startupValidation] Add to validate on start push.
|
|
19
19
|
* @param {boolean} [config.disablePushgateway] Disable pushing to Pushgateway (use HTTP scraping instead)
|
|
20
|
+
* @param {boolean} [config.includeNodeDefaultMetrics] Register prom-client default process metrics (heap, etc.); defaults true unless MetricsClient forces false for web+Redis HTTP recording dynos.
|
|
20
21
|
*/
|
|
21
22
|
constructor(config?: {
|
|
22
23
|
appName?: string | undefined;
|
|
@@ -30,10 +31,12 @@ export class BaseMetricsClient {
|
|
|
30
31
|
removeOldMetrics?: boolean | undefined;
|
|
31
32
|
startupValidation?: Function | undefined;
|
|
32
33
|
disablePushgateway?: boolean | undefined;
|
|
34
|
+
includeNodeDefaultMetrics?: boolean | undefined;
|
|
33
35
|
});
|
|
34
36
|
appName: string;
|
|
35
37
|
dynoId: string;
|
|
36
38
|
processType: string;
|
|
39
|
+
includeNodeDefaultMetrics: boolean;
|
|
37
40
|
enabled: boolean;
|
|
38
41
|
logValues: boolean;
|
|
39
42
|
pushgatewayUrl: string;
|
|
@@ -49,6 +52,11 @@ export class BaseMetricsClient {
|
|
|
49
52
|
dyno_id: string;
|
|
50
53
|
process_type: string;
|
|
51
54
|
};
|
|
55
|
+
/** Default labels without dyno_id (for HTTP metrics so dyno_id is not included). */
|
|
56
|
+
defaultLabelsWithoutDynoId: {
|
|
57
|
+
app: string;
|
|
58
|
+
process_type: string;
|
|
59
|
+
};
|
|
52
60
|
gateway: any;
|
|
53
61
|
gauges: {};
|
|
54
62
|
counters: {};
|
|
@@ -80,15 +88,17 @@ export class BaseMetricsClient {
|
|
|
80
88
|
* @param {string} params.name - Metric name
|
|
81
89
|
* @param {string} params.help - Metric description
|
|
82
90
|
* @param {string[]} [params.labelNames] - Optional list of label names. Defaults to this.defaultLabels keys.
|
|
91
|
+
* @param {boolean} [params.useLabelsWithoutDynoId=false] - If true, counter uses app/process_type only (no dyno_id). Use for HTTP metrics.
|
|
83
92
|
*
|
|
84
93
|
* @returns {(labels?: Object, incrementValue?: number) => void}
|
|
85
94
|
* A function to increment the counter.
|
|
86
95
|
* Usage: (labels?, incrementValue?)
|
|
87
96
|
*/
|
|
88
|
-
createCounter({ name, help, labelNames }: {
|
|
97
|
+
createCounter({ name, help, labelNames, useLabelsWithoutDynoId, }: {
|
|
89
98
|
name: string;
|
|
90
99
|
help: string;
|
|
91
100
|
labelNames?: string[] | undefined;
|
|
101
|
+
useLabelsWithoutDynoId?: boolean | undefined;
|
|
92
102
|
}): (labels?: Object, incrementValue?: number) => void;
|
|
93
103
|
/**
|
|
94
104
|
* Clear all collected counters
|
|
@@ -158,6 +168,14 @@ export class BaseMetricsClient {
|
|
|
158
168
|
* @returns {string[]} Combined label names
|
|
159
169
|
*/
|
|
160
170
|
withDefaultLabels: (labels?: string[]) => string[];
|
|
171
|
+
/**
|
|
172
|
+
* Merge default labels without dyno_id (`app`, `process_type`) with custom label names.
|
|
173
|
+
* Use for HTTP metrics so dyno_id is not included.
|
|
174
|
+
*
|
|
175
|
+
* @param {string[]} labels Additional label names
|
|
176
|
+
* @returns {string[]} Combined label names
|
|
177
|
+
*/
|
|
178
|
+
withDefaultLabelsWithoutDynoId: (labels?: string[]) => string[];
|
|
161
179
|
getDefaultLabels: (labels?: any[]) => {
|
|
162
180
|
app: string;
|
|
163
181
|
dyno_id: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"baseMetricsClient.d.ts","sourceRoot":"","sources":["../../src/metrics/baseMetricsClient.js"],"names":[],"mappings":"AAKA;;;;GAIG;AACH;IACE
|
|
1
|
+
{"version":3,"file":"baseMetricsClient.d.ts","sourceRoot":"","sources":["../../src/metrics/baseMetricsClient.js"],"names":[],"mappings":"AAKA;;;;GAIG;AACH;IACE;;;;;;;;;;;;;;OAcG;IACH;QAb2B,OAAO;QACP,MAAM;QACN,WAAW;QACV,OAAO;QACP,SAAS;QACV,cAAc;QACd,iBAAiB;QACjB,WAAW;QACV,gBAAgB;QACf,iBAAiB;QAClB,kBAAkB;QAClB,yBAAyB;OA8DpD;IA3DC,gBAA4E;IAC5E,eAAqE;IACrE,oBAG6B;IAC7B,mCAC4C;IAC5C,iBAAuE;IACvE,mBAC+D;IAC/D,uBACoE;IACpE,kBAC0E;IAC1E,oBAGI;IACJ,wCAAiD;IACjD,4BAEoD;IACpD,0BAEmD;IAEnD,mBAAyF;IAEzF,uEAAsC;IAKtC;;;;MAIC;IAED,oFAAoF;IACpF;;;MAGC;IAGD,aAAmB;IACnB,WAAgB;IAChB,aAAkB;IAClB,sBAA2B;IAE3B,mEAAmE;IACnE;YADkB,MAAM,SAAc,MAAM,GAAG,QAAQ,MAAM,CAAC;MACvC;IAKvB,sCAAwC;IAG1C;;;;;;;;OAQG;IACH;QAN2B,IAAI,EAApB,MAAM;QACU,IAAI,EAApB,MAAM;QACuC,QAAQ,UAAzC,MAAM,GAAC,QAAQ,MAAM,CAAC;QACf,UAAU;UAC3B,OAAO,aAAa,EAAE,KAAK,CAuBvC;IAED;;;;;;;;;;;;OAYG;IACH;QAT0B,IAAI,EAAnB,MAAM;QACS,IAAI,EAAnB,MAAM;QACY,UAAU;QACX,sBAAsB;kBAE3B,MAAM,mBAAmB,MAAM,KAAK,IAAI,CAgC9D;IAED;;OAEG;IACH,6BAKC;IAED;;OAEG;IACH,kCAiCC;IAED,sEAmDC;IA/CK,+CAA0D;IAiDhE,iCAEC;IAED;;;;;;;;;OASG;IACH,iFAEC;IAED;;;OAGG;IACH,eAFa,QAAQ,IAAI,CAAC,CAOzB;IAED;;;;;;;;;OASG;IACH,yBAEC;IAED;;;OAGG;IACH,qBAFa,QAAQ,IAAI,CAAC,CAezB;IAED;;;OAGG;IACH,8BAkDC;IAED;;;;;OAKG;IACH,8CAFa,QAAQ,IAAI,CAAC,CAgBzB;IAED;;;OAGG;IACH,uBAoDC;IAED;;;;;;OAMG;IACH,6BAHW,MAAM,EAAE,KACN,MAAM,EAAE,CAIpB;IAED;;;;;;OAMG;IACH,0CAHW,MAAM,EAAE,KACN,MAAM,EAAE,CAIpB;IAED;;;;MAEC;IAED,gCAGC;IAID,8BAEC;IAED,gCAEC;IAED,4EAEC;IAED,sCAEC;IAED,2DAWC;CACF"}
|
|
@@ -26,11 +26,13 @@ class BaseMetricsClient {
|
|
|
26
26
|
* @param {boolean} [config.removeOldMetrics] Enable to clear metrics by service name
|
|
27
27
|
* @param {function} [config.startupValidation] Add to validate on start push.
|
|
28
28
|
* @param {boolean} [config.disablePushgateway] Disable pushing to Pushgateway (use HTTP scraping instead)
|
|
29
|
+
* @param {boolean} [config.includeNodeDefaultMetrics] Register prom-client default process metrics (heap, etc.); defaults true unless MetricsClient forces false for web+Redis HTTP recording dynos.
|
|
29
30
|
*/
|
|
30
31
|
constructor(config = {}) {
|
|
31
32
|
this.appName = config.appName || process.env.BUILD_APP_NAME || 'unknown-app';
|
|
32
33
|
this.dynoId = config.dynoId || process.env.HOSTNAME || 'unknown-dyno';
|
|
33
34
|
this.processType = config.processType || process.env.BUILD_DYNO_PROCESS_TYPE || 'undefined_build_dyno_type';
|
|
35
|
+
this.includeNodeDefaultMetrics = config.includeNodeDefaultMetrics !== false;
|
|
34
36
|
this.enabled = config.enabled ?? process.env.METRICS_ENABLED === 'true';
|
|
35
37
|
this.logValues = config.logValues ?? process.env.METRICS_LOG_VALUES === 'true';
|
|
36
38
|
this.pushgatewayUrl = config.pushgatewayUrl || process.env.METRICS_PUSHGATEWAY_URL || '';
|
|
@@ -41,15 +43,23 @@ class BaseMetricsClient {
|
|
|
41
43
|
this.removeOldMetrics = config.removeOldMetrics ?? process.env.METRICS_REMOVE_OLD_METRICS === 'true';
|
|
42
44
|
this.prefixLogs = `[${this.processType}] [${this.appName}] [${this.dynoId}] [Monitoring]`;
|
|
43
45
|
this._registry = new client.Registry();
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
46
|
+
if (this.includeNodeDefaultMetrics) {
|
|
47
|
+
client.collectDefaultMetrics({
|
|
48
|
+
register: this._registry
|
|
49
|
+
});
|
|
50
|
+
}
|
|
47
51
|
this.defaultLabels = {
|
|
48
52
|
app: this.appName,
|
|
49
53
|
dyno_id: this.dynoId,
|
|
50
54
|
process_type: this.processType
|
|
51
55
|
};
|
|
52
56
|
|
|
57
|
+
/** Default labels without dyno_id (for HTTP metrics so dyno_id is not included). */
|
|
58
|
+
this.defaultLabelsWithoutDynoId = {
|
|
59
|
+
app: this.appName,
|
|
60
|
+
process_type: this.processType
|
|
61
|
+
};
|
|
62
|
+
|
|
53
63
|
// Always push to configured URL (VM-agent). No Pushgateway.
|
|
54
64
|
this.gateway = null;
|
|
55
65
|
this.gauges = {};
|
|
@@ -99,6 +109,7 @@ class BaseMetricsClient {
|
|
|
99
109
|
* @param {string} params.name - Metric name
|
|
100
110
|
* @param {string} params.help - Metric description
|
|
101
111
|
* @param {string[]} [params.labelNames] - Optional list of label names. Defaults to this.defaultLabels keys.
|
|
112
|
+
* @param {boolean} [params.useLabelsWithoutDynoId=false] - If true, counter uses app/process_type only (no dyno_id). Use for HTTP metrics.
|
|
102
113
|
*
|
|
103
114
|
* @returns {(labels?: Object, incrementValue?: number) => void}
|
|
104
115
|
* A function to increment the counter.
|
|
@@ -107,9 +118,11 @@ class BaseMetricsClient {
|
|
|
107
118
|
createCounter({
|
|
108
119
|
name,
|
|
109
120
|
help,
|
|
110
|
-
labelNames = Object.keys(this.defaultLabels)
|
|
121
|
+
labelNames = Object.keys(this.defaultLabels),
|
|
122
|
+
useLabelsWithoutDynoId = false
|
|
111
123
|
}) {
|
|
112
124
|
if (this.counters[name]) return this.countersFunctions[name];
|
|
125
|
+
const defaultSet = useLabelsWithoutDynoId ? this.defaultLabelsWithoutDynoId : this.defaultLabels;
|
|
113
126
|
const c = new client.Counter({
|
|
114
127
|
name,
|
|
115
128
|
help,
|
|
@@ -121,7 +134,7 @@ class BaseMetricsClient {
|
|
|
121
134
|
...this.countersFunctions,
|
|
122
135
|
[name]: (data = {}, value = 1) => {
|
|
123
136
|
c.inc({
|
|
124
|
-
...
|
|
137
|
+
...defaultSet,
|
|
125
138
|
...data
|
|
126
139
|
}, value);
|
|
127
140
|
}
|
|
@@ -181,50 +194,38 @@ class BaseMetricsClient {
|
|
|
181
194
|
clearInterval(this._idleInterval);
|
|
182
195
|
this._idleInterval = null;
|
|
183
196
|
}
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
}
|
|
189
|
-
return this.pushMetrics();
|
|
190
|
-
};
|
|
197
|
+
if (this.startupValidation && !this.startupValidation()) {
|
|
198
|
+
return;
|
|
199
|
+
}
|
|
200
|
+
const runPush = () => {
|
|
191
201
|
if (customPushMetics && typeof customPushMetics === 'function') {
|
|
192
|
-
|
|
193
|
-
} else {
|
|
194
|
-
setInterval(() => {
|
|
195
|
-
runPush().catch(err => {
|
|
196
|
-
console.error(`${this.prefixLogs} Failed to push metrics:`, err);
|
|
197
|
-
});
|
|
198
|
-
}, interval * 1000);
|
|
202
|
+
return Promise.resolve(customPushMetics());
|
|
199
203
|
}
|
|
200
|
-
|
|
201
|
-
// First push immediately so metrics appear without waiting for the first interval
|
|
202
|
-
runPush().catch(err => {
|
|
203
|
-
console.error(`${this.prefixLogs} Failed to push metrics (initial):`, err);
|
|
204
|
-
});
|
|
205
|
-
let pushOrigin = 'none';
|
|
206
|
-
try {
|
|
207
|
-
if (this.pushgatewayUrl && this.pushgatewayUrl.trim()) {
|
|
208
|
-
pushOrigin = new URL(this.pushgatewayUrl.trim()).origin;
|
|
209
|
-
}
|
|
210
|
-
} catch {
|
|
211
|
-
pushOrigin = 'invalid URL';
|
|
212
|
-
}
|
|
213
|
-
console.warn(`${this.prefixLogs} Metrics collection started. (interval: ${this.intervalSec}s, push: ${pushOrigin})`);
|
|
204
|
+
return this.pushMetrics();
|
|
214
205
|
};
|
|
215
|
-
if (
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
console.error(`${this.prefixLogs} Startup validation failed:`, err);
|
|
206
|
+
if (customPushMetics && typeof customPushMetics === 'function') {
|
|
207
|
+
setInterval(() => customPushMetics(), interval * 1000);
|
|
208
|
+
} else {
|
|
209
|
+
setInterval(() => {
|
|
210
|
+
runPush().catch(err => {
|
|
211
|
+
console.error(`${this.prefixLogs} Failed to push metrics:`, err);
|
|
222
212
|
});
|
|
223
|
-
|
|
213
|
+
}, interval * 1000);
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
// First push immediately so metrics appear without waiting for the first interval
|
|
217
|
+
runPush().catch(err => {
|
|
218
|
+
console.error(`${this.prefixLogs} Failed to push metrics (initial):`, err);
|
|
219
|
+
});
|
|
220
|
+
let pushOrigin = 'none';
|
|
221
|
+
try {
|
|
222
|
+
if (this.pushgatewayUrl && this.pushgatewayUrl.trim()) {
|
|
223
|
+
pushOrigin = new URL(this.pushgatewayUrl.trim()).origin;
|
|
224
224
|
}
|
|
225
|
-
|
|
225
|
+
} catch {
|
|
226
|
+
pushOrigin = 'invalid URL';
|
|
226
227
|
}
|
|
227
|
-
|
|
228
|
+
console.warn(`${this.prefixLogs} Metrics collection started. (interval: ${this.intervalSec}s, push: ${pushOrigin})`);
|
|
228
229
|
};
|
|
229
230
|
pushMetrics = async () => {
|
|
230
231
|
return this._pushMetrics();
|
|
@@ -399,6 +400,17 @@ class BaseMetricsClient {
|
|
|
399
400
|
withDefaultLabels = (labels = []) => {
|
|
400
401
|
return [...Object.keys(this.defaultLabels), ...labels];
|
|
401
402
|
};
|
|
403
|
+
|
|
404
|
+
/**
|
|
405
|
+
* Merge default labels without dyno_id (`app`, `process_type`) with custom label names.
|
|
406
|
+
* Use for HTTP metrics so dyno_id is not included.
|
|
407
|
+
*
|
|
408
|
+
* @param {string[]} labels Additional label names
|
|
409
|
+
* @returns {string[]} Combined label names
|
|
410
|
+
*/
|
|
411
|
+
withDefaultLabelsWithoutDynoId = (labels = []) => {
|
|
412
|
+
return [...Object.keys(this.defaultLabelsWithoutDynoId), ...labels];
|
|
413
|
+
};
|
|
402
414
|
getDefaultLabels = (labels = []) => {
|
|
403
415
|
return this.defaultLabels;
|
|
404
416
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"baseMetricsClient.js","names":["client","require","https","http","URL","BaseMetricsClient","constructor","config","appName","process","env","BUILD_APP_NAME","dynoId","HOSTNAME","processType","BUILD_DYNO_PROCESS_TYPE","enabled","METRICS_ENABLED","logValues","METRICS_LOG_VALUES","pushgatewayUrl","METRICS_PUSHGATEWAY_URL","authToken","pushgatewaySecret","METRICS_PUSHGATEWAY_SECRET","intervalSec","parseInt","METRICS_INTERVAL_SEC","startupValidation","disablePushgateway","METRICS_DISABLE_PUSHGATEWAY","removeOldMetrics","METRICS_REMOVE_OLD_METRICS","prefixLogs","_registry","Registry","collectDefaultMetrics","register","defaultLabels","app","dyno_id","process_type","gateway","gauges","counters","countersFunctions","gaugeUpdaters","_clearOldWorkers","_setCleanupHandlers","keepProcessAliveWhenDisabled","createGauge","name","help","updateFn","labelNames","Object","keys","g","Gauge","registers","createCounter","c","Counter","data","value","inc","clearAllCounters","metricsLogValues","console","log","values","forEach","counter","reset","_pushMetrics","entries","result","val","Promise","undefined","set","err","error","gatewayPush","metrics","getMetricsAsJSON","JSON","stringify","_startPush","interval","customPushMetics","warn","_idleInterval","setInterval","clearInterval","doStart","runPush","resolve","pushMetrics","catch","pushOrigin","trim","origin","then","ok","startPush","cleanup","gatewayDelete","exit","_deleteFromVMByLabels","message","esc","s","String","replace","selector","u","reject","Error","path","encodeURIComponent","req","protocol","request","hostname","port","method","headers","Authorization","agent","Agent","keepAlive","res","statusCode","on","chunk","end","params","_pushToVMAgent","pushUrl","pathname","search","contentType","setHeader","Buffer","byteLength","withDefaultLabels","labels","getDefaultLabels","metricsEnabled","registry","getMetricsAsString","metricsMiddleware","status","module","exports"],"sources":["../../src/metrics/baseMetricsClient.js"],"sourcesContent":["const client = require('prom-client')\nconst https = require('https')\nconst http = require('http')\nconst { URL } = require('url')\n\n/**\n * BaseMetricsClient provides common functionality for all metrics clients.\n * Handles registry setup, push to remote (VM-agent), default labels, and common operations.\n * Always pushes registry to the configured URL (POST Prometheus text format + Basic auth). No Pushgateway.\n */\nclass BaseMetricsClient {\n /**\n * @param {Object} config\n * @param {string} [config.appName] Name of the application\n * @param {string} [config.dynoId] Dyno/instance ID\n * @param {string} [config.processType] Process type (web, worker, etc.)\n * @param {boolean} [config.enabled] Enable metrics collection\n * @param {boolean} [config.logValues] Log metrics values to console\n * @param {string} [config.pushgatewayUrl] Push URL (VM-agent import endpoint, e.g. .../api/v1/import/prometheus). /metrics is for GET (scrape), not POST (push).\n * @param {string} [config.pushgatewaySecret] Basic auth secret (Base64 of user:password)\n * @param {number} [config.intervalSec] Interval in seconds for pushing metrics\n * @param {boolean} [config.removeOldMetrics] Enable to clear metrics by service name\n * @param {function} [config.startupValidation] Add to validate on start push.\n * @param {boolean} [config.disablePushgateway] Disable pushing to Pushgateway (use HTTP scraping instead)\n */\n constructor(config = {}) {\n this.appName = config.appName || process.env.BUILD_APP_NAME || 'unknown-app'\n this.dynoId = config.dynoId || process.env.HOSTNAME || 'unknown-dyno'\n this.processType =\n config.processType ||\n process.env.BUILD_DYNO_PROCESS_TYPE ||\n 'undefined_build_dyno_type'\n this.enabled = config.enabled ?? process.env.METRICS_ENABLED === 'true'\n this.logValues =\n config.logValues ?? process.env.METRICS_LOG_VALUES === 'true'\n this.pushgatewayUrl =\n config.pushgatewayUrl || process.env.METRICS_PUSHGATEWAY_URL || ''\n this.authToken =\n config.pushgatewaySecret || process.env.METRICS_PUSHGATEWAY_SECRET || ''\n this.intervalSec =\n config.intervalSec ||\n parseInt(process.env.METRICS_INTERVAL_SEC || '', 10) ||\n 15\n this.startupValidation = config.startupValidation\n this.disablePushgateway =\n config.disablePushgateway ??\n process.env.METRICS_DISABLE_PUSHGATEWAY === 'true'\n this.removeOldMetrics =\n config.removeOldMetrics ??\n process.env.METRICS_REMOVE_OLD_METRICS === 'true'\n\n this.prefixLogs = `[${this.processType}] [${this.appName}] [${this.dynoId}] [Monitoring]`\n\n this._registry = new client.Registry()\n client.collectDefaultMetrics({ register: this._registry })\n\n this.defaultLabels = {\n app: this.appName,\n dyno_id: this.dynoId,\n process_type: this.processType,\n }\n\n // Always push to configured URL (VM-agent). No Pushgateway.\n this.gateway = null\n this.gauges = {}\n this.counters = {}\n this.countersFunctions = {}\n\n /** @type {Object<string, function(): number | Promise<number>>} */\n this.gaugeUpdaters = {}\n\n this._clearOldWorkers(config.removeOldMetrics)\n this._setCleanupHandlers()\n\n this.keepProcessAliveWhenDisabled = true\n }\n\n /**\n * Create a gauge metric.\n * @param {Object} options - Gauge configuration\n * @param {string} options.name - Name of the gauge\n * @param {string} options.help - Help text describing the gauge\n * @param {function(): number|Promise<number>} [options.updateFn] - Optional function returning the gauge value\n * @param {string[]} [options.labelNames] - Optional custom label names\n * @returns {import('prom-client').Gauge} The created Prometheus gauge\n */\n createGauge = ({\n name,\n help,\n updateFn,\n labelNames = Object.keys(this.defaultLabels),\n }) => {\n if (this.gauges[name]) return this.gauges[name]\n\n const g = new client.Gauge({\n name,\n help,\n labelNames,\n registers: [this._registry],\n })\n this.gauges[name] = g\n\n if (updateFn && typeof updateFn === 'function') {\n this.gaugeUpdaters[name] = updateFn\n }\n\n return g\n }\n\n /**\n * Create a Prometheus Counter metric.\n *\n * @param {Object} params - Counter configuration\n * @param {string} params.name - Metric name\n * @param {string} params.help - Metric description\n * @param {string[]} [params.labelNames] - Optional list of label names. Defaults to this.defaultLabels keys.\n *\n * @returns {(labels?: Object, incrementValue?: number) => void}\n * A function to increment the counter.\n * Usage: (labels?, incrementValue?)\n */\n createCounter({ name, help, labelNames = Object.keys(this.defaultLabels) }) {\n if (this.counters[name]) return this.countersFunctions[name]\n\n const c = new client.Counter({\n name,\n help,\n labelNames,\n registers: [this._registry],\n })\n this.counters[name] = c\n\n this.countersFunctions = {\n ...this.countersFunctions,\n [name]: (data = {}, value = 1) => {\n c.inc({ ...this.defaultLabels, ...data }, value)\n },\n }\n\n return this.countersFunctions[name]\n }\n\n /**\n * Clear all collected counters\n */\n clearAllCounters = () => {\n if (this.metricsLogValues) {\n console.log('Counters to clear: ', Object.keys(this.counters))\n }\n Object.values(this.counters).forEach(counter => counter.reset())\n }\n\n /**\n * Push all gauges and counters to PushGateway and optionally log.\n */\n _pushMetrics = async () => {\n try {\n for (const [name, updateFn] of Object.entries(this.gaugeUpdaters)) {\n try {\n if (!updateFn) {\n continue\n }\n const result = updateFn()\n const val = result instanceof Promise ? await result : result\n if (val !== undefined) this.gauges[name].set(this.defaultLabels, val)\n } catch (err) {\n console.error(\n `${this.prefixLogs} Failed to update gauge ${name}:`,\n err\n )\n }\n }\n\n if (!this.disablePushgateway) {\n await this.gatewayPush()\n }\n // this.clearAllCounters() //TODO: or uncommit or delete (based on grafana expectation)\n\n if (this.logValues) {\n const metrics = await this._registry.getMetricsAsJSON()\n console.log(\n `${this.prefixLogs} Metrics:\\n`,\n JSON.stringify(metrics, null, 2)\n )\n }\n } catch (err) {\n console.error(`${this.prefixLogs} Failed to push metrics:`, err)\n }\n }\n\n _startPush = (interval = this.intervalSec, customPushMetics = undefined) => {\n if (!this.enabled) {\n console.warn(`${this.prefixLogs} Metrics disabled`)\n if (this.keepProcessAliveWhenDisabled && !this._idleInterval) {\n this._idleInterval = setInterval(() => {}, 60 * 60 * 1000)\n }\n return\n }\n\n if (this._idleInterval) {\n clearInterval(this._idleInterval)\n this._idleInterval = null\n }\n\n const doStart = () => {\n const runPush = () => {\n if (customPushMetics && typeof customPushMetics === 'function') {\n return Promise.resolve(customPushMetics())\n }\n return this.pushMetrics()\n }\n\n if (customPushMetics && typeof customPushMetics === 'function') {\n setInterval(() => customPushMetics(), interval * 1000)\n } else {\n setInterval(() => {\n runPush().catch(err => {\n console.error(`${this.prefixLogs} Failed to push metrics:`, err)\n })\n }, interval * 1000)\n }\n\n // First push immediately so metrics appear without waiting for the first interval\n runPush().catch(err => {\n console.error(`${this.prefixLogs} Failed to push metrics (initial):`, err)\n })\n\n let pushOrigin = 'none'\n try {\n if (this.pushgatewayUrl && this.pushgatewayUrl.trim()) {\n pushOrigin = new URL(this.pushgatewayUrl.trim()).origin\n }\n } catch {\n pushOrigin = 'invalid URL'\n }\n console.warn(\n `${this.prefixLogs} Metrics collection started. (interval: ${this.intervalSec}s, push: ${pushOrigin})`\n )\n }\n\n if (this.startupValidation) {\n const result = this.startupValidation()\n if (result != null && typeof result.then === 'function') {\n result\n .then(ok => {\n if (ok !== false) doStart()\n })\n .catch(err => {\n console.error(\n `${this.prefixLogs} Startup validation failed:`,\n err\n )\n })\n return\n }\n if (result === false) return\n }\n doStart()\n }\n\n pushMetrics = async () => {\n return this._pushMetrics()\n }\n\n /**\n * Start periodic metrics collection and push.\n *\n * This method wraps the internal `_startPush` method.\n * If a `customPushMetrics` function is provided, it will be executed\n * at the given interval instead of the default `pushMetrics` behavior.\n *\n * @param {number} [interval=this.intervalSec] - Interval in seconds between pushes.\n * @param {() => void | Promise<void>} [customPushMetrics] - Optional custom push function. If provided, Prometheus push is skipped.\n */\n startPush = (interval, customPushMetics = undefined) => {\n this._startPush(interval, customPushMetics)\n }\n\n /**\n * Cleanup metrics and exit process.\n * @returns {Promise<void>}\n */\n cleanup = async () => {\n if (this.enabled) {\n await this.gatewayDelete()\n }\n process.exit(0)\n }\n\n /**\n * Remove old/stale dyno/instance metrics from PushGateway.\n *\n * Compares existing PushGateway metrics for this job and deletes any instances\n * that do not match the current dynoId.\n *\n * @param {boolean} removeOldMetrics If true, performs cleanup; otherwise does nothing\n * @returns {Promise<void>}\n * @private\n */\n _clearOldWorkers = async removeOldMetrics => {\n // No Pushgateway; VM-agent does not support per-instance delete. Skip.\n }\n\n /**\n * On shutdown: optionally delete this instance's metrics from VictoriaMetrics (by app, dyno_id, process_type).\n * @returns {Promise<void>}\n */\n gatewayDelete = async () => {\n if (\n this.removeOldMetrics &&\n this.pushgatewayUrl &&\n this.pushgatewayUrl.trim()\n ) {\n await this._deleteFromVMByLabels().catch(err => {\n console.warn(\n `${this.prefixLogs} Deletion from VM on shutdown failed:`,\n err.message\n )\n })\n }\n }\n\n /**\n * Call VictoriaMetrics delete_series API to remove all series matching this instance's labels (app, dyno_id, process_type).\n * @private\n */\n _deleteFromVMByLabels = () => {\n const esc = s => String(s).replace(/\\\\/g, '\\\\\\\\').replace(/\"/g, '\\\\\"')\n const selector = `{app=\"${esc(this.appName)}\",dyno_id=\"${esc(\n this.dynoId\n )}\",process_type=\"${esc(this.processType)}\"}`\n let origin\n try {\n const u = new URL((this.pushgatewayUrl || '').trim())\n origin = u.origin\n } catch {\n return Promise.reject(new Error('Invalid push URL'))\n }\n const path = `/api/v1/admin/tsdb/delete_series?match[]=${encodeURIComponent(\n selector\n )}`\n return new Promise((resolve, reject) => {\n const u = new URL(origin)\n const req = (u.protocol === 'https:' ? https : http).request(\n {\n hostname: u.hostname,\n port: u.port || (u.protocol === 'https:' ? 443 : 80),\n path,\n method: 'POST',\n headers: {\n 'Content-Length': '0',\n Authorization: this.authToken\n ? `Basic ${this.authToken}`\n : undefined,\n },\n agent:\n u.protocol === 'https:'\n ? new https.Agent({ keepAlive: false })\n : undefined,\n },\n res => {\n if (res.statusCode >= 200 && res.statusCode < 300) resolve()\n else {\n let data = ''\n res.on('data', chunk => {\n data += chunk\n })\n res.on('end', () =>\n reject(new Error(`Delete failed: ${res.statusCode} ${data}`))\n )\n }\n }\n )\n req.on('error', reject)\n req.end()\n })\n }\n\n /**\n * Push registry to configured URL (VM-agent). POST Prometheus text format + Basic auth.\n *\n * @param {object} [params] Unused; kept for API compatibility.\n * @returns {Promise<void>}\n */\n gatewayPush = async (params = {}) => {\n if (this.disablePushgateway) {\n console.warn(\n `${this.prefixLogs} Metrics push skipped: METRICS_DISABLE_PUSHGATEWAY is set`\n )\n return Promise.resolve()\n }\n if (!this.pushgatewayUrl || !this.pushgatewayUrl.trim()) {\n console.warn(\n `${this.prefixLogs} Metrics push skipped: METRICS_PUSHGATEWAY_URL is not set`\n )\n return Promise.resolve()\n }\n return this._pushToVMAgent()\n }\n\n /**\n * POST registry (Prometheus text format) to VM-agent. VM-agent accepts push at /api/v1/import/prometheus; /metrics is GET (scrape) only.\n * @private\n */\n _pushToVMAgent = () => {\n let pushUrl = (this.pushgatewayUrl || '').trim()\n try {\n const u = new URL(pushUrl)\n if (!u.pathname || u.pathname === '/' || u.pathname === '/metrics') {\n pushUrl = `${u.origin}/api/v1/import/prometheus${u.search}`\n }\n } catch {\n // leave pushUrl as-is\n }\n return new Promise((resolve, reject) => {\n const u = new URL(pushUrl)\n const req = (u.protocol === 'https:' ? https : http).request(\n {\n hostname: u.hostname,\n port: u.port || (u.protocol === 'https:' ? 443 : 80),\n path: u.pathname + u.search,\n method: 'POST',\n headers: {\n 'Content-Type': client.register.contentType,\n Authorization: this.authToken\n ? `Basic ${this.authToken}`\n : undefined,\n },\n agent:\n u.protocol === 'https:'\n ? new https.Agent({ keepAlive: true })\n : undefined,\n },\n res => {\n if (res.statusCode >= 200 && res.statusCode < 300) {\n resolve()\n } else {\n let data = ''\n res.on('data', chunk => {\n data += chunk\n })\n res.on('end', () =>\n reject(new Error(`Push failed: ${res.statusCode} ${data}`))\n )\n }\n }\n )\n req.on('error', reject)\n this._registry\n .metrics()\n .then(metrics => {\n req.setHeader('Content-Length', Buffer.byteLength(metrics, 'utf8'))\n req.end(metrics, 'utf8')\n })\n .catch(reject)\n })\n }\n\n /**\n * Merge the default metric labels (`app`, `dyno_id`, `process_type`)\n * with custom label names.\n *\n * @param {string[]} labels Additional label names\n * @returns {string[]} Combined label names\n */\n withDefaultLabels = (labels = []) => {\n return [...Object.keys(this.defaultLabels), ...labels]\n }\n\n getDefaultLabels = (labels = []) => {\n return this.defaultLabels\n }\n\n _setCleanupHandlers = () => {\n process.on('SIGINT', this.cleanup)\n process.on('SIGTERM', this.cleanup)\n }\n\n // GETTERS\n\n get metricsEnabled() {\n return this.enabled\n }\n\n get metricsLogValues() {\n return this.logValues\n }\n\n get registry() {\n return this._registry\n }\n\n async getMetricsAsString() {\n return this._registry.metrics()\n }\n\n metricsMiddleware() {\n return async (req, res) => {\n try {\n const metrics = await this.getMetricsAsString()\n res.set('Content-Type', client.register.contentType)\n res.end(metrics)\n } catch (err) {\n console.error(`${this.prefixLogs} Failed to get metrics:`, err)\n res.status(500).end('Failed to collect metrics')\n }\n }\n }\n}\n\nmodule.exports = { BaseMetricsClient }\n"],"mappings":";;AAAA,MAAMA,MAAM,GAAGC,OAAO,CAAC,aAAa,CAAC;AACrC,MAAMC,KAAK,GAAGD,OAAO,CAAC,OAAO,CAAC;AAC9B,MAAME,IAAI,GAAGF,OAAO,CAAC,MAAM,CAAC;AAC5B,MAAM;EAAEG;AAAI,CAAC,GAAGH,OAAO,CAAC,KAAK,CAAC;;AAE9B;AACA;AACA;AACA;AACA;AACA,MAAMI,iBAAiB,CAAC;EACtB;AACF;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;EACEC,WAAWA,CAACC,MAAM,GAAG,CAAC,CAAC,EAAE;IACvB,IAAI,CAACC,OAAO,GAAGD,MAAM,CAACC,OAAO,IAAIC,OAAO,CAACC,GAAG,CAACC,cAAc,IAAI,aAAa;IAC5E,IAAI,CAACC,MAAM,GAAGL,MAAM,CAACK,MAAM,IAAIH,OAAO,CAACC,GAAG,CAACG,QAAQ,IAAI,cAAc;IACrE,IAAI,CAACC,WAAW,GACdP,MAAM,CAACO,WAAW,IAClBL,OAAO,CAACC,GAAG,CAACK,uBAAuB,IACnC,2BAA2B;IAC7B,IAAI,CAACC,OAAO,GAAGT,MAAM,CAACS,OAAO,IAAIP,OAAO,CAACC,GAAG,CAACO,eAAe,KAAK,MAAM;IACvE,IAAI,CAACC,SAAS,GACZX,MAAM,CAACW,SAAS,IAAIT,OAAO,CAACC,GAAG,CAACS,kBAAkB,KAAK,MAAM;IAC/D,IAAI,CAACC,cAAc,GACjBb,MAAM,CAACa,cAAc,IAAIX,OAAO,CAACC,GAAG,CAACW,uBAAuB,IAAI,EAAE;IACpE,IAAI,CAACC,SAAS,GACZf,MAAM,CAACgB,iBAAiB,IAAId,OAAO,CAACC,GAAG,CAACc,0BAA0B,IAAI,EAAE;IAC1E,IAAI,CAACC,WAAW,GACdlB,MAAM,CAACkB,WAAW,IAClBC,QAAQ,CAACjB,OAAO,CAACC,GAAG,CAACiB,oBAAoB,IAAI,EAAE,EAAE,EAAE,CAAC,IACpD,EAAE;IACJ,IAAI,CAACC,iBAAiB,GAAGrB,MAAM,CAACqB,iBAAiB;IACjD,IAAI,CAACC,kBAAkB,GACrBtB,MAAM,CAACsB,kBAAkB,IACzBpB,OAAO,CAACC,GAAG,CAACoB,2BAA2B,KAAK,MAAM;IACpD,IAAI,CAACC,gBAAgB,GACnBxB,MAAM,CAACwB,gBAAgB,IACvBtB,OAAO,CAACC,GAAG,CAACsB,0BAA0B,KAAK,MAAM;IAEnD,IAAI,CAACC,UAAU,GAAG,IAAI,IAAI,CAACnB,WAAW,MAAM,IAAI,CAACN,OAAO,MAAM,IAAI,CAACI,MAAM,gBAAgB;IAEzF,IAAI,CAACsB,SAAS,GAAG,IAAIlC,MAAM,CAACmC,QAAQ,CAAC,CAAC;IACtCnC,MAAM,CAACoC,qBAAqB,CAAC;MAAEC,QAAQ,EAAE,IAAI,CAACH;IAAU,CAAC,CAAC;IAE1D,IAAI,CAACI,aAAa,GAAG;MACnBC,GAAG,EAAE,IAAI,CAAC/B,OAAO;MACjBgC,OAAO,EAAE,IAAI,CAAC5B,MAAM;MACpB6B,YAAY,EAAE,IAAI,CAAC3B;IACrB,CAAC;;IAED;IACA,IAAI,CAAC4B,OAAO,GAAG,IAAI;IACnB,IAAI,CAACC,MAAM,GAAG,CAAC,CAAC;IAChB,IAAI,CAACC,QAAQ,GAAG,CAAC,CAAC;IAClB,IAAI,CAACC,iBAAiB,GAAG,CAAC,CAAC;;IAE3B;IACA,IAAI,CAACC,aAAa,GAAG,CAAC,CAAC;IAEvB,IAAI,CAACC,gBAAgB,CAACxC,MAAM,CAACwB,gBAAgB,CAAC;IAC9C,IAAI,CAACiB,mBAAmB,CAAC,CAAC;IAE1B,IAAI,CAACC,4BAA4B,GAAG,IAAI;EAC1C;;EAEA;AACF;AACA;AACA;AACA;AACA;AACA;AACA;AACA;EACEC,WAAW,GAAGA,CAAC;IACbC,IAAI;IACJC,IAAI;IACJC,QAAQ;IACRC,UAAU,GAAGC,MAAM,CAACC,IAAI,CAAC,IAAI,CAAClB,aAAa;EAC7C,CAAC,KAAK;IACJ,IAAI,IAAI,CAACK,MAAM,CAACQ,IAAI,CAAC,EAAE,OAAO,IAAI,CAACR,MAAM,CAACQ,IAAI,CAAC;IAE/C,MAAMM,CAAC,GAAG,IAAIzD,MAAM,CAAC0D,KAAK,CAAC;MACzBP,IAAI;MACJC,IAAI;MACJE,UAAU;MACVK,SAAS,EAAE,CAAC,IAAI,CAACzB,SAAS;IAC5B,CAAC,CAAC;IACF,IAAI,CAACS,MAAM,CAACQ,IAAI,CAAC,GAAGM,CAAC;IAErB,IAAIJ,QAAQ,IAAI,OAAOA,QAAQ,KAAK,UAAU,EAAE;MAC9C,IAAI,CAACP,aAAa,CAACK,IAAI,CAAC,GAAGE,QAAQ;IACrC;IAEA,OAAOI,CAAC;EACV,CAAC;;EAED;AACF;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;EACEG,aAAaA,CAAC;IAAET,IAAI;IAAEC,IAAI;IAAEE,UAAU,GAAGC,MAAM,CAACC,IAAI,CAAC,IAAI,CAAClB,aAAa;EAAE,CAAC,EAAE;IAC1E,IAAI,IAAI,CAACM,QAAQ,CAACO,IAAI,CAAC,EAAE,OAAO,IAAI,CAACN,iBAAiB,CAACM,IAAI,CAAC;IAE5D,MAAMU,CAAC,GAAG,IAAI7D,MAAM,CAAC8D,OAAO,CAAC;MAC3BX,IAAI;MACJC,IAAI;MACJE,UAAU;MACVK,SAAS,EAAE,CAAC,IAAI,CAACzB,SAAS;IAC5B,CAAC,CAAC;IACF,IAAI,CAACU,QAAQ,CAACO,IAAI,CAAC,GAAGU,CAAC;IAEvB,IAAI,CAAChB,iBAAiB,GAAG;MACvB,GAAG,IAAI,CAACA,iBAAiB;MACzB,CAACM,IAAI,GAAG,CAACY,IAAI,GAAG,CAAC,CAAC,EAAEC,KAAK,GAAG,CAAC,KAAK;QAChCH,CAAC,CAACI,GAAG,CAAC;UAAE,GAAG,IAAI,CAAC3B,aAAa;UAAE,GAAGyB;QAAK,CAAC,EAAEC,KAAK,CAAC;MAClD;IACF,CAAC;IAED,OAAO,IAAI,CAACnB,iBAAiB,CAACM,IAAI,CAAC;EACrC;;EAEA;AACF;AACA;EACEe,gBAAgB,GAAGA,CAAA,KAAM;IACvB,IAAI,IAAI,CAACC,gBAAgB,EAAE;MACzBC,OAAO,CAACC,GAAG,CAAC,qBAAqB,EAAEd,MAAM,CAACC,IAAI,CAAC,IAAI,CAACZ,QAAQ,CAAC,CAAC;IAChE;IACAW,MAAM,CAACe,MAAM,CAAC,IAAI,CAAC1B,QAAQ,CAAC,CAAC2B,OAAO,CAACC,OAAO,IAAIA,OAAO,CAACC,KAAK,CAAC,CAAC,CAAC;EAClE,CAAC;;EAED;AACF;AACA;EACEC,YAAY,GAAG,MAAAA,CAAA,KAAY;IACzB,IAAI;MACF,KAAK,MAAM,CAACvB,IAAI,EAAEE,QAAQ,CAAC,IAAIE,MAAM,CAACoB,OAAO,CAAC,IAAI,CAAC7B,aAAa,CAAC,EAAE;QACjE,IAAI;UACF,IAAI,CAACO,QAAQ,EAAE;YACb;UACF;UACA,MAAMuB,MAAM,GAAGvB,QAAQ,CAAC,CAAC;UACzB,MAAMwB,GAAG,GAAGD,MAAM,YAAYE,OAAO,GAAG,MAAMF,MAAM,GAAGA,MAAM;UAC7D,IAAIC,GAAG,KAAKE,SAAS,EAAE,IAAI,CAACpC,MAAM,CAACQ,IAAI,CAAC,CAAC6B,GAAG,CAAC,IAAI,CAAC1C,aAAa,EAAEuC,GAAG,CAAC;QACvE,CAAC,CAAC,OAAOI,GAAG,EAAE;UACZb,OAAO,CAACc,KAAK,CACX,GAAG,IAAI,CAACjD,UAAU,2BAA2BkB,IAAI,GAAG,EACpD8B,GACF,CAAC;QACH;MACF;MAEA,IAAI,CAAC,IAAI,CAACpD,kBAAkB,EAAE;QAC5B,MAAM,IAAI,CAACsD,WAAW,CAAC,CAAC;MAC1B;MACA;;MAEA,IAAI,IAAI,CAACjE,SAAS,EAAE;QAClB,MAAMkE,OAAO,GAAG,MAAM,IAAI,CAAClD,SAAS,CAACmD,gBAAgB,CAAC,CAAC;QACvDjB,OAAO,CAACC,GAAG,CACT,GAAG,IAAI,CAACpC,UAAU,aAAa,EAC/BqD,IAAI,CAACC,SAAS,CAACH,OAAO,EAAE,IAAI,EAAE,CAAC,CACjC,CAAC;MACH;IACF,CAAC,CAAC,OAAOH,GAAG,EAAE;MACZb,OAAO,CAACc,KAAK,CAAC,GAAG,IAAI,CAACjD,UAAU,0BAA0B,EAAEgD,GAAG,CAAC;IAClE;EACF,CAAC;EAEDO,UAAU,GAAGA,CAACC,QAAQ,GAAG,IAAI,CAAChE,WAAW,EAAEiE,gBAAgB,GAAGX,SAAS,KAAK;IAC1E,IAAI,CAAC,IAAI,CAAC/D,OAAO,EAAE;MACjBoD,OAAO,CAACuB,IAAI,CAAC,GAAG,IAAI,CAAC1D,UAAU,mBAAmB,CAAC;MACnD,IAAI,IAAI,CAACgB,4BAA4B,IAAI,CAAC,IAAI,CAAC2C,aAAa,EAAE;QAC5D,IAAI,CAACA,aAAa,GAAGC,WAAW,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;MAC5D;MACA;IACF;IAEA,IAAI,IAAI,CAACD,aAAa,EAAE;MACtBE,aAAa,CAAC,IAAI,CAACF,aAAa,CAAC;MACjC,IAAI,CAACA,aAAa,GAAG,IAAI;IAC3B;IAEA,MAAMG,OAAO,GAAGA,CAAA,KAAM;MACpB,MAAMC,OAAO,GAAGA,CAAA,KAAM;QACpB,IAAIN,gBAAgB,IAAI,OAAOA,gBAAgB,KAAK,UAAU,EAAE;UAC9D,OAAOZ,OAAO,CAACmB,OAAO,CAACP,gBAAgB,CAAC,CAAC,CAAC;QAC5C;QACA,OAAO,IAAI,CAACQ,WAAW,CAAC,CAAC;MAC3B,CAAC;MAED,IAAIR,gBAAgB,IAAI,OAAOA,gBAAgB,KAAK,UAAU,EAAE;QAC9DG,WAAW,CAAC,MAAMH,gBAAgB,CAAC,CAAC,EAAED,QAAQ,GAAG,IAAI,CAAC;MACxD,CAAC,MAAM;QACLI,WAAW,CAAC,MAAM;UAChBG,OAAO,CAAC,CAAC,CAACG,KAAK,CAAClB,GAAG,IAAI;YACrBb,OAAO,CAACc,KAAK,CAAC,GAAG,IAAI,CAACjD,UAAU,0BAA0B,EAAEgD,GAAG,CAAC;UAClE,CAAC,CAAC;QACJ,CAAC,EAAEQ,QAAQ,GAAG,IAAI,CAAC;MACrB;;MAEA;MACAO,OAAO,CAAC,CAAC,CAACG,KAAK,CAAClB,GAAG,IAAI;QACrBb,OAAO,CAACc,KAAK,CAAC,GAAG,IAAI,CAACjD,UAAU,oCAAoC,EAAEgD,GAAG,CAAC;MAC5E,CAAC,CAAC;MAEF,IAAImB,UAAU,GAAG,MAAM;MACvB,IAAI;QACF,IAAI,IAAI,CAAChF,cAAc,IAAI,IAAI,CAACA,cAAc,CAACiF,IAAI,CAAC,CAAC,EAAE;UACrDD,UAAU,GAAG,IAAIhG,GAAG,CAAC,IAAI,CAACgB,cAAc,CAACiF,IAAI,CAAC,CAAC,CAAC,CAACC,MAAM;QACzD;MACF,CAAC,CAAC,MAAM;QACNF,UAAU,GAAG,aAAa;MAC5B;MACAhC,OAAO,CAACuB,IAAI,CACV,GAAG,IAAI,CAAC1D,UAAU,2CAA2C,IAAI,CAACR,WAAW,YAAY2E,UAAU,GACrG,CAAC;IACH,CAAC;IAED,IAAI,IAAI,CAACxE,iBAAiB,EAAE;MAC1B,MAAMgD,MAAM,GAAG,IAAI,CAAChD,iBAAiB,CAAC,CAAC;MACvC,IAAIgD,MAAM,IAAI,IAAI,IAAI,OAAOA,MAAM,CAAC2B,IAAI,KAAK,UAAU,EAAE;QACvD3B,MAAM,CACH2B,IAAI,CAACC,EAAE,IAAI;UACV,IAAIA,EAAE,KAAK,KAAK,EAAET,OAAO,CAAC,CAAC;QAC7B,CAAC,CAAC,CACDI,KAAK,CAAClB,GAAG,IAAI;UACZb,OAAO,CAACc,KAAK,CACX,GAAG,IAAI,CAACjD,UAAU,6BAA6B,EAC/CgD,GACF,CAAC;QACH,CAAC,CAAC;QACJ;MACF;MACA,IAAIL,MAAM,KAAK,KAAK,EAAE;IACxB;IACAmB,OAAO,CAAC,CAAC;EACX,CAAC;EAEDG,WAAW,GAAG,MAAAA,CAAA,KAAY;IACxB,OAAO,IAAI,CAACxB,YAAY,CAAC,CAAC;EAC5B,CAAC;;EAED;AACF;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;EACE+B,SAAS,GAAGA,CAAChB,QAAQ,EAAEC,gBAAgB,GAAGX,SAAS,KAAK;IACtD,IAAI,CAACS,UAAU,CAACC,QAAQ,EAAEC,gBAAgB,CAAC;EAC7C,CAAC;;EAED;AACF;AACA;AACA;EACEgB,OAAO,GAAG,MAAAA,CAAA,KAAY;IACpB,IAAI,IAAI,CAAC1F,OAAO,EAAE;MAChB,MAAM,IAAI,CAAC2F,aAAa,CAAC,CAAC;IAC5B;IACAlG,OAAO,CAACmG,IAAI,CAAC,CAAC,CAAC;EACjB,CAAC;;EAED;AACF;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;EACE7D,gBAAgB,GAAG,MAAMhB,gBAAgB,IAAI;IAC3C;EAAA,CACD;;EAED;AACF;AACA;AACA;EACE4E,aAAa,GAAG,MAAAA,CAAA,KAAY;IAC1B,IACE,IAAI,CAAC5E,gBAAgB,IACrB,IAAI,CAACX,cAAc,IACnB,IAAI,CAACA,cAAc,CAACiF,IAAI,CAAC,CAAC,EAC1B;MACA,MAAM,IAAI,CAACQ,qBAAqB,CAAC,CAAC,CAACV,KAAK,CAAClB,GAAG,IAAI;QAC9Cb,OAAO,CAACuB,IAAI,CACV,GAAG,IAAI,CAAC1D,UAAU,uCAAuC,EACzDgD,GAAG,CAAC6B,OACN,CAAC;MACH,CAAC,CAAC;IACJ;EACF,CAAC;;EAED;AACF;AACA;AACA;EACED,qBAAqB,GAAGA,CAAA,KAAM;IAC5B,MAAME,GAAG,GAAGC,CAAC,IAAIC,MAAM,CAACD,CAAC,CAAC,CAACE,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,CAACA,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC;IACtE,MAAMC,QAAQ,GAAG,SAASJ,GAAG,CAAC,IAAI,CAACvG,OAAO,CAAC,cAAcuG,GAAG,CAC1D,IAAI,CAACnG,MACP,CAAC,mBAAmBmG,GAAG,CAAC,IAAI,CAACjG,WAAW,CAAC,IAAI;IAC7C,IAAIwF,MAAM;IACV,IAAI;MACF,MAAMc,CAAC,GAAG,IAAIhH,GAAG,CAAC,CAAC,IAAI,CAACgB,cAAc,IAAI,EAAE,EAAEiF,IAAI,CAAC,CAAC,CAAC;MACrDC,MAAM,GAAGc,CAAC,CAACd,MAAM;IACnB,CAAC,CAAC,MAAM;MACN,OAAOxB,OAAO,CAACuC,MAAM,CAAC,IAAIC,KAAK,CAAC,kBAAkB,CAAC,CAAC;IACtD;IACA,MAAMC,IAAI,GAAG,4CAA4CC,kBAAkB,CACzEL,QACF,CAAC,EAAE;IACH,OAAO,IAAIrC,OAAO,CAAC,CAACmB,OAAO,EAAEoB,MAAM,KAAK;MACtC,MAAMD,CAAC,GAAG,IAAIhH,GAAG,CAACkG,MAAM,CAAC;MACzB,MAAMmB,GAAG,GAAG,CAACL,CAAC,CAACM,QAAQ,KAAK,QAAQ,GAAGxH,KAAK,GAAGC,IAAI,EAAEwH,OAAO,CAC1D;QACEC,QAAQ,EAAER,CAAC,CAACQ,QAAQ;QACpBC,IAAI,EAAET,CAAC,CAACS,IAAI,KAAKT,CAAC,CAACM,QAAQ,KAAK,QAAQ,GAAG,GAAG,GAAG,EAAE,CAAC;QACpDH,IAAI;QACJO,MAAM,EAAE,MAAM;QACdC,OAAO,EAAE;UACP,gBAAgB,EAAE,GAAG;UACrBC,aAAa,EAAE,IAAI,CAAC1G,SAAS,GACzB,SAAS,IAAI,CAACA,SAAS,EAAE,GACzByD;QACN,CAAC;QACDkD,KAAK,EACHb,CAAC,CAACM,QAAQ,KAAK,QAAQ,GACnB,IAAIxH,KAAK,CAACgI,KAAK,CAAC;UAAEC,SAAS,EAAE;QAAM,CAAC,CAAC,GACrCpD;MACR,CAAC,EACDqD,GAAG,IAAI;QACL,IAAIA,GAAG,CAACC,UAAU,IAAI,GAAG,IAAID,GAAG,CAACC,UAAU,GAAG,GAAG,EAAEpC,OAAO,CAAC,CAAC,MACvD;UACH,IAAIlC,IAAI,GAAG,EAAE;UACbqE,GAAG,CAACE,EAAE,CAAC,MAAM,EAAEC,KAAK,IAAI;YACtBxE,IAAI,IAAIwE,KAAK;UACf,CAAC,CAAC;UACFH,GAAG,CAACE,EAAE,CAAC,KAAK,EAAE,MACZjB,MAAM,CAAC,IAAIC,KAAK,CAAC,kBAAkBc,GAAG,CAACC,UAAU,IAAItE,IAAI,EAAE,CAAC,CAC9D,CAAC;QACH;MACF,CACF,CAAC;MACD0D,GAAG,CAACa,EAAE,CAAC,OAAO,EAAEjB,MAAM,CAAC;MACvBI,GAAG,CAACe,GAAG,CAAC,CAAC;IACX,CAAC,CAAC;EACJ,CAAC;;EAED;AACF;AACA;AACA;AACA;AACA;EACErD,WAAW,GAAG,MAAAA,CAAOsD,MAAM,GAAG,CAAC,CAAC,KAAK;IACnC,IAAI,IAAI,CAAC5G,kBAAkB,EAAE;MAC3BuC,OAAO,CAACuB,IAAI,CACV,GAAG,IAAI,CAAC1D,UAAU,2DACpB,CAAC;MACD,OAAO6C,OAAO,CAACmB,OAAO,CAAC,CAAC;IAC1B;IACA,IAAI,CAAC,IAAI,CAAC7E,cAAc,IAAI,CAAC,IAAI,CAACA,cAAc,CAACiF,IAAI,CAAC,CAAC,EAAE;MACvDjC,OAAO,CAACuB,IAAI,CACV,GAAG,IAAI,CAAC1D,UAAU,2DACpB,CAAC;MACD,OAAO6C,OAAO,CAACmB,OAAO,CAAC,CAAC;IAC1B;IACA,OAAO,IAAI,CAACyC,cAAc,CAAC,CAAC;EAC9B,CAAC;;EAED;AACF;AACA;AACA;EACEA,cAAc,GAAGA,CAAA,KAAM;IACrB,IAAIC,OAAO,GAAG,CAAC,IAAI,CAACvH,cAAc,IAAI,EAAE,EAAEiF,IAAI,CAAC,CAAC;IAChD,IAAI;MACF,MAAMe,CAAC,GAAG,IAAIhH,GAAG,CAACuI,OAAO,CAAC;MAC1B,IAAI,CAACvB,CAAC,CAACwB,QAAQ,IAAIxB,CAAC,CAACwB,QAAQ,KAAK,GAAG,IAAIxB,CAAC,CAACwB,QAAQ,KAAK,UAAU,EAAE;QAClED,OAAO,GAAG,GAAGvB,CAAC,CAACd,MAAM,4BAA4Bc,CAAC,CAACyB,MAAM,EAAE;MAC7D;IACF,CAAC,CAAC,MAAM;MACN;IAAA;IAEF,OAAO,IAAI/D,OAAO,CAAC,CAACmB,OAAO,EAAEoB,MAAM,KAAK;MACtC,MAAMD,CAAC,GAAG,IAAIhH,GAAG,CAACuI,OAAO,CAAC;MAC1B,MAAMlB,GAAG,GAAG,CAACL,CAAC,CAACM,QAAQ,KAAK,QAAQ,GAAGxH,KAAK,GAAGC,IAAI,EAAEwH,OAAO,CAC1D;QACEC,QAAQ,EAAER,CAAC,CAACQ,QAAQ;QACpBC,IAAI,EAAET,CAAC,CAACS,IAAI,KAAKT,CAAC,CAACM,QAAQ,KAAK,QAAQ,GAAG,GAAG,GAAG,EAAE,CAAC;QACpDH,IAAI,EAAEH,CAAC,CAACwB,QAAQ,GAAGxB,CAAC,CAACyB,MAAM;QAC3Bf,MAAM,EAAE,MAAM;QACdC,OAAO,EAAE;UACP,cAAc,EAAE/H,MAAM,CAACqC,QAAQ,CAACyG,WAAW;UAC3Cd,aAAa,EAAE,IAAI,CAAC1G,SAAS,GACzB,SAAS,IAAI,CAACA,SAAS,EAAE,GACzByD;QACN,CAAC;QACDkD,KAAK,EACHb,CAAC,CAACM,QAAQ,KAAK,QAAQ,GACnB,IAAIxH,KAAK,CAACgI,KAAK,CAAC;UAAEC,SAAS,EAAE;QAAK,CAAC,CAAC,GACpCpD;MACR,CAAC,EACDqD,GAAG,IAAI;QACL,IAAIA,GAAG,CAACC,UAAU,IAAI,GAAG,IAAID,GAAG,CAACC,UAAU,GAAG,GAAG,EAAE;UACjDpC,OAAO,CAAC,CAAC;QACX,CAAC,MAAM;UACL,IAAIlC,IAAI,GAAG,EAAE;UACbqE,GAAG,CAACE,EAAE,CAAC,MAAM,EAAEC,KAAK,IAAI;YACtBxE,IAAI,IAAIwE,KAAK;UACf,CAAC,CAAC;UACFH,GAAG,CAACE,EAAE,CAAC,KAAK,EAAE,MACZjB,MAAM,CAAC,IAAIC,KAAK,CAAC,gBAAgBc,GAAG,CAACC,UAAU,IAAItE,IAAI,EAAE,CAAC,CAC5D,CAAC;QACH;MACF,CACF,CAAC;MACD0D,GAAG,CAACa,EAAE,CAAC,OAAO,EAAEjB,MAAM,CAAC;MACvB,IAAI,CAACnF,SAAS,CACXkD,OAAO,CAAC,CAAC,CACTmB,IAAI,CAACnB,OAAO,IAAI;QACfqC,GAAG,CAACsB,SAAS,CAAC,gBAAgB,EAAEC,MAAM,CAACC,UAAU,CAAC7D,OAAO,EAAE,MAAM,CAAC,CAAC;QACnEqC,GAAG,CAACe,GAAG,CAACpD,OAAO,EAAE,MAAM,CAAC;MAC1B,CAAC,CAAC,CACDe,KAAK,CAACkB,MAAM,CAAC;IAClB,CAAC,CAAC;EACJ,CAAC;;EAED;AACF;AACA;AACA;AACA;AACA;AACA;EACE6B,iBAAiB,GAAGA,CAACC,MAAM,GAAG,EAAE,KAAK;IACnC,OAAO,CAAC,GAAG5F,MAAM,CAACC,IAAI,CAAC,IAAI,CAAClB,aAAa,CAAC,EAAE,GAAG6G,MAAM,CAAC;EACxD,CAAC;EAEDC,gBAAgB,GAAGA,CAACD,MAAM,GAAG,EAAE,KAAK;IAClC,OAAO,IAAI,CAAC7G,aAAa;EAC3B,CAAC;EAEDU,mBAAmB,GAAGA,CAAA,KAAM;IAC1BvC,OAAO,CAAC6H,EAAE,CAAC,QAAQ,EAAE,IAAI,CAAC5B,OAAO,CAAC;IAClCjG,OAAO,CAAC6H,EAAE,CAAC,SAAS,EAAE,IAAI,CAAC5B,OAAO,CAAC;EACrC,CAAC;;EAED;;EAEA,IAAI2C,cAAcA,CAAA,EAAG;IACnB,OAAO,IAAI,CAACrI,OAAO;EACrB;EAEA,IAAImD,gBAAgBA,CAAA,EAAG;IACrB,OAAO,IAAI,CAACjD,SAAS;EACvB;EAEA,IAAIoI,QAAQA,CAAA,EAAG;IACb,OAAO,IAAI,CAACpH,SAAS;EACvB;EAEA,MAAMqH,kBAAkBA,CAAA,EAAG;IACzB,OAAO,IAAI,CAACrH,SAAS,CAACkD,OAAO,CAAC,CAAC;EACjC;EAEAoE,iBAAiBA,CAAA,EAAG;IAClB,OAAO,OAAO/B,GAAG,EAAEW,GAAG,KAAK;MACzB,IAAI;QACF,MAAMhD,OAAO,GAAG,MAAM,IAAI,CAACmE,kBAAkB,CAAC,CAAC;QAC/CnB,GAAG,CAACpD,GAAG,CAAC,cAAc,EAAEhF,MAAM,CAACqC,QAAQ,CAACyG,WAAW,CAAC;QACpDV,GAAG,CAACI,GAAG,CAACpD,OAAO,CAAC;MAClB,CAAC,CAAC,OAAOH,GAAG,EAAE;QACZb,OAAO,CAACc,KAAK,CAAC,GAAG,IAAI,CAACjD,UAAU,yBAAyB,EAAEgD,GAAG,CAAC;QAC/DmD,GAAG,CAACqB,MAAM,CAAC,GAAG,CAAC,CAACjB,GAAG,CAAC,2BAA2B,CAAC;MAClD;IACF,CAAC;EACH;AACF;AAEAkB,MAAM,CAACC,OAAO,GAAG;EAAEtJ;AAAkB,CAAC","ignoreList":[]}
|
|
1
|
+
{"version":3,"file":"baseMetricsClient.js","names":["client","require","https","http","URL","BaseMetricsClient","constructor","config","appName","process","env","BUILD_APP_NAME","dynoId","HOSTNAME","processType","BUILD_DYNO_PROCESS_TYPE","includeNodeDefaultMetrics","enabled","METRICS_ENABLED","logValues","METRICS_LOG_VALUES","pushgatewayUrl","METRICS_PUSHGATEWAY_URL","authToken","pushgatewaySecret","METRICS_PUSHGATEWAY_SECRET","intervalSec","parseInt","METRICS_INTERVAL_SEC","startupValidation","disablePushgateway","METRICS_DISABLE_PUSHGATEWAY","removeOldMetrics","METRICS_REMOVE_OLD_METRICS","prefixLogs","_registry","Registry","collectDefaultMetrics","register","defaultLabels","app","dyno_id","process_type","defaultLabelsWithoutDynoId","gateway","gauges","counters","countersFunctions","gaugeUpdaters","_clearOldWorkers","_setCleanupHandlers","keepProcessAliveWhenDisabled","createGauge","name","help","updateFn","labelNames","Object","keys","g","Gauge","registers","createCounter","useLabelsWithoutDynoId","defaultSet","c","Counter","data","value","inc","clearAllCounters","metricsLogValues","console","log","values","forEach","counter","reset","_pushMetrics","entries","result","val","Promise","undefined","set","err","error","gatewayPush","metrics","getMetricsAsJSON","JSON","stringify","_startPush","interval","customPushMetics","warn","_idleInterval","setInterval","clearInterval","runPush","resolve","pushMetrics","catch","pushOrigin","trim","origin","startPush","cleanup","gatewayDelete","exit","_deleteFromVMByLabels","message","esc","s","String","replace","selector","u","reject","Error","path","encodeURIComponent","req","protocol","request","hostname","port","method","headers","Authorization","agent","Agent","keepAlive","res","statusCode","on","chunk","end","params","_pushToVMAgent","pushUrl","pathname","search","contentType","then","setHeader","Buffer","byteLength","withDefaultLabels","labels","withDefaultLabelsWithoutDynoId","getDefaultLabels","metricsEnabled","registry","getMetricsAsString","metricsMiddleware","status","module","exports"],"sources":["../../src/metrics/baseMetricsClient.js"],"sourcesContent":["const client = require('prom-client')\nconst https = require('https')\nconst http = require('http')\nconst { URL } = require('url')\n\n/**\n * BaseMetricsClient provides common functionality for all metrics clients.\n * Handles registry setup, push to remote (VM-agent), default labels, and common operations.\n * Always pushes registry to the configured URL (POST Prometheus text format + Basic auth). No Pushgateway.\n */\nclass BaseMetricsClient {\n /**\n * @param {Object} config\n * @param {string} [config.appName] Name of the application\n * @param {string} [config.dynoId] Dyno/instance ID\n * @param {string} [config.processType] Process type (web, worker, etc.)\n * @param {boolean} [config.enabled] Enable metrics collection\n * @param {boolean} [config.logValues] Log metrics values to console\n * @param {string} [config.pushgatewayUrl] Push URL (VM-agent import endpoint, e.g. .../api/v1/import/prometheus). /metrics is for GET (scrape), not POST (push).\n * @param {string} [config.pushgatewaySecret] Basic auth secret (Base64 of user:password)\n * @param {number} [config.intervalSec] Interval in seconds for pushing metrics\n * @param {boolean} [config.removeOldMetrics] Enable to clear metrics by service name\n * @param {function} [config.startupValidation] Add to validate on start push.\n * @param {boolean} [config.disablePushgateway] Disable pushing to Pushgateway (use HTTP scraping instead)\n * @param {boolean} [config.includeNodeDefaultMetrics] Register prom-client default process metrics (heap, etc.); defaults true unless MetricsClient forces false for web+Redis HTTP recording dynos.\n */\n constructor(config = {}) {\n this.appName = config.appName || process.env.BUILD_APP_NAME || 'unknown-app'\n this.dynoId = config.dynoId || process.env.HOSTNAME || 'unknown-dyno'\n this.processType =\n config.processType ||\n process.env.BUILD_DYNO_PROCESS_TYPE ||\n 'undefined_build_dyno_type'\n this.includeNodeDefaultMetrics =\n config.includeNodeDefaultMetrics !== false\n this.enabled = config.enabled ?? process.env.METRICS_ENABLED === 'true'\n this.logValues =\n config.logValues ?? process.env.METRICS_LOG_VALUES === 'true'\n this.pushgatewayUrl =\n config.pushgatewayUrl || process.env.METRICS_PUSHGATEWAY_URL || ''\n this.authToken =\n config.pushgatewaySecret || process.env.METRICS_PUSHGATEWAY_SECRET || ''\n this.intervalSec =\n config.intervalSec ||\n parseInt(process.env.METRICS_INTERVAL_SEC || '', 10) ||\n 15\n this.startupValidation = config.startupValidation\n this.disablePushgateway =\n config.disablePushgateway ??\n process.env.METRICS_DISABLE_PUSHGATEWAY === 'true'\n this.removeOldMetrics =\n config.removeOldMetrics ??\n process.env.METRICS_REMOVE_OLD_METRICS === 'true'\n\n this.prefixLogs = `[${this.processType}] [${this.appName}] [${this.dynoId}] [Monitoring]`\n\n this._registry = new client.Registry()\n if (this.includeNodeDefaultMetrics) {\n client.collectDefaultMetrics({ register: this._registry })\n }\n\n this.defaultLabels = {\n app: this.appName,\n dyno_id: this.dynoId,\n process_type: this.processType,\n }\n\n /** Default labels without dyno_id (for HTTP metrics so dyno_id is not included). */\n this.defaultLabelsWithoutDynoId = {\n app: this.appName,\n process_type: this.processType,\n }\n\n // Always push to configured URL (VM-agent). No Pushgateway.\n this.gateway = null\n this.gauges = {}\n this.counters = {}\n this.countersFunctions = {}\n\n /** @type {Object<string, function(): number | Promise<number>>} */\n this.gaugeUpdaters = {}\n\n this._clearOldWorkers(config.removeOldMetrics)\n this._setCleanupHandlers()\n\n this.keepProcessAliveWhenDisabled = true\n }\n\n /**\n * Create a gauge metric.\n * @param {Object} options - Gauge configuration\n * @param {string} options.name - Name of the gauge\n * @param {string} options.help - Help text describing the gauge\n * @param {function(): number|Promise<number>} [options.updateFn] - Optional function returning the gauge value\n * @param {string[]} [options.labelNames] - Optional custom label names\n * @returns {import('prom-client').Gauge} The created Prometheus gauge\n */\n createGauge = ({\n name,\n help,\n updateFn,\n labelNames = Object.keys(this.defaultLabels),\n }) => {\n if (this.gauges[name]) return this.gauges[name]\n\n const g = new client.Gauge({\n name,\n help,\n labelNames,\n registers: [this._registry],\n })\n this.gauges[name] = g\n\n if (updateFn && typeof updateFn === 'function') {\n this.gaugeUpdaters[name] = updateFn\n }\n\n return g\n }\n\n /**\n * Create a Prometheus Counter metric.\n *\n * @param {Object} params - Counter configuration\n * @param {string} params.name - Metric name\n * @param {string} params.help - Metric description\n * @param {string[]} [params.labelNames] - Optional list of label names. Defaults to this.defaultLabels keys.\n * @param {boolean} [params.useLabelsWithoutDynoId=false] - If true, counter uses app/process_type only (no dyno_id). Use for HTTP metrics.\n *\n * @returns {(labels?: Object, incrementValue?: number) => void}\n * A function to increment the counter.\n * Usage: (labels?, incrementValue?)\n */\n createCounter({\n name,\n help,\n labelNames = Object.keys(this.defaultLabels),\n useLabelsWithoutDynoId = false,\n }) {\n if (this.counters[name]) return this.countersFunctions[name]\n\n const defaultSet = useLabelsWithoutDynoId\n ? this.defaultLabelsWithoutDynoId\n : this.defaultLabels\n\n const c = new client.Counter({\n name,\n help,\n labelNames,\n registers: [this._registry],\n })\n this.counters[name] = c\n\n this.countersFunctions = {\n ...this.countersFunctions,\n [name]: (data = {}, value = 1) => {\n c.inc({ ...defaultSet, ...data }, value)\n },\n }\n\n return this.countersFunctions[name]\n }\n\n /**\n * Clear all collected counters\n */\n clearAllCounters = () => {\n if (this.metricsLogValues) {\n console.log('Counters to clear: ', Object.keys(this.counters))\n }\n Object.values(this.counters).forEach(counter => counter.reset())\n }\n\n /**\n * Push all gauges and counters to PushGateway and optionally log.\n */\n _pushMetrics = async () => {\n try {\n for (const [name, updateFn] of Object.entries(this.gaugeUpdaters)) {\n try {\n if (!updateFn) {\n continue\n }\n const result = updateFn()\n const val = result instanceof Promise ? await result : result\n if (val !== undefined) this.gauges[name].set(this.defaultLabels, val)\n } catch (err) {\n console.error(\n `${this.prefixLogs} Failed to update gauge ${name}:`,\n err\n )\n }\n }\n\n if (!this.disablePushgateway) {\n await this.gatewayPush()\n }\n // this.clearAllCounters() //TODO: or uncommit or delete (based on grafana expectation)\n\n if (this.logValues) {\n const metrics = await this._registry.getMetricsAsJSON()\n console.log(\n `${this.prefixLogs} Metrics:\\n`,\n JSON.stringify(metrics, null, 2)\n )\n }\n } catch (err) {\n console.error(`${this.prefixLogs} Failed to push metrics:`, err)\n }\n }\n\n _startPush = (interval = this.intervalSec, customPushMetics = undefined) => {\n if (!this.enabled) {\n console.warn(`${this.prefixLogs} Metrics disabled`)\n if (this.keepProcessAliveWhenDisabled && !this._idleInterval) {\n this._idleInterval = setInterval(() => {}, 60 * 60 * 1000)\n }\n return\n }\n\n if (this._idleInterval) {\n clearInterval(this._idleInterval)\n this._idleInterval = null\n }\n\n if (this.startupValidation && !this.startupValidation()) {\n return\n }\n\n const runPush = () => {\n if (customPushMetics && typeof customPushMetics === 'function') {\n return Promise.resolve(customPushMetics())\n }\n return this.pushMetrics()\n }\n\n if (customPushMetics && typeof customPushMetics === 'function') {\n setInterval(() => customPushMetics(), interval * 1000)\n } else {\n setInterval(() => {\n runPush().catch(err => {\n console.error(`${this.prefixLogs} Failed to push metrics:`, err)\n })\n }, interval * 1000)\n }\n\n // First push immediately so metrics appear without waiting for the first interval\n runPush().catch(err => {\n console.error(`${this.prefixLogs} Failed to push metrics (initial):`, err)\n })\n\n let pushOrigin = 'none'\n try {\n if (this.pushgatewayUrl && this.pushgatewayUrl.trim()) {\n pushOrigin = new URL(this.pushgatewayUrl.trim()).origin\n }\n } catch {\n pushOrigin = 'invalid URL'\n }\n console.warn(\n `${this.prefixLogs} Metrics collection started. (interval: ${this.intervalSec}s, push: ${pushOrigin})`\n )\n }\n\n pushMetrics = async () => {\n return this._pushMetrics()\n }\n\n /**\n * Start periodic metrics collection and push.\n *\n * This method wraps the internal `_startPush` method.\n * If a `customPushMetrics` function is provided, it will be executed\n * at the given interval instead of the default `pushMetrics` behavior.\n *\n * @param {number} [interval=this.intervalSec] - Interval in seconds between pushes.\n * @param {() => void | Promise<void>} [customPushMetrics] - Optional custom push function. If provided, Prometheus push is skipped.\n */\n startPush = (interval, customPushMetics = undefined) => {\n this._startPush(interval, customPushMetics)\n }\n\n /**\n * Cleanup metrics and exit process.\n * @returns {Promise<void>}\n */\n cleanup = async () => {\n if (this.enabled) {\n await this.gatewayDelete()\n }\n process.exit(0)\n }\n\n /**\n * Remove old/stale dyno/instance metrics from PushGateway.\n *\n * Compares existing PushGateway metrics for this job and deletes any instances\n * that do not match the current dynoId.\n *\n * @param {boolean} removeOldMetrics If true, performs cleanup; otherwise does nothing\n * @returns {Promise<void>}\n * @private\n */\n _clearOldWorkers = async removeOldMetrics => {\n // No Pushgateway; VM-agent does not support per-instance delete. Skip.\n }\n\n /**\n * On shutdown: optionally delete this instance's metrics from VictoriaMetrics (by app, dyno_id, process_type).\n * @returns {Promise<void>}\n */\n gatewayDelete = async () => {\n if (\n this.removeOldMetrics &&\n this.pushgatewayUrl &&\n this.pushgatewayUrl.trim()\n ) {\n await this._deleteFromVMByLabels().catch(err => {\n console.warn(\n `${this.prefixLogs} Deletion from VM on shutdown failed:`,\n err.message\n )\n })\n }\n }\n\n /**\n * Call VictoriaMetrics delete_series API to remove all series matching this instance's labels (app, dyno_id, process_type).\n * @private\n */\n _deleteFromVMByLabels = () => {\n const esc = s => String(s).replace(/\\\\/g, '\\\\\\\\').replace(/\"/g, '\\\\\"')\n const selector = `{app=\"${esc(this.appName)}\",dyno_id=\"${esc(\n this.dynoId\n )}\",process_type=\"${esc(this.processType)}\"}`\n let origin\n try {\n const u = new URL((this.pushgatewayUrl || '').trim())\n origin = u.origin\n } catch {\n return Promise.reject(new Error('Invalid push URL'))\n }\n const path = `/api/v1/admin/tsdb/delete_series?match[]=${encodeURIComponent(\n selector\n )}`\n return new Promise((resolve, reject) => {\n const u = new URL(origin)\n const req = (u.protocol === 'https:' ? https : http).request(\n {\n hostname: u.hostname,\n port: u.port || (u.protocol === 'https:' ? 443 : 80),\n path,\n method: 'POST',\n headers: {\n 'Content-Length': '0',\n Authorization: this.authToken\n ? `Basic ${this.authToken}`\n : undefined,\n },\n agent:\n u.protocol === 'https:'\n ? new https.Agent({ keepAlive: false })\n : undefined,\n },\n res => {\n if (res.statusCode >= 200 && res.statusCode < 300) resolve()\n else {\n let data = ''\n res.on('data', chunk => {\n data += chunk\n })\n res.on('end', () =>\n reject(new Error(`Delete failed: ${res.statusCode} ${data}`))\n )\n }\n }\n )\n req.on('error', reject)\n req.end()\n })\n }\n\n /**\n * Push registry to configured URL (VM-agent). POST Prometheus text format + Basic auth.\n *\n * @param {object} [params] Unused; kept for API compatibility.\n * @returns {Promise<void>}\n */\n gatewayPush = async (params = {}) => {\n if (this.disablePushgateway) {\n console.warn(\n `${this.prefixLogs} Metrics push skipped: METRICS_DISABLE_PUSHGATEWAY is set`\n )\n return Promise.resolve()\n }\n if (!this.pushgatewayUrl || !this.pushgatewayUrl.trim()) {\n console.warn(\n `${this.prefixLogs} Metrics push skipped: METRICS_PUSHGATEWAY_URL is not set`\n )\n return Promise.resolve()\n }\n return this._pushToVMAgent()\n }\n\n /**\n * POST registry (Prometheus text format) to VM-agent. VM-agent accepts push at /api/v1/import/prometheus; /metrics is GET (scrape) only.\n * @private\n */\n _pushToVMAgent = () => {\n let pushUrl = (this.pushgatewayUrl || '').trim()\n try {\n const u = new URL(pushUrl)\n if (!u.pathname || u.pathname === '/' || u.pathname === '/metrics') {\n pushUrl = `${u.origin}/api/v1/import/prometheus${u.search}`\n }\n } catch {\n // leave pushUrl as-is\n }\n return new Promise((resolve, reject) => {\n const u = new URL(pushUrl)\n const req = (u.protocol === 'https:' ? https : http).request(\n {\n hostname: u.hostname,\n port: u.port || (u.protocol === 'https:' ? 443 : 80),\n path: u.pathname + u.search,\n method: 'POST',\n headers: {\n 'Content-Type': client.register.contentType,\n Authorization: this.authToken\n ? `Basic ${this.authToken}`\n : undefined,\n },\n agent:\n u.protocol === 'https:'\n ? new https.Agent({ keepAlive: true })\n : undefined,\n },\n res => {\n if (res.statusCode >= 200 && res.statusCode < 300) {\n resolve()\n } else {\n let data = ''\n res.on('data', chunk => {\n data += chunk\n })\n res.on('end', () =>\n reject(new Error(`Push failed: ${res.statusCode} ${data}`))\n )\n }\n }\n )\n req.on('error', reject)\n this._registry\n .metrics()\n .then(metrics => {\n req.setHeader('Content-Length', Buffer.byteLength(metrics, 'utf8'))\n req.end(metrics, 'utf8')\n })\n .catch(reject)\n })\n }\n\n /**\n * Merge the default metric labels (`app`, `dyno_id`, `process_type`)\n * with custom label names.\n *\n * @param {string[]} labels Additional label names\n * @returns {string[]} Combined label names\n */\n withDefaultLabels = (labels = []) => {\n return [...Object.keys(this.defaultLabels), ...labels]\n }\n\n /**\n * Merge default labels without dyno_id (`app`, `process_type`) with custom label names.\n * Use for HTTP metrics so dyno_id is not included.\n *\n * @param {string[]} labels Additional label names\n * @returns {string[]} Combined label names\n */\n withDefaultLabelsWithoutDynoId = (labels = []) => {\n return [...Object.keys(this.defaultLabelsWithoutDynoId), ...labels]\n }\n\n getDefaultLabels = (labels = []) => {\n return this.defaultLabels\n }\n\n _setCleanupHandlers = () => {\n process.on('SIGINT', this.cleanup)\n process.on('SIGTERM', this.cleanup)\n }\n\n // GETTERS\n\n get metricsEnabled() {\n return this.enabled\n }\n\n get metricsLogValues() {\n return this.logValues\n }\n\n get registry() {\n return this._registry\n }\n\n async getMetricsAsString() {\n return this._registry.metrics()\n }\n\n metricsMiddleware() {\n return async (req, res) => {\n try {\n const metrics = await this.getMetricsAsString()\n res.set('Content-Type', client.register.contentType)\n res.end(metrics)\n } catch (err) {\n console.error(`${this.prefixLogs} Failed to get metrics:`, err)\n res.status(500).end('Failed to collect metrics')\n }\n }\n }\n}\n\nmodule.exports = { BaseMetricsClient }\n"],"mappings":";;AAAA,MAAMA,MAAM,GAAGC,OAAO,CAAC,aAAa,CAAC;AACrC,MAAMC,KAAK,GAAGD,OAAO,CAAC,OAAO,CAAC;AAC9B,MAAME,IAAI,GAAGF,OAAO,CAAC,MAAM,CAAC;AAC5B,MAAM;EAAEG;AAAI,CAAC,GAAGH,OAAO,CAAC,KAAK,CAAC;;AAE9B;AACA;AACA;AACA;AACA;AACA,MAAMI,iBAAiB,CAAC;EACtB;AACF;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;EACEC,WAAWA,CAACC,MAAM,GAAG,CAAC,CAAC,EAAE;IACvB,IAAI,CAACC,OAAO,GAAGD,MAAM,CAACC,OAAO,IAAIC,OAAO,CAACC,GAAG,CAACC,cAAc,IAAI,aAAa;IAC5E,IAAI,CAACC,MAAM,GAAGL,MAAM,CAACK,MAAM,IAAIH,OAAO,CAACC,GAAG,CAACG,QAAQ,IAAI,cAAc;IACrE,IAAI,CAACC,WAAW,GACdP,MAAM,CAACO,WAAW,IAClBL,OAAO,CAACC,GAAG,CAACK,uBAAuB,IACnC,2BAA2B;IAC7B,IAAI,CAACC,yBAAyB,GAC5BT,MAAM,CAACS,yBAAyB,KAAK,KAAK;IAC5C,IAAI,CAACC,OAAO,GAAGV,MAAM,CAACU,OAAO,IAAIR,OAAO,CAACC,GAAG,CAACQ,eAAe,KAAK,MAAM;IACvE,IAAI,CAACC,SAAS,GACZZ,MAAM,CAACY,SAAS,IAAIV,OAAO,CAACC,GAAG,CAACU,kBAAkB,KAAK,MAAM;IAC/D,IAAI,CAACC,cAAc,GACjBd,MAAM,CAACc,cAAc,IAAIZ,OAAO,CAACC,GAAG,CAACY,uBAAuB,IAAI,EAAE;IACpE,IAAI,CAACC,SAAS,GACZhB,MAAM,CAACiB,iBAAiB,IAAIf,OAAO,CAACC,GAAG,CAACe,0BAA0B,IAAI,EAAE;IAC1E,IAAI,CAACC,WAAW,GACdnB,MAAM,CAACmB,WAAW,IAClBC,QAAQ,CAAClB,OAAO,CAACC,GAAG,CAACkB,oBAAoB,IAAI,EAAE,EAAE,EAAE,CAAC,IACpD,EAAE;IACJ,IAAI,CAACC,iBAAiB,GAAGtB,MAAM,CAACsB,iBAAiB;IACjD,IAAI,CAACC,kBAAkB,GACrBvB,MAAM,CAACuB,kBAAkB,IACzBrB,OAAO,CAACC,GAAG,CAACqB,2BAA2B,KAAK,MAAM;IACpD,IAAI,CAACC,gBAAgB,GACnBzB,MAAM,CAACyB,gBAAgB,IACvBvB,OAAO,CAACC,GAAG,CAACuB,0BAA0B,KAAK,MAAM;IAEnD,IAAI,CAACC,UAAU,GAAG,IAAI,IAAI,CAACpB,WAAW,MAAM,IAAI,CAACN,OAAO,MAAM,IAAI,CAACI,MAAM,gBAAgB;IAEzF,IAAI,CAACuB,SAAS,GAAG,IAAInC,MAAM,CAACoC,QAAQ,CAAC,CAAC;IACtC,IAAI,IAAI,CAACpB,yBAAyB,EAAE;MAClChB,MAAM,CAACqC,qBAAqB,CAAC;QAAEC,QAAQ,EAAE,IAAI,CAACH;MAAU,CAAC,CAAC;IAC5D;IAEA,IAAI,CAACI,aAAa,GAAG;MACnBC,GAAG,EAAE,IAAI,CAAChC,OAAO;MACjBiC,OAAO,EAAE,IAAI,CAAC7B,MAAM;MACpB8B,YAAY,EAAE,IAAI,CAAC5B;IACrB,CAAC;;IAED;IACA,IAAI,CAAC6B,0BAA0B,GAAG;MAChCH,GAAG,EAAE,IAAI,CAAChC,OAAO;MACjBkC,YAAY,EAAE,IAAI,CAAC5B;IACrB,CAAC;;IAED;IACA,IAAI,CAAC8B,OAAO,GAAG,IAAI;IACnB,IAAI,CAACC,MAAM,GAAG,CAAC,CAAC;IAChB,IAAI,CAACC,QAAQ,GAAG,CAAC,CAAC;IAClB,IAAI,CAACC,iBAAiB,GAAG,CAAC,CAAC;;IAE3B;IACA,IAAI,CAACC,aAAa,GAAG,CAAC,CAAC;IAEvB,IAAI,CAACC,gBAAgB,CAAC1C,MAAM,CAACyB,gBAAgB,CAAC;IAC9C,IAAI,CAACkB,mBAAmB,CAAC,CAAC;IAE1B,IAAI,CAACC,4BAA4B,GAAG,IAAI;EAC1C;;EAEA;AACF;AACA;AACA;AACA;AACA;AACA;AACA;AACA;EACEC,WAAW,GAAGA,CAAC;IACbC,IAAI;IACJC,IAAI;IACJC,QAAQ;IACRC,UAAU,GAAGC,MAAM,CAACC,IAAI,CAAC,IAAI,CAACnB,aAAa;EAC7C,CAAC,KAAK;IACJ,IAAI,IAAI,CAACM,MAAM,CAACQ,IAAI,CAAC,EAAE,OAAO,IAAI,CAACR,MAAM,CAACQ,IAAI,CAAC;IAE/C,MAAMM,CAAC,GAAG,IAAI3D,MAAM,CAAC4D,KAAK,CAAC;MACzBP,IAAI;MACJC,IAAI;MACJE,UAAU;MACVK,SAAS,EAAE,CAAC,IAAI,CAAC1B,SAAS;IAC5B,CAAC,CAAC;IACF,IAAI,CAACU,MAAM,CAACQ,IAAI,CAAC,GAAGM,CAAC;IAErB,IAAIJ,QAAQ,IAAI,OAAOA,QAAQ,KAAK,UAAU,EAAE;MAC9C,IAAI,CAACP,aAAa,CAACK,IAAI,CAAC,GAAGE,QAAQ;IACrC;IAEA,OAAOI,CAAC;EACV,CAAC;;EAED;AACF;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;EACEG,aAAaA,CAAC;IACZT,IAAI;IACJC,IAAI;IACJE,UAAU,GAAGC,MAAM,CAACC,IAAI,CAAC,IAAI,CAACnB,aAAa,CAAC;IAC5CwB,sBAAsB,GAAG;EAC3B,CAAC,EAAE;IACD,IAAI,IAAI,CAACjB,QAAQ,CAACO,IAAI,CAAC,EAAE,OAAO,IAAI,CAACN,iBAAiB,CAACM,IAAI,CAAC;IAE5D,MAAMW,UAAU,GAAGD,sBAAsB,GACrC,IAAI,CAACpB,0BAA0B,GAC/B,IAAI,CAACJ,aAAa;IAEtB,MAAM0B,CAAC,GAAG,IAAIjE,MAAM,CAACkE,OAAO,CAAC;MAC3Bb,IAAI;MACJC,IAAI;MACJE,UAAU;MACVK,SAAS,EAAE,CAAC,IAAI,CAAC1B,SAAS;IAC5B,CAAC,CAAC;IACF,IAAI,CAACW,QAAQ,CAACO,IAAI,CAAC,GAAGY,CAAC;IAEvB,IAAI,CAAClB,iBAAiB,GAAG;MACvB,GAAG,IAAI,CAACA,iBAAiB;MACzB,CAACM,IAAI,GAAG,CAACc,IAAI,GAAG,CAAC,CAAC,EAAEC,KAAK,GAAG,CAAC,KAAK;QAChCH,CAAC,CAACI,GAAG,CAAC;UAAE,GAAGL,UAAU;UAAE,GAAGG;QAAK,CAAC,EAAEC,KAAK,CAAC;MAC1C;IACF,CAAC;IAED,OAAO,IAAI,CAACrB,iBAAiB,CAACM,IAAI,CAAC;EACrC;;EAEA;AACF;AACA;EACEiB,gBAAgB,GAAGA,CAAA,KAAM;IACvB,IAAI,IAAI,CAACC,gBAAgB,EAAE;MACzBC,OAAO,CAACC,GAAG,CAAC,qBAAqB,EAAEhB,MAAM,CAACC,IAAI,CAAC,IAAI,CAACZ,QAAQ,CAAC,CAAC;IAChE;IACAW,MAAM,CAACiB,MAAM,CAAC,IAAI,CAAC5B,QAAQ,CAAC,CAAC6B,OAAO,CAACC,OAAO,IAAIA,OAAO,CAACC,KAAK,CAAC,CAAC,CAAC;EAClE,CAAC;;EAED;AACF;AACA;EACEC,YAAY,GAAG,MAAAA,CAAA,KAAY;IACzB,IAAI;MACF,KAAK,MAAM,CAACzB,IAAI,EAAEE,QAAQ,CAAC,IAAIE,MAAM,CAACsB,OAAO,CAAC,IAAI,CAAC/B,aAAa,CAAC,EAAE;QACjE,IAAI;UACF,IAAI,CAACO,QAAQ,EAAE;YACb;UACF;UACA,MAAMyB,MAAM,GAAGzB,QAAQ,CAAC,CAAC;UACzB,MAAM0B,GAAG,GAAGD,MAAM,YAAYE,OAAO,GAAG,MAAMF,MAAM,GAAGA,MAAM;UAC7D,IAAIC,GAAG,KAAKE,SAAS,EAAE,IAAI,CAACtC,MAAM,CAACQ,IAAI,CAAC,CAAC+B,GAAG,CAAC,IAAI,CAAC7C,aAAa,EAAE0C,GAAG,CAAC;QACvE,CAAC,CAAC,OAAOI,GAAG,EAAE;UACZb,OAAO,CAACc,KAAK,CACX,GAAG,IAAI,CAACpD,UAAU,2BAA2BmB,IAAI,GAAG,EACpDgC,GACF,CAAC;QACH;MACF;MAEA,IAAI,CAAC,IAAI,CAACvD,kBAAkB,EAAE;QAC5B,MAAM,IAAI,CAACyD,WAAW,CAAC,CAAC;MAC1B;MACA;;MAEA,IAAI,IAAI,CAACpE,SAAS,EAAE;QAClB,MAAMqE,OAAO,GAAG,MAAM,IAAI,CAACrD,SAAS,CAACsD,gBAAgB,CAAC,CAAC;QACvDjB,OAAO,CAACC,GAAG,CACT,GAAG,IAAI,CAACvC,UAAU,aAAa,EAC/BwD,IAAI,CAACC,SAAS,CAACH,OAAO,EAAE,IAAI,EAAE,CAAC,CACjC,CAAC;MACH;IACF,CAAC,CAAC,OAAOH,GAAG,EAAE;MACZb,OAAO,CAACc,KAAK,CAAC,GAAG,IAAI,CAACpD,UAAU,0BAA0B,EAAEmD,GAAG,CAAC;IAClE;EACF,CAAC;EAEDO,UAAU,GAAGA,CAACC,QAAQ,GAAG,IAAI,CAACnE,WAAW,EAAEoE,gBAAgB,GAAGX,SAAS,KAAK;IAC1E,IAAI,CAAC,IAAI,CAAClE,OAAO,EAAE;MACjBuD,OAAO,CAACuB,IAAI,CAAC,GAAG,IAAI,CAAC7D,UAAU,mBAAmB,CAAC;MACnD,IAAI,IAAI,CAACiB,4BAA4B,IAAI,CAAC,IAAI,CAAC6C,aAAa,EAAE;QAC5D,IAAI,CAACA,aAAa,GAAGC,WAAW,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;MAC5D;MACA;IACF;IAEA,IAAI,IAAI,CAACD,aAAa,EAAE;MACtBE,aAAa,CAAC,IAAI,CAACF,aAAa,CAAC;MACjC,IAAI,CAACA,aAAa,GAAG,IAAI;IAC3B;IAEA,IAAI,IAAI,CAACnE,iBAAiB,IAAI,CAAC,IAAI,CAACA,iBAAiB,CAAC,CAAC,EAAE;MACvD;IACF;IAEA,MAAMsE,OAAO,GAAGA,CAAA,KAAM;MACpB,IAAIL,gBAAgB,IAAI,OAAOA,gBAAgB,KAAK,UAAU,EAAE;QAC9D,OAAOZ,OAAO,CAACkB,OAAO,CAACN,gBAAgB,CAAC,CAAC,CAAC;MAC5C;MACA,OAAO,IAAI,CAACO,WAAW,CAAC,CAAC;IAC3B,CAAC;IAED,IAAIP,gBAAgB,IAAI,OAAOA,gBAAgB,KAAK,UAAU,EAAE;MAC9DG,WAAW,CAAC,MAAMH,gBAAgB,CAAC,CAAC,EAAED,QAAQ,GAAG,IAAI,CAAC;IACxD,CAAC,MAAM;MACLI,WAAW,CAAC,MAAM;QAChBE,OAAO,CAAC,CAAC,CAACG,KAAK,CAACjB,GAAG,IAAI;UACrBb,OAAO,CAACc,KAAK,CAAC,GAAG,IAAI,CAACpD,UAAU,0BAA0B,EAAEmD,GAAG,CAAC;QAClE,CAAC,CAAC;MACJ,CAAC,EAAEQ,QAAQ,GAAG,IAAI,CAAC;IACrB;;IAEA;IACAM,OAAO,CAAC,CAAC,CAACG,KAAK,CAACjB,GAAG,IAAI;MACrBb,OAAO,CAACc,KAAK,CAAC,GAAG,IAAI,CAACpD,UAAU,oCAAoC,EAAEmD,GAAG,CAAC;IAC5E,CAAC,CAAC;IAEF,IAAIkB,UAAU,GAAG,MAAM;IACvB,IAAI;MACF,IAAI,IAAI,CAAClF,cAAc,IAAI,IAAI,CAACA,cAAc,CAACmF,IAAI,CAAC,CAAC,EAAE;QACrDD,UAAU,GAAG,IAAInG,GAAG,CAAC,IAAI,CAACiB,cAAc,CAACmF,IAAI,CAAC,CAAC,CAAC,CAACC,MAAM;MACzD;IACF,CAAC,CAAC,MAAM;MACNF,UAAU,GAAG,aAAa;IAC5B;IACA/B,OAAO,CAACuB,IAAI,CACV,GAAG,IAAI,CAAC7D,UAAU,2CAA2C,IAAI,CAACR,WAAW,YAAY6E,UAAU,GACrG,CAAC;EACH,CAAC;EAEDF,WAAW,GAAG,MAAAA,CAAA,KAAY;IACxB,OAAO,IAAI,CAACvB,YAAY,CAAC,CAAC;EAC5B,CAAC;;EAED;AACF;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;EACE4B,SAAS,GAAGA,CAACb,QAAQ,EAAEC,gBAAgB,GAAGX,SAAS,KAAK;IACtD,IAAI,CAACS,UAAU,CAACC,QAAQ,EAAEC,gBAAgB,CAAC;EAC7C,CAAC;;EAED;AACF;AACA;AACA;EACEa,OAAO,GAAG,MAAAA,CAAA,KAAY;IACpB,IAAI,IAAI,CAAC1F,OAAO,EAAE;MAChB,MAAM,IAAI,CAAC2F,aAAa,CAAC,CAAC;IAC5B;IACAnG,OAAO,CAACoG,IAAI,CAAC,CAAC,CAAC;EACjB,CAAC;;EAED;AACF;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;EACE5D,gBAAgB,GAAG,MAAMjB,gBAAgB,IAAI;IAC3C;EAAA,CACD;;EAED;AACF;AACA;AACA;EACE4E,aAAa,GAAG,MAAAA,CAAA,KAAY;IAC1B,IACE,IAAI,CAAC5E,gBAAgB,IACrB,IAAI,CAACX,cAAc,IACnB,IAAI,CAACA,cAAc,CAACmF,IAAI,CAAC,CAAC,EAC1B;MACA,MAAM,IAAI,CAACM,qBAAqB,CAAC,CAAC,CAACR,KAAK,CAACjB,GAAG,IAAI;QAC9Cb,OAAO,CAACuB,IAAI,CACV,GAAG,IAAI,CAAC7D,UAAU,uCAAuC,EACzDmD,GAAG,CAAC0B,OACN,CAAC;MACH,CAAC,CAAC;IACJ;EACF,CAAC;;EAED;AACF;AACA;AACA;EACED,qBAAqB,GAAGA,CAAA,KAAM;IAC5B,MAAME,GAAG,GAAGC,CAAC,IAAIC,MAAM,CAACD,CAAC,CAAC,CAACE,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,CAACA,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC;IACtE,MAAMC,QAAQ,GAAG,SAASJ,GAAG,CAAC,IAAI,CAACxG,OAAO,CAAC,cAAcwG,GAAG,CAC1D,IAAI,CAACpG,MACP,CAAC,mBAAmBoG,GAAG,CAAC,IAAI,CAAClG,WAAW,CAAC,IAAI;IAC7C,IAAI2F,MAAM;IACV,IAAI;MACF,MAAMY,CAAC,GAAG,IAAIjH,GAAG,CAAC,CAAC,IAAI,CAACiB,cAAc,IAAI,EAAE,EAAEmF,IAAI,CAAC,CAAC,CAAC;MACrDC,MAAM,GAAGY,CAAC,CAACZ,MAAM;IACnB,CAAC,CAAC,MAAM;MACN,OAAOvB,OAAO,CAACoC,MAAM,CAAC,IAAIC,KAAK,CAAC,kBAAkB,CAAC,CAAC;IACtD;IACA,MAAMC,IAAI,GAAG,4CAA4CC,kBAAkB,CACzEL,QACF,CAAC,EAAE;IACH,OAAO,IAAIlC,OAAO,CAAC,CAACkB,OAAO,EAAEkB,MAAM,KAAK;MACtC,MAAMD,CAAC,GAAG,IAAIjH,GAAG,CAACqG,MAAM,CAAC;MACzB,MAAMiB,GAAG,GAAG,CAACL,CAAC,CAACM,QAAQ,KAAK,QAAQ,GAAGzH,KAAK,GAAGC,IAAI,EAAEyH,OAAO,CAC1D;QACEC,QAAQ,EAAER,CAAC,CAACQ,QAAQ;QACpBC,IAAI,EAAET,CAAC,CAACS,IAAI,KAAKT,CAAC,CAACM,QAAQ,KAAK,QAAQ,GAAG,GAAG,GAAG,EAAE,CAAC;QACpDH,IAAI;QACJO,MAAM,EAAE,MAAM;QACdC,OAAO,EAAE;UACP,gBAAgB,EAAE,GAAG;UACrBC,aAAa,EAAE,IAAI,CAAC1G,SAAS,GACzB,SAAS,IAAI,CAACA,SAAS,EAAE,GACzB4D;QACN,CAAC;QACD+C,KAAK,EACHb,CAAC,CAACM,QAAQ,KAAK,QAAQ,GACnB,IAAIzH,KAAK,CAACiI,KAAK,CAAC;UAAEC,SAAS,EAAE;QAAM,CAAC,CAAC,GACrCjD;MACR,CAAC,EACDkD,GAAG,IAAI;QACL,IAAIA,GAAG,CAACC,UAAU,IAAI,GAAG,IAAID,GAAG,CAACC,UAAU,GAAG,GAAG,EAAElC,OAAO,CAAC,CAAC,MACvD;UACH,IAAIjC,IAAI,GAAG,EAAE;UACbkE,GAAG,CAACE,EAAE,CAAC,MAAM,EAAEC,KAAK,IAAI;YACtBrE,IAAI,IAAIqE,KAAK;UACf,CAAC,CAAC;UACFH,GAAG,CAACE,EAAE,CAAC,KAAK,EAAE,MACZjB,MAAM,CAAC,IAAIC,KAAK,CAAC,kBAAkBc,GAAG,CAACC,UAAU,IAAInE,IAAI,EAAE,CAAC,CAC9D,CAAC;QACH;MACF,CACF,CAAC;MACDuD,GAAG,CAACa,EAAE,CAAC,OAAO,EAAEjB,MAAM,CAAC;MACvBI,GAAG,CAACe,GAAG,CAAC,CAAC;IACX,CAAC,CAAC;EACJ,CAAC;;EAED;AACF;AACA;AACA;AACA;AACA;EACElD,WAAW,GAAG,MAAAA,CAAOmD,MAAM,GAAG,CAAC,CAAC,KAAK;IACnC,IAAI,IAAI,CAAC5G,kBAAkB,EAAE;MAC3B0C,OAAO,CAACuB,IAAI,CACV,GAAG,IAAI,CAAC7D,UAAU,2DACpB,CAAC;MACD,OAAOgD,OAAO,CAACkB,OAAO,CAAC,CAAC;IAC1B;IACA,IAAI,CAAC,IAAI,CAAC/E,cAAc,IAAI,CAAC,IAAI,CAACA,cAAc,CAACmF,IAAI,CAAC,CAAC,EAAE;MACvDhC,OAAO,CAACuB,IAAI,CACV,GAAG,IAAI,CAAC7D,UAAU,2DACpB,CAAC;MACD,OAAOgD,OAAO,CAACkB,OAAO,CAAC,CAAC;IAC1B;IACA,OAAO,IAAI,CAACuC,cAAc,CAAC,CAAC;EAC9B,CAAC;;EAED;AACF;AACA;AACA;EACEA,cAAc,GAAGA,CAAA,KAAM;IACrB,IAAIC,OAAO,GAAG,CAAC,IAAI,CAACvH,cAAc,IAAI,EAAE,EAAEmF,IAAI,CAAC,CAAC;IAChD,IAAI;MACF,MAAMa,CAAC,GAAG,IAAIjH,GAAG,CAACwI,OAAO,CAAC;MAC1B,IAAI,CAACvB,CAAC,CAACwB,QAAQ,IAAIxB,CAAC,CAACwB,QAAQ,KAAK,GAAG,IAAIxB,CAAC,CAACwB,QAAQ,KAAK,UAAU,EAAE;QAClED,OAAO,GAAG,GAAGvB,CAAC,CAACZ,MAAM,4BAA4BY,CAAC,CAACyB,MAAM,EAAE;MAC7D;IACF,CAAC,CAAC,MAAM;MACN;IAAA;IAEF,OAAO,IAAI5D,OAAO,CAAC,CAACkB,OAAO,EAAEkB,MAAM,KAAK;MACtC,MAAMD,CAAC,GAAG,IAAIjH,GAAG,CAACwI,OAAO,CAAC;MAC1B,MAAMlB,GAAG,GAAG,CAACL,CAAC,CAACM,QAAQ,KAAK,QAAQ,GAAGzH,KAAK,GAAGC,IAAI,EAAEyH,OAAO,CAC1D;QACEC,QAAQ,EAAER,CAAC,CAACQ,QAAQ;QACpBC,IAAI,EAAET,CAAC,CAACS,IAAI,KAAKT,CAAC,CAACM,QAAQ,KAAK,QAAQ,GAAG,GAAG,GAAG,EAAE,CAAC;QACpDH,IAAI,EAAEH,CAAC,CAACwB,QAAQ,GAAGxB,CAAC,CAACyB,MAAM;QAC3Bf,MAAM,EAAE,MAAM;QACdC,OAAO,EAAE;UACP,cAAc,EAAEhI,MAAM,CAACsC,QAAQ,CAACyG,WAAW;UAC3Cd,aAAa,EAAE,IAAI,CAAC1G,SAAS,GACzB,SAAS,IAAI,CAACA,SAAS,EAAE,GACzB4D;QACN,CAAC;QACD+C,KAAK,EACHb,CAAC,CAACM,QAAQ,KAAK,QAAQ,GACnB,IAAIzH,KAAK,CAACiI,KAAK,CAAC;UAAEC,SAAS,EAAE;QAAK,CAAC,CAAC,GACpCjD;MACR,CAAC,EACDkD,GAAG,IAAI;QACL,IAAIA,GAAG,CAACC,UAAU,IAAI,GAAG,IAAID,GAAG,CAACC,UAAU,GAAG,GAAG,EAAE;UACjDlC,OAAO,CAAC,CAAC;QACX,CAAC,MAAM;UACL,IAAIjC,IAAI,GAAG,EAAE;UACbkE,GAAG,CAACE,EAAE,CAAC,MAAM,EAAEC,KAAK,IAAI;YACtBrE,IAAI,IAAIqE,KAAK;UACf,CAAC,CAAC;UACFH,GAAG,CAACE,EAAE,CAAC,KAAK,EAAE,MACZjB,MAAM,CAAC,IAAIC,KAAK,CAAC,gBAAgBc,GAAG,CAACC,UAAU,IAAInE,IAAI,EAAE,CAAC,CAC5D,CAAC;QACH;MACF,CACF,CAAC;MACDuD,GAAG,CAACa,EAAE,CAAC,OAAO,EAAEjB,MAAM,CAAC;MACvB,IAAI,CAACnF,SAAS,CACXqD,OAAO,CAAC,CAAC,CACTwD,IAAI,CAACxD,OAAO,IAAI;QACfkC,GAAG,CAACuB,SAAS,CAAC,gBAAgB,EAAEC,MAAM,CAACC,UAAU,CAAC3D,OAAO,EAAE,MAAM,CAAC,CAAC;QACnEkC,GAAG,CAACe,GAAG,CAACjD,OAAO,EAAE,MAAM,CAAC;MAC1B,CAAC,CAAC,CACDc,KAAK,CAACgB,MAAM,CAAC;IAClB,CAAC,CAAC;EACJ,CAAC;;EAED;AACF;AACA;AACA;AACA;AACA;AACA;EACE8B,iBAAiB,GAAGA,CAACC,MAAM,GAAG,EAAE,KAAK;IACnC,OAAO,CAAC,GAAG5F,MAAM,CAACC,IAAI,CAAC,IAAI,CAACnB,aAAa,CAAC,EAAE,GAAG8G,MAAM,CAAC;EACxD,CAAC;;EAED;AACF;AACA;AACA;AACA;AACA;AACA;EACEC,8BAA8B,GAAGA,CAACD,MAAM,GAAG,EAAE,KAAK;IAChD,OAAO,CAAC,GAAG5F,MAAM,CAACC,IAAI,CAAC,IAAI,CAACf,0BAA0B,CAAC,EAAE,GAAG0G,MAAM,CAAC;EACrE,CAAC;EAEDE,gBAAgB,GAAGA,CAACF,MAAM,GAAG,EAAE,KAAK;IAClC,OAAO,IAAI,CAAC9G,aAAa;EAC3B,CAAC;EAEDW,mBAAmB,GAAGA,CAAA,KAAM;IAC1BzC,OAAO,CAAC8H,EAAE,CAAC,QAAQ,EAAE,IAAI,CAAC5B,OAAO,CAAC;IAClClG,OAAO,CAAC8H,EAAE,CAAC,SAAS,EAAE,IAAI,CAAC5B,OAAO,CAAC;EACrC,CAAC;;EAED;;EAEA,IAAI6C,cAAcA,CAAA,EAAG;IACnB,OAAO,IAAI,CAACvI,OAAO;EACrB;EAEA,IAAIsD,gBAAgBA,CAAA,EAAG;IACrB,OAAO,IAAI,CAACpD,SAAS;EACvB;EAEA,IAAIsI,QAAQA,CAAA,EAAG;IACb,OAAO,IAAI,CAACtH,SAAS;EACvB;EAEA,MAAMuH,kBAAkBA,CAAA,EAAG;IACzB,OAAO,IAAI,CAACvH,SAAS,CAACqD,OAAO,CAAC,CAAC;EACjC;EAEAmE,iBAAiBA,CAAA,EAAG;IAClB,OAAO,OAAOjC,GAAG,EAAEW,GAAG,KAAK;MACzB,IAAI;QACF,MAAM7C,OAAO,GAAG,MAAM,IAAI,CAACkE,kBAAkB,CAAC,CAAC;QAC/CrB,GAAG,CAACjD,GAAG,CAAC,cAAc,EAAEpF,MAAM,CAACsC,QAAQ,CAACyG,WAAW,CAAC;QACpDV,GAAG,CAACI,GAAG,CAACjD,OAAO,CAAC;MAClB,CAAC,CAAC,OAAOH,GAAG,EAAE;QACZb,OAAO,CAACc,KAAK,CAAC,GAAG,IAAI,CAACpD,UAAU,yBAAyB,EAAEmD,GAAG,CAAC;QAC/DgD,GAAG,CAACuB,MAAM,CAAC,GAAG,CAAC,CAACnB,GAAG,CAAC,2BAA2B,CAAC;MAClD;IACF,CAAC;EACH;AACF;AAEAoB,MAAM,CAACC,OAAO,GAAG;EAAEzJ;AAAkB,CAAC","ignoreList":[]}
|