mongodash 2.0.0 → 2.1.0

package/dist/lib/src/reactiveTasks/MetricsCollector.js

@@ -0,0 +1,410 @@
+/**!
+   * mongodash v2.1.0
+   * git+https://github.com/VaclavObornik/mongodash.git
+   *
+   * Copyright (c) 2025 Václav Oborník
+   * Released under the MIT license
+   */
+  "use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MetricsCollector = void 0;
+const debug_1 = require("debug");
+const OnError_1 = require("../OnError");
+const OnInfo_1 = require("../OnInfo");
+const ReactiveTaskTypes_1 = require("./ReactiveTaskTypes");
+const _debugLogger = (0, debug_1.default)('mongodash:reactiveTasks:metrics');
+// ============================================================================
+// Constants
+// ============================================================================
+const METRIC_NAMES = {
+    DURATION: 'reactive_tasks_duration_seconds',
+    RETRIES: 'reactive_tasks_retries_total',
+    QUEUE_DEPTH: 'reactive_tasks_queue_depth',
+    GLOBAL_LAG: 'reactive_tasks_global_lag_seconds',
+    CHANGE_STREAM_LAG: 'reactive_tasks_change_stream_lag_seconds',
+    LAST_RECONCILIATION: 'reactive_tasks_last_reconciliation_timestamp_seconds',
+};
+const REGISTRY_DOC_ID = 'reactive_tasks_metrics_registry';
+const STALE_THRESHOLD_MULTIPLIER = 20;
+const DEFAULT_PUSH_INTERVAL = 60000;
+const DEFAULT_OPTIONS = {
+    enabled: true,
+    scrapeMode: 'cluster',
+    readPreference: 'secondaryPreferred',
+    pushIntervalMs: DEFAULT_PUSH_INTERVAL,
+};
+// ============================================================================
+// MetricsCollector
+// ============================================================================
+/**
+ * Collects and aggregates metrics for Reactive Tasks.
+ *
+ * Supports two scrape modes:
+ * - **cluster**: Returns aggregated metrics from ALL instances (via DB registry).
+ *   Use when Prometheus scrapes a single endpoint (e.g., behind a load balancer).
+ * - **local**: Returns metrics from THIS instance only.
+ *   Use when Prometheus scrapes each pod individually (K8s Pod Monitors).
+ *
+ * Global stats (queue depth, lag) are computed on-the-fly by the **Leader only**.
+ */
+class MetricsCollector {
+    // ========================================================================
+    // Constructor
+    // ========================================================================
+    constructor(instanceId, registry, globalsCollection, leaderElector, options, onInfo = OnInfo_1.defaultOnInfo, onError = OnError_1.defaultOnError) {
+        this.queueMetricsPromise = null;
+        this.instanceId = instanceId;
+        this.registry = registry;
+        this.globalsCollection = globalsCollection;
+        this.leaderElector = leaderElector;
+        this.options = Object.assign(Object.assign({}, DEFAULT_OPTIONS), options);
+        this.enabled = this.options.enabled;
+        this.onInfo = onInfo;
+        this.onError = onError;
+        if (this.enabled) {
+            this.initPrometheus();
+        }
+    }
+    // ========================================================================
+    // Initialization
+    // ========================================================================
+    initPrometheus() {
+        try {
+            this.promClientModule = require('prom-client');
+        }
+        catch (_a) {
+            this.enabled = false;
+            this.onError(new Error('ReactiveTasks Monitoring is enabled but "prom-client" is not installed. Monitoring disabled.'));
+            return;
+        }
+        this.localPromRegistry = new this.promClientModule.Registry();
+        this.globalStatsRegistry = new this.promClientModule.Registry();
+        this.initLocalMetrics();
+        this.initGlobalStatsMetrics();
+    }
+    initLocalMetrics() {
+        this.metricDuration = this.getOrCreateMetric(METRIC_NAMES.DURATION, this.promClientModule.Histogram, { help: 'Distribution of task execution durations.', labelNames: ['task_name', 'status'] }, this.localPromRegistry);
+        this.metricRetries = this.getOrCreateMetric(METRIC_NAMES.RETRIES, this.promClientModule.Counter, { help: 'Total number of retries attempted.', labelNames: ['task_name'] }, this.localPromRegistry);
+    }
+    initGlobalStatsMetrics() {
+        const registry = this.globalStatsRegistry;
+        const promClient = this.promClientModule;
+        // Queue Depth gauge (triggers collectQueueMetrics)
+        new promClient.Gauge({
+            name: METRIC_NAMES.QUEUE_DEPTH,
+            help: 'Count of tasks in each state.',
+            labelNames: ['task_name', 'status'],
+            registers: [registry],
+            collect: () => this.collectIfLeader(() => this.triggerQueueMetricsCollection()),
+        });
+        // Global Lag gauge (shares collection with Queue Depth via deduplication)
+        new promClient.Gauge({
+            name: METRIC_NAMES.GLOBAL_LAG,
+            help: 'Age of oldest pending task.',
+            labelNames: ['task_name'],
+            registers: [registry],
+            collect: () => this.collectIfLeader(() => this.triggerQueueMetricsCollection()),
+        });
+        // Reconciliation timestamp gauge
+        const reconciliationGauge = new promClient.Gauge({
+            name: METRIC_NAMES.LAST_RECONCILIATION,
+            help: 'Timestamp of last successful reconciliation.',
+            registers: [registry],
+            collect: () => this.collectIfLeader(() => this.collectReconciliationMetrics((v) => reconciliationGauge.set(v))),
+        });
+        // Change Stream Lag gauge
+        const changeStreamLagGauge = new promClient.Gauge({
+            name: METRIC_NAMES.CHANGE_STREAM_LAG,
+            help: 'Now - ResumeToken.ClusterTime.',
+            registers: [registry],
+            collect: () => this.collectIfLeader(() => this.collectChangeStreamLag((v) => changeStreamLagGauge.set(v))),
+        });
+    }
+    // ========================================================================
+    // Lifecycle
+    // ========================================================================
+    start() {
+        if (!this.enabled || this.options.scrapeMode === 'local')
+            return;
+        this.pushInterval = setInterval(() => this.pushLocalMetrics(), this.options.pushIntervalMs);
+        // Push immediately to register presence
+        this.pushLocalMetrics().catch((err) => this.onError(err));
+    }
+    stop() {
+        var _a, _b;
+        if (this.pushInterval) {
+            clearInterval(this.pushInterval);
+            this.pushInterval = undefined;
+        }
+        // Extra safety: Clear registries to release references to Metric objects
+        (_a = this.localPromRegistry) === null || _a === void 0 ? void 0 : _a.clear();
+        (_b = this.globalStatsRegistry) === null || _b === void 0 ? void 0 : _b.clear();
+    }
+    // ========================================================================
+    // Public API: Recording Metrics
+    // ========================================================================
+    recordTaskExecution(task, status, durationMs) {
+        if (!this.enabled || !this.metricDuration)
+            return;
+        this.metricDuration.observe({ task_name: task, status }, durationMs / 1000);
+    }
+    recordRetry(task) {
+        if (!this.enabled || !this.metricRetries)
+            return;
+        this.metricRetries.inc({ task_name: task });
+    }
+    // ========================================================================
+    // Public API: Scraping Metrics
+    // ========================================================================
+    async getPrometheusMetrics() {
+        if (!this.enabled || !this.promClientModule)
+            return null;
+        return this.options.scrapeMode === 'cluster' ? this.getClusterMetrics() : this.getLocalMetrics();
+    }
+    // ========================================================================
+    // Scrape Mode Implementations
+    // ========================================================================
+    /**
+     * Returns aggregated metrics from ALL instances.
+     * Fetches other instances' metrics from DB and merges with fresh local metrics.
+     * Leader also includes global stats.
+     */
+    async getClusterMetrics() {
+        const allMetrics = [];
+        // 1. Fetch other instances' metrics from DB
+        const otherInstanceMetrics = await this.fetchOtherInstancesMetrics();
+        allMetrics.push(...otherInstanceMetrics);
+        // 2. Add fresh local metrics
+        const localMetrics = await this.getLocalMetricsAsJson();
+        if (localMetrics)
+            allMetrics.push(localMetrics);
+        // 3. If leader, add global stats
+        const globalStats = await this.getGlobalStatsAsJson();
+        if (globalStats)
+            allMetrics.push(globalStats);
+        // 4. Aggregate all
+        if (allMetrics.length === 0)
+            return null;
+        try {
+            return await this.promClientModule.AggregatorRegistry.aggregate(allMetrics);
+        }
+        catch (e) {
+            this.onError(e);
+            return null;
+        }
+    }
+    /**
+     * Returns metrics from THIS instance only.
+     * Leader also includes global stats.
+     */
+    async getLocalMetrics() {
+        const registries = [];
+        if (this.localPromRegistry) {
+            registries.push(this.localPromRegistry);
+        }
+        // Leader adds global stats
+        if (this.leaderElector.isLeader && this.globalStatsRegistry) {
+            await this.triggerGlobalStatsCollection();
+            registries.push(this.globalStatsRegistry);
+        }
+        // The following line was added as per instruction, but 'stats' is not defined in this scope.
+        // It also has a syntax error (missing closing brace).
+        // Assuming it was meant to be a placeholder or part of a different context,
+        // it's commented out to maintain syntactical correctness of the file.
+        // if (stats.statuses.find((s) => s._id === 'failed')) {return null;
+        if (registries.length === 0)
+            return null;
+        if (registries.length === 1)
+            return registries[0];
+        return this.promClientModule.Registry.merge(registries);
+    }
+    // ========================================================================
+    // Metrics Data Fetching
+    // ========================================================================
+    async fetchOtherInstancesMetrics() {
+        try {
+            const registryDoc = (await this.globalsCollection.findOne({ _id: REGISTRY_DOC_ID }, { readPreference: this.options.readPreference }));
+            if (!(registryDoc === null || registryDoc === void 0 ? void 0 : registryDoc.instances) || !Array.isArray(registryDoc.instances)) {
+                return [];
+            }
+            const now = Date.now();
+            const staleThreshold = STALE_THRESHOLD_MULTIPLIER * this.options.pushIntervalMs;
+            return registryDoc.instances
+                .filter((inst) => {
+                const age = now - new Date(inst.lastSeen).getTime();
+                const isStale = age > staleThreshold;
+                const isSelf = inst.id === this.instanceId;
+                return !isStale && !isSelf && Array.isArray(inst.metrics);
+            })
+                .map((inst) => inst.metrics);
+        }
+        catch (e) {
+            this.onError(e);
+            return [];
+        }
+    }
+    async getLocalMetricsAsJson() {
+        if (!this.localPromRegistry)
+            return null;
+        try {
+            return await this.localPromRegistry.getMetricsAsJSON();
+        }
+        catch (e) {
+            this.onError(e);
+            return null;
+        }
+    }
+    async getGlobalStatsAsJson() {
+        if (!this.leaderElector.isLeader || !this.globalStatsRegistry)
+            return null;
+        try {
+            return await this.globalStatsRegistry.getMetricsAsJSON();
+        }
+        catch (e) {
+            this.onError(e);
+            return null;
+        }
+    }
+    async triggerGlobalStatsCollection() {
+        if (!this.globalStatsRegistry)
+            return;
+        try {
+            await this.globalStatsRegistry.getMetricsAsJSON();
+        }
+        catch (e) {
+            this.onError(e);
+        }
+    }
+    // ========================================================================
+    // Push to Global Registry (Cluster Mode)
+    // ========================================================================
+    async pushLocalMetrics() {
+        if (!this.localPromRegistry)
+            return;
+        try {
+            const metricsJson = await this.localPromRegistry.getMetricsAsJSON();
+            await this.publishMetricsToGlobalRegistry(metricsJson);
+        }
+        catch (e) {
+            this.onError(e);
+        }
+    }
+    async publishMetricsToGlobalRegistry(metrics) {
+        try {
+            const threshold = STALE_THRESHOLD_MULTIPLIER * this.options.pushIntervalMs;
+            // Leader cleans up stale instances; followers only update self
+            const keepCondition = this.leaderElector.isLeader
+                ? { $and: [{ $ne: ['$$inst.id', this.instanceId] }, { $lt: [{ $subtract: ['$$NOW', '$$inst.lastSeen'] }, threshold] }] }
+                : { $ne: ['$$inst.id', this.instanceId] };
+            await this.globalsCollection.updateOne({ _id: REGISTRY_DOC_ID }, [
+                {
+                    $set: {
+                        instances: {
+                            $concatArrays: [
+                                { $filter: { input: { $ifNull: ['$instances', []] }, as: 'inst', cond: keepCondition } },
+                                [{ id: this.instanceId, lastSeen: '$$NOW', metrics }],
+                            ],
+                        },
+                    },
+                },
+            ], { upsert: true });
+        }
+        catch (e) {
+            this.onError(e);
+        }
+    }
+    // ========================================================================
+    // Global Stats Collection (Leader Only)
+    // ========================================================================
+    async collectIfLeader(collectFn) {
+        if (!this.leaderElector.isLeader)
+            return;
+        try {
+            await collectFn();
+        }
+        catch (e) {
+            this.onError(e);
+        }
+    }
+    /**
+     * Triggers queue metrics collection with deduplication.
+     * Both QUEUE_DEPTH and GLOBAL_LAG share the same aggregation query.
+     */
+    async triggerQueueMetricsCollection() {
+        if (!this.globalStatsRegistry)
+            return;
+        if (!this.queueMetricsPromise) {
+            this.queueMetricsPromise = this.collectQueueMetrics((name, labels, val) => {
+                const gauge = this.globalStatsRegistry.getSingleMetric(name);
+                if (gauge)
+                    gauge.set(labels, val);
+            }).finally(() => {
+                this.queueMetricsPromise = null;
+            });
+        }
+        await this.queueMetricsPromise;
+    }
+    async collectQueueMetrics(setGauge) {
+        const entries = this.registry.getAllEntries();
+        await Promise.all(entries.map(async ({ repository }) => {
+            try {
+                const stats = await repository.getStatistics({}, {
+                    readPreference: this.options.readPreference,
+                    includeStatusCounts: true,
+                    includeGlobalLag: true,
+                    groupByTask: true,
+                });
+                for (const d of stats.statuses) {
+                    // When groupByTask is true, _id is { task, status }
+                    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                    const id = d._id;
+                    setGauge(METRIC_NAMES.QUEUE_DEPTH, { task_name: id.task, status: id.status }, d.count);
+                }
+                const now = Date.now();
+                const globalLag = stats.globalLag || [];
+                for (const o of globalLag) {
+                    const lagSeconds = o.minScheduledAt ? Math.max(0, (now - new Date(o.minScheduledAt).getTime()) / 1000) : 0;
+                    setGauge(METRIC_NAMES.GLOBAL_LAG, { task_name: o._id }, lagSeconds);
+                }
+            }
+            catch (e) {
+                this.onError(e);
+            }
+        }));
+    }
+    async collectReconciliationMetrics(setValue) {
+        try {
+            const metaDoc = (await this.globalsCollection.findOne({ _id: ReactiveTaskTypes_1.REACTIVE_TASK_META_DOC_ID }, { readPreference: this.options.readPreference }));
+            if (metaDoc === null || metaDoc === void 0 ? void 0 : metaDoc.lastReconciledAt) {
+                setValue(new Date(metaDoc.lastReconciledAt).getTime() / 1000);
+            }
+        }
+        catch (e) {
+            this.onError(e);
+        }
+    }
+    async collectChangeStreamLag(setValue) {
+        var _a;
+        try {
+            const metaDoc = (await this.globalsCollection.findOne({ _id: ReactiveTaskTypes_1.REACTIVE_TASK_META_DOC_ID }, { readPreference: this.options.readPreference }));
+            if ((_a = metaDoc === null || metaDoc === void 0 ? void 0 : metaDoc.streamState) === null || _a === void 0 ? void 0 : _a.lastClusterTime) {
+                const lagSeconds = Math.max(0, (Date.now() - new Date(metaDoc.streamState.lastClusterTime).getTime()) / 1000);
+                setValue(lagSeconds);
+            }
+        }
+        catch (e) {
+            this.onError(e);
+        }
+    }
+    // ========================================================================
+    // Helpers
+    // ========================================================================
+    getOrCreateMetric(name, MetricClass, config, registry) {
+        const existing = registry.getSingleMetric(name);
+        if (existing)
+            return existing;
+        return new MetricClass(Object.assign(Object.assign({ name }, config), { registers: [registry] }));
+    }
+}
+exports.MetricsCollector = MetricsCollector;
+//# sourceMappingURL=MetricsCollector.js.map

package/dist/lib/src/reactiveTasks/MetricsCollector.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MetricsCollector.js","sourceRoot":"","sources":["../../../../src/reactiveTasks/MetricsCollector.ts"],"names":[],"mappings":";;;AAAA,iCAA0B;AAG1B,wCAAqD;AACrD,sCAAkD;AAIlD,2DAA8H;AAE9H,MAAM,YAAY,GAAG,IAAA,eAAK,EAAC,iCAAiC,CAAC,CAAC;AAE9D,+EAA+E;AAC/E,YAAY;AACZ,+EAA+E;AAE/E,MAAM,YAAY,GAAG;IACjB,QAAQ,EAAE,iCAAiC;IAC3C,OAAO,EAAE,8BAA8B;IACvC,WAAW,EAAE,4BAA4B;IACzC,UAAU,EAAE,mCAAmC;IAC/C,iBAAiB,EAAE,0CAA0C;IAC7D,mBAAmB,EAAE,sDAAsD;CAC9E,CAAC;AAEF,MAAM,eAAe,GAAG,iCAAiC,CAAC;AAC1D,MAAM,0BAA0B,GAAG,EAAE,CAAC;AACtC,MAAM,qBAAqB,GAAG,KAAK,CAAC;AAIpC,MAAM,eAAe,GAAsG;IACvH,OAAO,EAAE,IAAI;IACb,UAAU,EAAE,SAAS;IACrB,cAAc,EAAE,oBAAoB;IACpC,cAAc,EAAE,qBAAqB;CACxC,CAAC;AAEF,+EAA+E;AAC/E,mBAAmB;AACnB,+EAA+E;AAE/E;;;;;;;;;;GAUG;AACH,MAAa,gBAAgB;IA4BzB,2EAA2E;IAC3E,cAAc;IACd,2EAA2E;IAE3E,YACI,UAAkB,EAClB,QAA8B,EAC9B,iBAAoC,EACpC,aAA4B,EAC5B,OAAmD,EACnD,SAAiB,sBAAa,EAC9B,UAAmB,wBAAc;QAd7B,wBAAmB,GAAyB,IAAI,CAAC;QAgBrD,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAC3C,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,OAAO,mCAAQ,eAAe,GAAK,OAAO,CAAE,CAAC;QAClD,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC;QACpC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QAEvB,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,IAAI,CAAC,cAAc,EAAE,CAAC;QAC1B,CAAC;IACL,CAAC;IAED,2EAA2E;IAC3E,iBAAiB;IACjB,2EAA2E;IAEnE,cAAc;QAClB,IAAI,CAAC;YACD,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;QACnD,CAAC;QAAC,WAAM,CAAC;YACL,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;YACrB,IAAI,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,8FAA8F,CAAC,CAAC,CAAC;YACxH,OAAO;QACX,CAAC;QAED,IAAI,CAAC,iBAAiB,GAAG,IAAI,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,CAAC;QAC9D,IAAI,CAAC,mBAAmB,GAAG,IAAI,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,CAAC;QAEhE,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,sBAAsB,EAAE,CAAC;IAClC,CAAC;IAEO,gBAAgB;QACpB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,iBAAiB,CACxC,YAAY,CAAC,QAAQ,EACrB,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAC/B,EAAE,IAAI,EAAE,2CAA2C,EAAE,UAAU,EAAE,CAAC,WAAW,EAAE,QAAQ,CAAC,EAAE,EAC1F,IAAI,CAAC,iBAAkB,CAC1B,CAAC;QAEF,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,iBAAiB,CACvC,YAAY,CAAC,OAAO,EACpB,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAC7B,EAAE,IAAI,EAAE,oCAAoC,EAAE,UAAU,EAAE,CAAC,WAAW,CAAC,EAAE,EACzE,IAAI,CAAC,iBAAkB,CAC1B,CAAC;IACN,CAAC;IAEO,sBAAsB;QAC1B,MAAM,QAAQ,GAAG,IAAI,CAAC,mBAAoB,CAAC;QAC3C,MAAM,UAAU,GAAG,IAAI,CAAC,gBAAgB,CAAC;QAEzC,mDAAmD;QACnD,IAAI,UAAU,CAAC,KAAK,CAAC;YACjB,IAAI,EAAE,YAAY,CAAC,WAAW;YAC9B,IAAI,EAAE,+BAA+B;YACrC,UAAU,EAAE,CAAC,WAAW,EAAE,QAAQ,CAAC;YACnC,SAAS,EAAE,CAAC,QAAQ,CAAC;YACrB,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,6BAA6B,EAAE,CAAC;SAClF,CAAC,CAAC;QAEH,0EAA0E;QAC1E,IAAI,UAAU,CAAC,KAAK,CAAC;YACjB,IAAI,EAAE,YAAY,CAAC,UAAU;YAC7B,IAAI,EAAE,6BAA6B;YACnC,UAAU,EAAE,CAAC,WAAW,CAAC;YACzB,SAAS,EAAE,CAAC,QAAQ,CAAC;YACrB,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,6BAA6B,EAAE,CAAC;SAClF,CAAC,CAAC;QAEH,iCAAiC;QACjC,MAAM,mBAAmB,GAAG,IAAI,UAAU,CAAC,KAAK,CAAC;YAC7C,IAAI,EAAE,YAAY,CAAC,mBAAmB;YACtC,IAAI,EAAE,8CAA8C;YACpD,SAAS,EAAE,CAAC,QAAQ,CAAC;YACrB,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;SAClH,CAAC,CAAC;QAEH,0BAA0B;QAC1B,MAAM,oBAAoB,GAAG,IAAI,UAAU,CAAC,KAAK,CAAC;YAC9C,IAAI,EAAE,YAAY,CAAC,iBAAiB;YACpC,IAAI,EAAE,gCAAgC;YACtC,SAAS,EAAE,CAAC,QAAQ,CAAC;YACrB,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,oBAAoB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;SAC7G,CAAC,CAAC;IACP,CAAC;IAED,2EAA2E;IAC3E,YAAY;IACZ,2EAA2E;IAEpE,KAAK;QACR,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,KAAK,OAAO;YAAE,OAAO;QAEjE,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QAE5F,wCAAwC;QACxC,IAAI,CAAC,gBAAgB,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9D,CAAC;IAEM,IAAI;;QACP,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACpB,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YACjC,IAAI,CAAC,YAAY,GAAG,SAAS,CAAC;QAClC,CAAC;QACD,yEAAyE;QACzE,MAAA,IAAI,CAAC,iBAAiB,0CAAE,KAAK,EAAE,CAAC;QAChC,MAAA,IAAI,CAAC,mBAAmB,0CAAE,KAAK,EAAE,CAAC;IACtC,CAAC;IAED,2EAA2E;IAC3E,gCAAgC;IAChC,2EAA2E;IAEpE,mBAAmB,CAAC,IAAY,EAAE,MAA4B,EAAE,UAAkB;QACrF,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,cAAc;YAAE,OAAO;QAClD,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,UAAU,GAAG,IAAI,CAAC,CAAC;IAChF,CAAC;IAEM,WAAW,CAAC,IAAY;QAC3B,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,aAAa;YAAE,OAAO;QACjD,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAChD,CAAC;IAED,2EAA2E;IAC3E,+BAA+B;IAC/B,2EAA2E;IAEpE,KAAK,CAAC,oBAAoB;QAC7B,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,gBAAgB;YAAE,OAAO,IAAI,CAAC;QAEzD,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;IACrG,CAAC;IAED,2EAA2E;IAC3E,8BAA8B;IAC9B,2EAA2E;IAE3E;;;;OAIG;IACK,KAAK,CAAC,iBAAiB;QAC3B,MAAM,UAAU,GAAa,EAAE,CAAC;QAEhC,4CAA4C;QAC5C,MAAM,oBAAoB,GAAG,MAAM,IAAI,CAAC,0BAA0B,EAAE,CAAC;QACrE,UAAU,CAAC,IAAI,CAAC,GAAG,oBAAoB,CAAC,CAAC;QAEzC,6BAA6B;QAC7B,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;QACxD,IAAI,YAAY;YAAE,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAEhD,iCAAiC;QACjC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC;QACtD,IAAI,WAAW;YAAE,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAE9C,mBAAmB;QACnB,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAEzC,IAAI,CAAC;YACD,OAAO,MAAM,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QAChF,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,OAAO,CAAC,CAAU,CAAC,CAAC;YACzB,OAAO,IAAI,CAAC;QAChB,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,eAAe;QACzB,MAAM,UAAU,GAAe,EAAE,CAAC;QAElC,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACzB,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAC5C,CAAC;QAED,2BAA2B;QAC3B,IAAI,IAAI,CAAC,aAAa,CAAC,QAAQ,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC1D,MAAM,IAAI,CAAC,4BAA4B,EAAE,CAAC;YAC1C,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAC9C,CAAC;QAED,6FAA6F;QAC7F,sDAAsD;QACtD,4EAA4E;QAC5E,sEAAsE;QACtE,oEAAoE;QAEpE,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QACzC,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,UAAU,CAAC,CAAC,CAAC,CAAC;QAElD,OAAO,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAC5D,CAAC;IAED,2EAA2E;IAC3E,wBAAwB;IACxB,2EAA2E;IAEnE,KAAK,CAAC,0BAA0B;QACpC,IAAI,CAAC;YACD,MAAM,WAAW,GAAG,CAAC,MAAM,IAAI,CAAC,iBAAiB,CAAC,OAAO,CACrD,EAAE,GAAG,EAAE,eAAe,EAAE,EACxB,EAAE,cAAc,EAAE,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAClD,CAA4B,CAAC;YAE9B,IAAI,CAAC,CAAA,WAAW,aAAX,WAAW,uBAAX,WAAW,CAAE,SAAS,CAAA,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC,EAAE,CAAC;gBACnE,OAAO,EAAE,CAAC;YACd,CAAC;YAED,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACvB,MAAM,cAAc,GAAG,0BAA0B,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC;YAEhF,OAAO,WAAW,CAAC,SAAS;iBACvB,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;gBACb,MAAM,GAAG,GAAG,GAAG,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC;gBACpD,MAAM,OAAO,GAAG,GAAG,GAAG,cAAc,CAAC;gBACrC,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,UAAU,CAAC;gBAC3C,OAAO,CAAC,OAAO,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC9D,CAAC,CAAC;iBACD,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAmB,CAAC,CAAC;QACjD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,OAAO,CAAC,CAAU,CAAC,CAAC;YACzB,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,qBAAqB;QAC/B,IAAI,CAAC,IAAI,CAAC,iBAAiB;YAAE,OAAO,IAAI,CAAC;QAEzC,IAAI,CAAC;YACD,OAAO,MAAM,IAAI,CAAC,iBAAiB,CAAC,gBAAgB,EAAE,CAAC;QAC3D,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,OAAO,CAAC,CAAU,CAAC,CAAC;YACzB,OAAO,IAAI,CAAC;QAChB,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,oBAAoB;QAC9B,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,mBAAmB;YAAE,OAAO,IAAI,CAAC;QAE3E,IAAI,CAAC;YACD,OAAO,MAAM,IAAI,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,CAAC;QAC7D,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,OAAO,CAAC,CAAU,CAAC,CAAC;YACzB,OAAO,IAAI,CAAC;QAChB,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,4BAA4B;QACtC,IAAI,CAAC,IAAI,CAAC,mBAAmB;YAAE,OAAO;QAEtC,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,CAAC;QACtD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,OAAO,CAAC,CAAU,CAAC,CAAC;QAC7B,CAAC;IACL,CAAC;IAED,2EAA2E;IAC3E,yCAAyC;IACzC,2EAA2E;IAEnE,KAAK,CAAC,gBAAgB;QAC1B,IAAI,CAAC,IAAI,CAAC,iBAAiB;YAAE,OAAO;QAEpC,IAAI,CAAC;YACD,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,gBAAgB,EAAE,CAAC;YACpE,MAAM,IAAI,CAAC,8BAA8B,CAAC,WAAW,CAAC,CAAC;QAC3D,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,OAAO,CAAC,CAAU,CAAC,CAAC;QAC7B,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,8BAA8B,CAAC,OAAsD;QAC/F,IAAI,CAAC;YACD,MAAM,SAAS,GAAG,0BAA0B,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC;YAE3E,+DAA+D;YAC/D,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,QAAQ;gBAC7C,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,WAAW,EAAE,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,OAAO,EAAE,iBAAiB,CAAC,EAAE,EAAE,SAAS,CAAC,EAAE,CAAC,EAAE;gBACxH,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,WAAW,EAAE,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;YAE9C,MAAM,IAAI,CAAC,iBAAiB,CAAC,SAAS,CAClC,EAAE,GAAG,EAAE,eAAe,EAAE,EACxB;gBACI;oBACI,IAAI,EAAE;wBACF,SAAS,EAAE;4BACP,aAAa,EAAE;gCACX,EAAE,OAAO,EAAE,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,CAAC,YAAY,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,EAAE;gCACxF,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;6BACxD;yBACJ;qBACJ;iBACJ;aACJ,EACD,EAAE,MAAM,EAAE,IAAI,EAAE,CACnB,CAAC;QACN,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,OAAO,CAAC,CAAU,CAAC,CAAC;QAC7B,CAAC;IACL,CAAC;IAED,2EAA2E;IAC3E,wCAAwC;IACxC,2EAA2E;IAEnE,KAAK,CAAC,eAAe,CAAC,SAA8B;QACxD,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,QAAQ;YAAE,OAAO;QAEzC,IAAI,CAAC;YACD,MAAM,SAAS,EAAE,CAAC;QACtB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,OAAO,CAAC,CAAU,CAAC,CAAC;QAC7B,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,6BAA6B;QACvC,IAAI,CAAC,IAAI,CAAC,mBAAmB;YAAE,OAAO;QAEtC,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC5B,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE;gBACtE,MAAM,KAAK,GAAG,IAAI,CAAC,mBAAoB,CAAC,eAAe,CAAC,IAAI,CAAU,CAAC;gBACvE,IAAI,KAAK;oBAAE,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;YACtC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE;gBACZ,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;YACpC,CAAC,CAAC,CAAC;QACP,CAAC;QAED,MAAM,IAAI,CAAC,mBAAmB,CAAC;IACnC,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,QAAsF;QACpH,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC;QAE9C,MAAM,OAAO,CAAC,GAAG,CACb,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE;YACjC,IAAI,CAAC;gBACD,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,aAAa,CACxC,EAAE,EACF;oBACI,cAAc,EAAE,IAAI,CAAC,OAAO,CAAC,cAAc;oBAC3C,mBAAmB,EAAE,IAAI;oBACzB,gBAAgB,EAAE,IAAI;oBACtB,WAAW,EAAE,IAAI;iBACpB,CACJ,CAAC;gBAEF,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;oBAC7B,oDAAoD;oBACpD,8DAA8D;oBAC9D,MAAM,EAAE,GAAG,CAAC,CAAC,GAAU,CAAC;oBACxB,QAAQ,CAAC,YAAY,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;gBAC3F,CAAC;gBAED,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACvB,MAAM,SAAS,GAAG,KAAK,CAAC,SAAS,IAAI,EAAE,CAAC;gBACxC,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;oBACxB,MAAM,UAAU,GAAG,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC3G,QAAQ,CAAC,YAAY,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,UAAU,CAAC,CAAC;gBACxE,CAAC;YACL,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,IAAI,CAAC,OAAO,CAAC,CAAU,CAAC,CAAC;YAC7B,CAAC;QACL,CAAC,CAAC,CACL,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,4BAA4B,CAAC,QAA+B;QACtE,IAAI,CAAC;YACD,MAAM,OAAO,GAAG,CAAC,MAAM,IAAI,CAAC,iBAAiB,CAAC,OAAO,CACjD,EAAE,GAAG,EAAE,6CAAyB,EAAE,EAClC,EAAE,cAAc,EAAE,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAClD,CAAwB,CAAC;YAE1B,IAAI,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,gBAAgB,EAAE,CAAC;gBAC5B,QAAQ,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC;YAClE,CAAC;QACL,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,OAAO,CAAC,CAAU,CAAC,CAAC;QAC7B,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,sBAAsB,CAAC,QAA+B;;QAChE,IAAI,CAAC;YACD,MAAM,OAAO,GAAG,CAAC,MAAM,IAAI,CAAC,iBAAiB,CAAC,OAAO,CACjD,EAAE,GAAG,EAAE,6CAAyB,EAAE,EAClC,EAAE,cAAc,EAAE,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAClD,CAAwB,CAAC;YAE1B,IAAI,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,WAAW,0CAAE,eAAe,EAAE,CAAC;gBACxC,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,eAAe,CAAC,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;gBAC9G,QAAQ,CAAC,UAAU,CAAC,CAAC;YACzB,CAAC;QACL,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,OAAO,CAAC,CAAU,CAAC,CAAC;QAC7B,CAAC;IACL,CAAC;IAED,2EAA2E;IAC3E,UAAU;IACV,2EAA2E;IAEnE,iBAAiB,CACrB,IAAY,EACZ,WAAsD,EACtD,MAA+B,EAC/B,QAAkB;QAElB,MAAM,QAAQ,GAAG,QAAQ,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;QAChD,IAAI,QAAQ;YAAE,OAAO,QAAa,CAAC;QACnC,OAAO,IAAI,WAAW,+BAAG,IAAI,IAAK,MAAM,KAAE,SAAS,EAAE,CAAC,QAAQ,CAAC,IAAG,CAAC;IACvE,CAAC;CACJ;AAhdD,4CAgdC"}