atomic-queues 2.3.0 → 3.0.0

package/dist/cluster/cluster-discovery.service.js

@@ -0,0 +1,423 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+var ClusterDiscoveryService_1;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ClusterDiscoveryService = void 0;
+const common_1 = require("@nestjs/common");
+const ioredis_1 = __importDefault(require("ioredis"));
+const uuid_1 = require("uuid");
+const utils_1 = require("../utils");
+const constants_1 = require("../services/constants");
+const grpc_peer_monitor_service_1 = require("./grpc-peer-monitor.service");
+const redis_health_monitor_service_1 = require("./redis-health-monitor.service");
+const HEARTBEAT_SCRIPT = `
+redis.call("HSET", KEYS[1], "heartbeat_at", ARGV[1])
+redis.call("PEXPIRE", KEYS[1], ARGV[2])
+return redis.call("HGET", KEYS[1], "ring_version")
+`;
+/**
+ * Cluster Discovery Service.
+ *
+ * Manages server heartbeats, ring membership, and ring change events
+ * using Redis as the source of truth.
+ *
+ * Only active when `config.grpc.enabled` is true.
+ */
+let ClusterDiscoveryService = ClusterDiscoveryService_1 = class ClusterDiscoveryService {
+    constructor(redis, config, peerMonitor, redisHealthMonitor) {
+        this.redis = redis;
+        this.config = config;
+        this.peerMonitor = peerMonitor;
+        this.redisHealthMonitor = redisHealthMonitor;
+        this.logger = new common_1.Logger(ClusterDiscoveryService_1.name);
+        this.instanceId = (0, uuid_1.v4)();
+        this.lastSeenRingVersion = 0;
+        this.heartbeatTimer = null;
+        this.reconcileTimer = null;
+        this.eventReconcileTimer = null;
+        this.subscriber = null;
+        this.redisAvailable = true;
+        this.unsubscribePeerMonitor = null;
+        this.unsubscribeRedisHealth = null;
+        this.changeListeners = [];
+        this.keyPrefix = (0, utils_1.resolveKeyPrefix)(config);
+        this.enabled = config.grpc?.enabled ?? false;
+        this.serverId = config.grpc?.serverId ?? 'unknown';
+        this.serviceGroup = config.grpc?.serviceGroup ?? 'default';
+        this.grpcAddress = config.grpc?.advertisedAddress ?? '0.0.0.0:50051';
+        this.heartbeatIntervalMs = config.grpc?.heartbeatMs ?? 400;
+        this.reconcileIntervalMs = config.grpc?.reconcileIntervalMs ?? 2000;
+        this.nodeTTL = config.grpc?.nodeTTLMs ?? 1500;
+    }
+    async onModuleInit() {
+        if (!this.enabled)
+            return;
+        // Clean up stale keys from a previous instance of this serverId
+        const oldPattern = `${this.keyPrefix}:cluster:nodes:${this.serverId}:*`;
+        const oldKeys = await this.scanKeys(oldPattern);
+        if (oldKeys.length > 0) {
+            await this.redis.del(...oldKeys);
+            const indexKey = this.getNodeIndexKey();
+            const pipeline = this.redis.pipeline();
+            for (const key of oldKeys) {
+                const member = key.replace(`${this.keyPrefix}:cluster:nodes:`, '');
+                pipeline.srem(indexKey, member);
+            }
+            await pipeline.exec();
+            this.logger.log(`Cleaned ${oldKeys.length} stale node key(s) from previous instance`);
+        }
+        // Register this node
+        await this.registerNode();
+        // Increment ring version
+        await this.incrementRingVersion();
+        // Publish join event
+        await this.publishEvent('join');
+        // Start heartbeat
+        this.heartbeatTimer = setInterval(() => {
+            this.heartbeat().catch((err) => {
+                this.logger.error(`Heartbeat failed: ${err.message}`);
+            });
+        }, this.heartbeatIntervalMs);
+        // Start ring reconciliation (slower than heartbeat — consistency fallback)
+        this.reconcileTimer = setInterval(() => {
+            this.reconcile().catch((err) => {
+                this.logger.error(`Reconciliation failed: ${err.message}`);
+            });
+        }, this.reconcileIntervalMs);
+        // Subscribe to ring events
+        this.subscriber = this.redis.duplicate();
+        const channel = this.getEventsChannel();
+        await this.subscriber.subscribe(channel);
+        this.subscriber.on('message', (_ch, payload) => {
+            this.handleEvent(payload);
+        });
+        // Subscribe to gRPC peer state changes for fast failure detection
+        if (this.peerMonitor) {
+            this.unsubscribePeerMonitor = this.peerMonitor.onPeerStateChange((peerId, state) => {
+                if (state === 'suspected-dead') {
+                    this.logger.warn(`Peer ${peerId} suspected dead via gRPC — triggering reconcile`);
+                    this.reconcile().catch((err) => {
+                        this.logger.error(`Triggered reconcile failed: ${err.message}`);
+                    });
+                }
+            });
+        }
+        // Subscribe to Redis health changes for voluntary step-down
+        if (this.redisHealthMonitor) {
+            this.unsubscribeRedisHealth = this.redisHealthMonitor.onHealthChange((healthy) => {
+                if (!healthy) {
+                    this.handleRedisLost();
+                }
+                else {
+                    this.handleRedisRecovered();
+                }
+            });
+        }
+        this.logger.log(`Cluster discovery started: serverId=${this.serverId}, group=${this.serviceGroup}`);
+    }
+    async onApplicationShutdown() {
+        if (!this.enabled)
+            return;
+        if (this.heartbeatTimer)
+            clearInterval(this.heartbeatTimer);
+        if (this.reconcileTimer)
+            clearInterval(this.reconcileTimer);
+        if (this.eventReconcileTimer)
+            clearTimeout(this.eventReconcileTimer);
+        if (this.unsubscribePeerMonitor)
+            this.unsubscribePeerMonitor();
+        if (this.unsubscribeRedisHealth)
+            this.unsubscribeRedisHealth();
+        // Remove this node from the ring
+        await this.redis.del(this.getNodeKey());
+        await this.redis.srem(this.getNodeIndexKey(), `${this.serverId}:${this.instanceId}`);
+        await this.incrementRingVersion();
+        await this.publishEvent('leave');
+        if (this.subscriber) {
+            await this.subscriber.unsubscribe();
+            await this.subscriber.quit();
+        }
+        this.logger.log('Cluster discovery stopped');
+    }
+    // =========================================================================
+    // PUBLIC API
+    // =========================================================================
+    /**
+     * Get all live nodes in the cluster.
+     */
+    async getNodes() {
+        const indexKey = this.getNodeIndexKey();
+        const members = await this.redis.smembers(indexKey);
+        const nodes = [];
+        const staleMembers = [];
+        for (const member of members) {
+            const nodeKey = `${this.keyPrefix}:cluster:nodes:${member}`;
+            const data = await this.redis.hgetall(nodeKey);
+            if (data && data.server_id) {
+                nodes.push(this.parseNodeData(data));
+            }
+            else {
+                staleMembers.push(member);
+            }
+        }
+        if (staleMembers.length > 0) {
+            const pipeline = this.redis.pipeline();
+            for (const member of staleMembers) {
+                pipeline.srem(indexKey, member);
+            }
+            pipeline.exec().catch(() => { });
+        }
+        return nodes;
+    }
+    /**
+     * Get the current ring version.
+     */
+    async getRingVersion() {
+        const versionKey = `${this.keyPrefix}:cluster:ring:version`;
+        const version = await this.redis.get(versionKey);
+        return version ? parseInt(version, 10) : 0;
+    }
+    /**
+     * Register a listener for ring change events.
+     */
+    onRingChange(listener) {
+        this.changeListeners.push(listener);
+        return () => {
+            const idx = this.changeListeners.indexOf(listener);
+            if (idx >= 0)
+                this.changeListeners.splice(idx, 1);
+        };
+    }
+    /**
+     * Whether the cluster is healthy (Redis reachable).
+     * When false, this node has stepped down and should not accept new work.
+     */
+    isClusterHealthy() {
+        return this.redisAvailable;
+    }
+    /**
+     * Get this server's ID.
+     */
+    getServerId() {
+        return this.serverId;
+    }
+    /**
+     * Resolve which service group owns an entity type (via Redis registry).
+     */
+    async resolveServiceGroup(entityType) {
+        const registryKey = `${this.keyPrefix}:cluster:entity-registry:${entityType}`;
+        return this.redis.get(registryKey);
+    }
+    // =========================================================================
+    // INTERNAL — Registration
+    // =========================================================================
+    async registerNode() {
+        const key = this.getNodeKey();
+        const indexKey = this.getNodeIndexKey();
+        const entityTypes = Object.keys(this.config.entities ?? {});
+        const data = {
+            server_id: this.serverId,
+            instance_id: this.instanceId,
+            grpc_address: this.grpcAddress,
+            service_group: this.serviceGroup,
+            entity_types: entityTypes.join(','),
+            ring_version: '0',
+            started_at: Date.now().toString(),
+            heartbeat_at: Date.now().toString(),
+        };
+        await this.redis.hset(key, data);
+        await this.redis.pexpire(key, this.nodeTTL);
+        await this.redis.sadd(indexKey, `${this.serverId}:${this.instanceId}`);
+        // Register entity type → service group mapping for cross-service routing
+        for (const et of entityTypes) {
+            const registryKey = `${this.keyPrefix}:cluster:entity-registry:${et}`;
+            await this.redis.set(registryKey, this.serviceGroup, 'PX', this.nodeTTL * 2);
+        }
+    }
+    async heartbeat() {
+        const key = this.getNodeKey();
+        const entityTypes = Object.keys(this.config.entities ?? {});
+        const pipeline = this.redis.pipeline();
+        pipeline.eval(HEARTBEAT_SCRIPT, 1, key, Date.now().toString(), this.nodeTTL.toString());
+        for (const et of entityTypes) {
+            pipeline.pexpire(`${this.keyPrefix}:cluster:entity-registry:${et}`, this.nodeTTL * 2);
+        }
+        await pipeline.exec();
+    }
+    async reconcile() {
+        if (!this.redisAvailable)
+            return;
+        const currentVersion = await this.getRingVersion();
+        if (currentVersion > this.lastSeenRingVersion) {
+            this.lastSeenRingVersion = currentVersion;
+        }
+        const nodes = await this.getNodes();
+        // Sync gRPC peer monitor with current Redis-known nodes
+        if (this.peerMonitor) {
+            this.peerMonitor.syncPeers(nodes
+                .filter((n) => n.serverId !== this.serverId)
+                .map((n) => ({ serverId: n.serverId, address: n.grpcAddress })));
+        }
+        this.notifyListeners(nodes);
+    }
+    // =========================================================================
+    // INTERNAL — Events
+    // =========================================================================
+    async publishEvent(action) {
+        const channel = this.getEventsChannel();
+        const ringVersion = await this.getRingVersion();
+        await this.redis.publish(channel, JSON.stringify({
+            action,
+            serverId: this.serverId,
+            serviceGroup: this.serviceGroup,
+            timestamp: Date.now(),
+            ringVersion,
+        }));
+    }
+    handleEvent(payload) {
+        try {
+            const event = JSON.parse(payload);
+            if (event.serverId === this.serverId)
+                return;
+            this.logger.log(`Ring event: ${event.action} from ${event.serverId}`);
+            const receivedVersion = event.ringVersion;
+            const gapDetected = receivedVersion !== undefined &&
+                this.lastSeenRingVersion > 0 &&
+                receivedVersion > this.lastSeenRingVersion + 1;
+            if (receivedVersion !== undefined && receivedVersion > this.lastSeenRingVersion) {
+                this.lastSeenRingVersion = receivedVersion;
+            }
+            if (gapDetected) {
+                this.logger.warn(`Ring version gap detected (expected <= ${this.lastSeenRingVersion}, got ${receivedVersion}) — immediate reconcile`);
+                if (this.eventReconcileTimer)
+                    clearTimeout(this.eventReconcileTimer);
+                this.eventReconcileTimer = null;
+                this.reconcile().catch((err) => this.logger.error(`Gap-triggered reconcile failed: ${err.message}`));
+                return;
+            }
+            if (this.eventReconcileTimer)
+                clearTimeout(this.eventReconcileTimer);
+            this.eventReconcileTimer = setTimeout(() => {
+                this.eventReconcileTimer = null;
+                this.reconcile().catch((err) => this.logger.error(`Event-triggered reconcile failed: ${err.message}`));
+            }, 200);
+        }
+        catch {
+            // Ignore malformed events
+        }
+    }
+    notifyListeners(nodes) {
+        for (const listener of this.changeListeners) {
+            try {
+                listener(nodes);
+            }
+            catch (err) {
+                this.logger.error(`Ring change listener error: ${err.message}`);
+            }
+        }
+    }
+    // =========================================================================
+    // INTERNAL — Redis health step-down / recovery
+    // =========================================================================
+    handleRedisLost() {
+        this.redisAvailable = false;
+        this.logger.error('Redis connectivity lost — stepping down from cluster');
+        if (this.heartbeatTimer) {
+            clearInterval(this.heartbeatTimer);
+            this.heartbeatTimer = null;
+        }
+        if (this.reconcileTimer) {
+            clearInterval(this.reconcileTimer);
+            this.reconcileTimer = null;
+        }
+        // Notify with empty node list — triggers leader resignation and worker cleanup
+        this.notifyListeners([]);
+    }
+    handleRedisRecovered() {
+        this.redisAvailable = true;
+        this.logger.log('Redis connectivity restored — rejoining cluster');
+        this.registerNode()
+            .then(() => this.incrementRingVersion())
+            .then(() => this.publishEvent('join'))
+            .catch((err) => {
+            this.logger.error(`Failed to rejoin cluster: ${err.message}`);
+        });
+        if (!this.heartbeatTimer) {
+            this.heartbeatTimer = setInterval(() => {
+                this.heartbeat().catch((err) => {
+                    this.logger.error(`Heartbeat failed: ${err.message}`);
+                });
+            }, this.heartbeatIntervalMs);
+        }
+        if (!this.reconcileTimer) {
+            this.reconcileTimer = setInterval(() => {
+                this.reconcile().catch((err) => {
+                    this.logger.error(`Reconciliation failed: ${err.message}`);
+                });
+            }, this.reconcileIntervalMs);
+        }
+    }
+    // =========================================================================
+    // INTERNAL — Helpers
+    // =========================================================================
+    getNodeKey() {
+        return `${this.keyPrefix}:cluster:nodes:${this.serverId}:${this.instanceId}`;
+    }
+    getNodeIndexKey() {
+        return `${this.keyPrefix}:cluster:node-index`;
+    }
+    getEventsChannel() {
+        return `${this.keyPrefix}:cluster:events`;
+    }
+    async incrementRingVersion() {
+        const versionKey = `${this.keyPrefix}:cluster:ring:version`;
+        return this.redis.incr(versionKey);
+    }
+    parseNodeData(data) {
+        return {
+            serverId: data.server_id,
+            instanceId: data.instance_id || undefined,
+            grpcAddress: data.grpc_address,
+            serviceGroup: data.service_group,
+            entityTypes: data.entity_types ? data.entity_types.split(',').filter(Boolean) : [],
+            ringVersion: parseInt(data.ring_version || '0', 10),
+            startedAt: parseInt(data.started_at || '0', 10),
+            heartbeatAt: parseInt(data.heartbeat_at || '0', 10),
+        };
+    }
+    async scanKeys(pattern) {
+        let cursor = '0';
+        const keys = [];
+        do {
+            const [nextCursor, foundKeys] = await this.redis.scan(cursor, 'MATCH', pattern, 'COUNT', 100);
+            cursor = nextCursor;
+            keys.push(...foundKeys);
+        } while (cursor !== '0');
+        return keys;
+    }
+};
+exports.ClusterDiscoveryService = ClusterDiscoveryService;
+exports.ClusterDiscoveryService = ClusterDiscoveryService = ClusterDiscoveryService_1 = __decorate([
+    (0, common_1.Injectable)(),
+    __param(0, (0, common_1.Inject)(constants_1.ATOMIC_QUEUES_REDIS)),
+    __param(1, (0, common_1.Inject)(constants_1.ATOMIC_QUEUES_CONFIG)),
+    __param(2, (0, common_1.Optional)()),
+    __param(3, (0, common_1.Optional)()),
+    __metadata("design:paramtypes", [ioredis_1.default, Object, grpc_peer_monitor_service_1.GrpcPeerMonitor,
+        redis_health_monitor_service_1.RedisHealthMonitor])
+], ClusterDiscoveryService);
+//# sourceMappingURL=cluster-discovery.service.js.map

package/dist/cluster/cluster-discovery.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cluster-discovery.service.js","sourceRoot":"","sources":["../../src/cluster/cluster-discovery.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;AAAA,2CAOwB;AACxB,sDAA4B;AAC5B,+BAAoC;AAEpC,oCAA4C;AAC5C,qDAAkF;AAClF,2EAA8D;AAC9D,iFAAoE;AAapE,MAAM,gBAAgB,GAAG;;;;CAIxB,CAAC;AAEF;;;;;;;GAOG;AAEI,IAAM,uBAAuB,+BAA7B,MAAM,uBAAuB;IAwBlC,YAC+B,KAA6B,EAC5B,MAAkD,EACpE,WAA8C,EAC9C,kBAAwD;QAHtB,UAAK,GAAL,KAAK,CAAO;QACX,WAAM,GAAN,MAAM,CAA2B;QACnD,gBAAW,GAAX,WAAW,CAAkB;QAC7B,uBAAkB,GAAlB,kBAAkB,CAAqB;QA3BrD,WAAM,GAAG,IAAI,eAAM,CAAC,yBAAuB,CAAC,IAAI,CAAC,CAAC;QAUlD,eAAU,GAAW,IAAA,SAAM,GAAE,CAAC;QACvC,wBAAmB,GAAG,CAAC,CAAC;QAExB,mBAAc,GAA0B,IAAI,CAAC;QAC7C,mBAAc,GAA0B,IAAI,CAAC;QAC7C,wBAAmB,GAA0B,IAAI,CAAC;QAClD,eAAU,GAAiB,IAAI,CAAC;QAChC,mBAAc,GAAG,IAAI,CAAC;QACtB,2BAAsB,GAAwB,IAAI,CAAC;QACnD,2BAAsB,GAAwB,IAAI,CAAC;QAE1C,oBAAe,GAA0C,EAAE,CAAC;QAQ3E,IAAI,CAAC,SAAS,GAAG,IAAA,wBAAgB,EAAC,MAAM,CAAC,CAAC;QAC1C,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,IAAI,EAAE,OAAO,IAAI,KAAK,CAAC;QAC7C,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,IAAI,EAAE,QAAQ,IAAI,SAAS,CAAC;QACnD,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,IAAI,EAAE,YAAY,IAAI,SAAS,CAAC;QAC3D,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,IAAI,EAAE,iBAAiB,IAAI,eAAe,CAAC;QACrE,IAAI,CAAC,mBAAmB,GAAG,MAAM,CAAC,IAAI,EAAE,WAAW,IAAI,GAAG,CAAC;QAC3D,IAAI,CAAC,mBAAmB,GAAG,MAAM,CAAC,IAAI,EAAE,mBAAmB,IAAI,IAAI,CAAC;QACpE,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,IAAI,EAAE,SAAS,IAAI,IAAI,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,YAAY;QAChB,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,OAAO;QAE1B,gEAAgE;QAChE,MAAM,UAAU,GAAG,GAAG,IAAI,CAAC,SAAS,kBAAkB,IAAI,CAAC,QAAQ,IAAI,CAAC;QACxE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAChD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC;YACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,EAAE,CAAC;YACxC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;YACvC,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;gBAC1B,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,SAAS,iBAAiB,EAAE,EAAE,CAAC,CAAC;gBACnE,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAClC,CAAC;YACD,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACtB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW,OAAO,CAAC,MAAM,2CAA2C,CAAC,CAAC;QACxF,CAAC;QAED,qBAAqB;QACrB,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAE1B,yBAAyB;QACzB,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAElC,qBAAqB;QACrB,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QAEhC,kBAAkB;QAClB,IAAI,CAAC,cAAc,GAAG,WAAW,CAAC,GAAG,EAAE;YACrC,IAAI,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBAC7B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qBAAsB,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;YACnE,CAAC,CAAC,CAAC;QACL,CAAC,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAE7B,2EAA2E;QAC3E,IAAI,CAAC,cAAc,GAAG,WAAW,CAAC,GAAG,EAAE;YACrC,IAAI,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBAC7B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,0BAA2B,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;YACxE,CAAC,CAAC,CAAC;QACL,CAAC,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAE7B,2BAA2B;QAC3B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxC,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACzC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAW,EAAE,OAAe,EAAE,EAAE;YAC7D,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;QAC5B,CAAC,CAAC,CAAC;QAEH,kEAAkE;QAClE,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC,WAAW,CAAC,iBAAiB,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;gBACjF,IAAI,KAAK,KAAK,gBAAgB,EAAE,CAAC;oBAC/B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,MAAM,iDAAiD,CAAC,CAAC;oBAClF,IAAI,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;wBAC7B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,+BAAgC,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;oBAC7E,CAAC,CAAC,CAAC;gBACL,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QAED,4DAA4D;QAC5D,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC5B,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC,kBAAkB,CAAC,cAAc,CAAC,CAAC,OAAO,EAAE,EAAE;gBAC/E,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,IAAI,CAAC,eAAe,EAAE,CAAC;gBACzB,CAAC;qBAAM,CAAC;oBACN,IAAI,CAAC,oBAAoB,EAAE,CAAC;gBAC9B,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,GAAG,CACb,uCAAuC,IAAI,CAAC,QAAQ,WAAW,IAAI,CAAC,YAAY,EAAE,CACnF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,qBAAqB;QACzB,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,OAAO;QAE1B,IAAI,IAAI,CAAC,cAAc;YAAE,aAAa,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAC5D,IAAI,IAAI,CAAC,cAAc;YAAE,aAAa,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAC5D,IAAI,IAAI,CAAC,mBAAmB;YAAE,YAAY,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QACrE,IAAI,IAAI,CAAC,sBAAsB;YAAE,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC/D,IAAI,IAAI,CAAC,sBAAsB;YAAE,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAE/D,iCAAiC;QACjC,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QACxC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,EAAE,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QACrF,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAClC,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;QAEjC,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,EAAE,CAAC;YACpC,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;QAC/B,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;IAC/C,CAAC;IAED,4EAA4E;IAC5E,aAAa;IACb,4EAA4E;IAE5E;;OAEG;IACH,KAAK,CAAC,QAAQ;QACZ,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,EAAE,CAAC;QACxC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAEpD,MAAM,KAAK,GAAkB,EAAE,CAAC;QAChC,MAAM,YAAY,GAAa,EAAE,CAAC;QAElC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,OAAO,GAAG,GAAG,IAAI,CAAC,SAAS,kBAAkB,MAAM,EAAE,CAAC;YAC5D,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAC/C,IAAI,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAC3B,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC;YACvC,CAAC;iBAAM,CAAC;gBACN,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;YACvC,KAAK,MAAM,MAAM,IAAI,YAAY,EAAE,CAAC;gBAClC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAClC,CAAC;YACD,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAClC,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc;QAClB,MAAM,UAAU,GAAG,GAAG,IAAI,CAAC,SAAS,uBAAuB,CAAC;QAC5D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACjD,OAAO,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7C,CAAC;IAED;;OAEG;IACH,YAAY,CAAC,QAAwC;QACnD,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACpC,OAAO,GAAG,EAAE;YACV,MAAM,GAAG,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YACnD,IAAI,GAAG,IAAI,CAAC;gBAAE,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QACpD,CAAC,CAAC;IACJ,CAAC;IAED;;;OAGG;IACH,gBAAgB;QACd,OAAO,IAAI,CAAC,cAAc,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,WAAW;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mBAAmB,CAAC,UAAkB;QAC1C,MAAM,WAAW,GAAG,GAAG,IAAI,CAAC,SAAS,4BAA4B,UAAU,EAAE,CAAC;QAC9E,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IACrC,CAAC;IAED,4EAA4E;IAC5E,0BAA0B;IAC1B,4EAA4E;IAEpE,KAAK,CAAC,YAAY;QACxB,MAAM,GAAG,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;QAC9B,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,EAAE,CAAC;QACxC,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;QAC5D,MAAM,IAAI,GAA2B;YACnC,SAAS,EAAE,IAAI,CAAC,QAAQ;YACxB,WAAW,EAAE,IAAI,CAAC,UAAU;YAC5B,YAAY,EAAE,IAAI,CAAC,WAAW;YAC9B,aAAa,EAAE,IAAI,CAAC,YAAY;YAChC,YAAY,EAAE,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC;YACnC,YAAY,EAAE,GAAG;YACjB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;YACjC,YAAY,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;SACpC,CAAC;QAEF,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QACjC,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QAEvE,yEAAyE;QACzE,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE,CAAC;YAC7B,MAAM,WAAW,GAAG,GAAG,IAAI,CAAC,SAAS,4BAA4B,EAAE,EAAE,CAAC;YACtE,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,CAAC,YAAY,EAAE,IAAI,EAAE,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;QAC/E,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,SAAS;QACrB,MAAM,GAAG,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;QAC9B,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;QAC5D,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;QAEvC,QAAQ,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;QACxF,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE,CAAC;YAC7B,QAAQ,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,SAAS,4BAA4B,EAAE,EAAE,EAAE,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;QACxF,CAAC;QAED,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACxB,CAAC;IAEO,KAAK,CAAC,SAAS;QACrB,IAAI,CAAC,IAAI,CAAC,cAAc;YAAE,OAAO;QAEjC,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QACnD,IAAI,cAAc,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC9C,IAAI,CAAC,mBAAmB,GAAG,cAAc,CAAC;QAC5C,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;QAEpC,wDAAwD;QACxD,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,IAAI,CAAC,WAAW,CAAC,SAAS,CACxB,KAAK;iBACF,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,IAAI,CAAC,QAAQ,CAAC;iBAC3C,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAClE,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;IAC9B,CAAC;IAED,4EAA4E;IAC5E,oBAAoB;IACpB,4EAA4E;IAEpE,KAAK,CAAC,YAAY,CAAC,MAAwB;QACjD,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAChD,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CACtB,OAAO,EACP,IAAI,CAAC,SAAS,CAAC;YACb,MAAM;YACN,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;YACrB,WAAW;SACZ,CAAC,CACH,CAAC;IACJ,CAAC;IAEO,WAAW,CAAC,OAAe;QACjC,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAClC,IAAI,KAAK,CAAC,QAAQ,KAAK,IAAI,CAAC,QAAQ;gBAAE,OAAO;YAE7C,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,eAAe,KAAK,CAAC,MAAM,SAAS,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;YAEtE,MAAM,eAAe,GAAG,KAAK,CAAC,WAAiC,CAAC;YAChE,MAAM,WAAW,GACf,eAAe,KAAK,SAAS;gBAC7B,IAAI,CAAC,mBAAmB,GAAG,CAAC;gBAC5B,eAAe,GAAG,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;YAEjD,IAAI,eAAe,KAAK,SAAS,IAAI,eAAe,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBAChF,IAAI,CAAC,mBAAmB,GAAG,eAAe,CAAC;YAC7C,CAAC;YAED,IAAI,WAAW,EAAE,CAAC;gBAChB,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,0CAA0C,IAAI,CAAC,mBAAmB,SAAS,eAAe,yBAAyB,CACpH,CAAC;gBACF,IAAI,IAAI,CAAC,mBAAmB;oBAAE,YAAY,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;gBACrE,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;gBAChC,IAAI,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAC7B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mCAAoC,GAAa,CAAC,OAAO,EAAE,CAAC,CAC/E,CAAC;gBACF,OAAO;YACT,CAAC;YAED,IAAI,IAAI,CAAC,mBAAmB;gBAAE,YAAY,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;YACrE,IAAI,CAAC,mBAAmB,GAAG,UAAU,CAAC,GAAG,EAAE;gBACzC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;gBAChC,IAAI,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAC7B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qCAAsC,GAAa,CAAC,OAAO,EAAE,CAAC,CACjF,CAAC;YACJ,CAAC,EAAE,GAAG,CAAC,CAAC;QACV,CAAC;QAAC,MAAM,CAAC;YACP,0BAA0B;QAC5B,CAAC;IACH,CAAC;IAEO,eAAe,CAAC,KAAoB;QAC1C,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YAC5C,IAAI,CAAC;gBACH,QAAQ,CAAC,KAAK,CAAC,CAAC;YAClB,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,+BAAgC,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;YAC7E,CAAC;QACH,CAAC;IACH,CAAC;IAED,4EAA4E;IAC5E,+CAA+C;IAC/C,4EAA4E;IAEpE,eAAe;QACrB,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;QAC5B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,sDAAsD,CAAC,CAAC;QAE1E,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,aAAa,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACnC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC7B,CAAC;QACD,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,aAAa,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACnC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC7B,CAAC;QAED,+EAA+E;QAC/E,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC;IAC3B,CAAC;IAEO,oBAAoB;QAC1B,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC3B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,iDAAiD,CAAC,CAAC;QAEnE,IAAI,CAAC,YAAY,EAAE;aAChB,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC;aACvC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;aACrC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACb,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,6BAA8B,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;QAC3E,CAAC,CAAC,CAAC;QAEL,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;YACzB,IAAI,CAAC,cAAc,GAAG,WAAW,CAAC,GAAG,EAAE;gBACrC,IAAI,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;oBAC7B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qBAAsB,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;gBACnE,CAAC,CAAC,CAAC;YACL,CAAC,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAC/B,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;YACzB,IAAI,CAAC,cAAc,GAAG,WAAW,CAAC,GAAG,EAAE;gBACrC,IAAI,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;oBAC7B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,0BAA2B,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;gBACxE,CAAC,CAAC,CAAC;YACL,CAAC,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,4EAA4E;IAC5E,qBAAqB;IACrB,4EAA4E;IAEpE,UAAU;QAChB,OAAO,GAAG,IAAI,CAAC,SAAS,kBAAkB,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;IAC/E,CAAC;IAEO,eAAe;QACrB,OAAO,GAAG,IAAI,CAAC,SAAS,qBAAqB,CAAC;IAChD,CAAC;IAEO,gBAAgB;QACtB,OAAO,GAAG,IAAI,CAAC,SAAS,iBAAiB,CAAC;IAC5C,CAAC;IAEO,KAAK,CAAC,oBAAoB;QAChC,MAAM,UAAU,GAAG,GAAG,IAAI,CAAC,SAAS,uBAAuB,CAAC;QAC5D,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACrC,CAAC;IAEO,aAAa,CAAC,IAA4B;QAChD,OAAO;YACL,QAAQ,EAAE,IAAI,CAAC,SAAS;YACxB,UAAU,EAAE,IAAI,CAAC,WAAW,IAAI,SAAS;YACzC,WAAW,EAAE,IAAI,CAAC,YAAY;YAC9B,YAAY,EAAE,IAAI,CAAC,aAAa;YAChC,WAAW,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE;YAClF,WAAW,EAAE,QAAQ,CAAC,IAAI,CAAC,YAAY,IAAI,GAAG,EAAE,EAAE,CAAC;YACnD,SAAS,EAAE,QAAQ,CAAC,IAAI,CAAC,UAAU,IAAI,GAAG,EAAE,EAAE,CAAC;YAC/C,WAAW,EAAE,QAAQ,CAAC,IAAI,CAAC,YAAY,IAAI,GAAG,EAAE,EAAE,CAAC;SACpD,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,OAAe;QACpC,IAAI,MAAM,GAAG,GAAG,CAAC;QACjB,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,GAAG,CAAC;YACF,MAAM,CAAC,UAAU,EAAE,SAAS,CAAC,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC;YAC9F,MAAM,GAAG,UAAU,CAAC;YACpB,IAAI,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;QAC1B,CAAC,QAAQ,MAAM,KAAK,GAAG,EAAE;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;CACF,CAAA;AA9bY,0DAAuB;kCAAvB,uBAAuB;IADnC,IAAA,mBAAU,GAAE;IA0BR,WAAA,IAAA,eAAM,EAAC,+BAAmB,CAAC,CAAA;IAC3B,WAAA,IAAA,eAAM,EAAC,gCAAoB,CAAC,CAAA;IAC5B,WAAA,IAAA,iBAAQ,GAAE,CAAA;IACV,WAAA,IAAA,iBAAQ,GAAE,CAAA;qCAH0C,iBAAK,UAEf,2CAAe;QACR,iDAAkB;GA5B3D,uBAAuB,CA8bnC"}

package/dist/cluster/grpc-peer-monitor.service.d.ts

@@ -0,0 +1,31 @@
+import { OnModuleInit, OnApplicationShutdown } from '@nestjs/common';
+import { IAtomicQueuesModuleConfig } from '../domain';
+export type PeerLivenessState = 'alive' | 'suspected-dead' | 'unknown';
+export declare class GrpcPeerMonitor implements OnModuleInit, OnApplicationShutdown {
+    private readonly config;
+    private readonly logger;
+    private readonly enabled;
+    private readonly debounceMs;
+    private readonly keepaliveTimeMs;
+    private readonly keepaliveTimeoutMs;
+    private readonly connectivityWatchMs;
+    private grpcModule;
+    private readonly peers;
+    private readonly stateChangeListeners;
+    constructor(config: IAtomicQueuesModuleConfig);
+    onModuleInit(): Promise<void>;
+    onApplicationShutdown(): Promise<void>;
+    getPeerState(serverId: string): PeerLivenessState;
+    onPeerStateChange(listener: (serverId: string, state: PeerLivenessState) => void): () => void;
+    watchPeer(serverId: string, address: string): void;
+    unwatchPeer(serverId: string): void;
+    syncPeers(peers: Array<{
+        serverId: string;
+        address: string;
+    }>): void;
+    private startWatchLoop;
+    private handleStateTransition;
+    private cleanupPeer;
+    private notifyListeners;
+}
+//# sourceMappingURL=grpc-peer-monitor.service.d.ts.map

package/dist/cluster/grpc-peer-monitor.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"grpc-peer-monitor.service.d.ts","sourceRoot":"","sources":["../../src/cluster/grpc-peer-monitor.service.ts"],"names":[],"mappings":"AAAA,OAAO,EAA8B,YAAY,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AACjG,OAAO,EAAE,yBAAyB,EAAE,MAAM,WAAW,CAAC;AAGtD,MAAM,MAAM,iBAAiB,GAAG,OAAO,GAAG,gBAAgB,GAAG,SAAS,CAAC;AAmCvE,qBACa,eAAgB,YAAW,YAAY,EAAE,qBAAqB;IAc/B,OAAO,CAAC,QAAQ,CAAC,MAAM;IAbjE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAoC;IAC3D,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAU;IAClC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAS;IACzC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAS;IAC5C,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAS;IAE7C,OAAO,CAAC,UAAU,CAA2B;IAC7C,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAgC;IACtD,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAE9B;gBAEoD,MAAM,EAAE,yBAAyB;IAQtF,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAY7B,qBAAqB,IAAI,OAAO,CAAC,IAAI,CAAC;IAW5C,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,iBAAiB;IAIjD,iBAAiB,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,iBAAiB,KAAK,IAAI,GAAG,MAAM,IAAI;IAQ7F,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI;IAgClD,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAOnC,SAAS,CAAC,KAAK,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,GAAG,IAAI;IAgBpE,OAAO,CAAC,cAAc;IAuBtB,OAAO,CAAC,qBAAqB;IAsC7B,OAAO,CAAC,WAAW;IAanB,OAAO,CAAC,eAAe;CASxB"}

package/dist/cluster/grpc-peer-monitor.service.js

@@ -0,0 +1,192 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
+var GrpcPeerMonitor_1;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GrpcPeerMonitor = void 0;
+const common_1 = require("@nestjs/common");
+const constants_1 = require("../services/constants");
+let GrpcPeerMonitor = GrpcPeerMonitor_1 = class GrpcPeerMonitor {
+    constructor(config) {
+        this.config = config;
+        this.logger = new common_1.Logger(GrpcPeerMonitor_1.name);
+        this.grpcModule = null;
+        this.peers = new Map();
+        this.stateChangeListeners = [];
+        this.enabled = (config.grpc?.enabled ?? false) && (config.grpc?.peerMonitorEnabled ?? true);
+        this.debounceMs = config.grpc?.peerSuspectDebounceMs ?? 500;
+        this.keepaliveTimeMs = config.grpc?.keepaliveTimeMs ?? 10000;
+        this.keepaliveTimeoutMs = config.grpc?.keepaliveTimeoutMs ?? 5000;
+        this.connectivityWatchMs = config.grpc?.deadlines?.connectivityWatchMs ?? 30000;
+    }
+    async onModuleInit() {
+        if (!this.enabled)
+            return;
+        try {
+            // eslint-disable-next-line @typescript-eslint/no-var-requires
+            this.grpcModule = require('@grpc/grpc-js');
+            this.logger.log('gRPC peer monitor initialized');
+        }
+        catch {
+            this.logger.warn('gRPC peer monitor disabled: @grpc/grpc-js not installed');
+        }
+    }
+    async onApplicationShutdown() {
+        for (const [, peer] of this.peers) {
+            this.cleanupPeer(peer);
+        }
+        this.peers.clear();
+    }
+    // =========================================================================
+    // PUBLIC API
+    // =========================================================================
+    getPeerState(serverId) {
+        return this.peers.get(serverId)?.state ?? 'unknown';
+    }
+    onPeerStateChange(listener) {
+        this.stateChangeListeners.push(listener);
+        return () => {
+            const idx = this.stateChangeListeners.indexOf(listener);
+            if (idx >= 0)
+                this.stateChangeListeners.splice(idx, 1);
+        };
+    }
+    watchPeer(serverId, address) {
+        if (!this.grpcModule)
+            return;
+        const existing = this.peers.get(serverId);
+        if (existing && existing.address === address)
+            return;
+        if (existing)
+            this.cleanupPeer(existing);
+        const channelOptions = {
+            'grpc.keepalive_time_ms': this.keepaliveTimeMs,
+            'grpc.keepalive_timeout_ms': this.keepaliveTimeoutMs,
+            'grpc.keepalive_permit_without_calls': 1,
+        };
+        const client = new this.grpcModule.Client(address, this.grpcModule.credentials.createInsecure(), channelOptions);
+        const entry = {
+            serverId,
+            address,
+            state: 'unknown',
+            client,
+            suspectTimer: null,
+            watchActive: true,
+        };
+        this.peers.set(serverId, entry);
+        this.startWatchLoop(entry);
+    }
+    unwatchPeer(serverId) {
+        const peer = this.peers.get(serverId);
+        if (!peer)
+            return;
+        this.cleanupPeer(peer);
+        this.peers.delete(serverId);
+    }
+    syncPeers(peers) {
+        const incoming = new Set(peers.map((p) => p.serverId));
+        for (const [id] of this.peers) {
+            if (!incoming.has(id))
+                this.unwatchPeer(id);
+        }
+        for (const p of peers) {
+            this.watchPeer(p.serverId, p.address);
+        }
+    }
+    // =========================================================================
+    // INTERNAL — connectivity state watch loop
+    // =========================================================================
+    startWatchLoop(entry) {
+        if (!this.grpcModule || !entry.watchActive)
+            return;
+        const channel = entry.client.getChannel();
+        const currentState = channel.getConnectivityState(true);
+        this.handleStateTransition(entry, currentState);
+        const deadline = new Date(Date.now() + this.connectivityWatchMs);
+        channel.watchConnectivityState(currentState, deadline, (err) => {
+            if (!entry.watchActive)
+                return;
+            if (err) {
+                // Deadline expired without state change — re-arm
+                this.startWatchLoop(entry);
+                return;
+            }
+            // State changed — process and re-arm
+            this.startWatchLoop(entry);
+        });
+    }
+    handleStateTransition(entry, grpcState) {
+        if (!this.grpcModule)
+            return;
+        const { connectivityState } = this.grpcModule;
+        const READY = connectivityState.READY;
+        const TRANSIENT_FAILURE = connectivityState.TRANSIENT_FAILURE;
+        const SHUTDOWN = connectivityState.SHUTDOWN;
+        if (grpcState === READY) {
+            if (entry.suspectTimer) {
+                clearTimeout(entry.suspectTimer);
+                entry.suspectTimer = null;
+            }
+            if (entry.state !== 'alive') {
+                entry.state = 'alive';
+                this.notifyListeners(entry.serverId, 'alive');
+            }
+        }
+        else if (grpcState === TRANSIENT_FAILURE || grpcState === SHUTDOWN) {
+            if (entry.state === 'alive' && !entry.suspectTimer) {
+                entry.suspectTimer = setTimeout(() => {
+                    entry.suspectTimer = null;
+                    if (entry.state !== 'suspected-dead') {
+                        entry.state = 'suspected-dead';
+                        this.logger.warn(`Peer ${entry.serverId} suspected dead (gRPC ${grpcState === SHUTDOWN ? 'SHUTDOWN' : 'TRANSIENT_FAILURE'})`);
+                        this.notifyListeners(entry.serverId, 'suspected-dead');
+                    }
+                }, this.debounceMs);
+            }
+        }
+        // IDLE and CONNECTING: no action, wait for READY or TRANSIENT_FAILURE
+    }
+    // =========================================================================
+    // INTERNAL — helpers
+    // =========================================================================
+    cleanupPeer(peer) {
+        peer.watchActive = false;
+        if (peer.suspectTimer) {
+            clearTimeout(peer.suspectTimer);
+            peer.suspectTimer = null;
+        }
+        try {
+            peer.client.close();
+        }
+        catch {
+            // ignore
+        }
+    }
+    notifyListeners(serverId, state) {
+        for (const listener of this.stateChangeListeners) {
+            try {
+                listener(serverId, state);
+            }
+            catch (err) {
+                this.logger.error(`Peer state change listener error: ${err.message}`);
+            }
+        }
+    }
+};
+exports.GrpcPeerMonitor = GrpcPeerMonitor;
+exports.GrpcPeerMonitor = GrpcPeerMonitor = GrpcPeerMonitor_1 = __decorate([
+    (0, common_1.Injectable)(),
+    __param(0, (0, common_1.Inject)(constants_1.ATOMIC_QUEUES_CONFIG)),
+    __metadata("design:paramtypes", [Object])
+], GrpcPeerMonitor);
+//# sourceMappingURL=grpc-peer-monitor.service.js.map