@xenon-device-management/xenon 1.1.0

package/lib/src/services/SessionHeartbeatService.js

@@ -0,0 +1,158 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SessionHeartbeatService = void 0;
+const typedi_1 = require("typedi");
+const SessionManager_1 = require("../sessions/SessionManager");
+const device_store_1 = require("../data-service/device-store");
+const logger_1 = __importDefault(require("../logger"));
+const SessionStatus_1 = require("../types/SessionStatus");
+const prisma_1 = require("../prisma");
+const SessionLifecycleService_1 = require("./SessionLifecycleService");
+const XenonSession_1 = require("../sessions/XenonSession");
+let SessionHeartbeatService = class SessionHeartbeatService {
+    constructor() {
+        this.log = logger_1.default.scope('SessionHeartbeat');
+        this.failureCounts = new Map();
+        this.THRESHOLD_DEGRADED = 1;
+        this.THRESHOLD_SUSPECT = 3;
+        this.THRESHOLD_DEAD = 6;
+    }
+    start(pluginArgs) {
+        const intervalMs = pluginArgs.sessionHeartbeatIntervalMs || 30000;
+        this.log.info(`Starting Session Heartbeat Service (Interval: ${intervalMs}ms)`);
+        this.interval = setInterval(() => this.checkAllSessions(), intervalMs);
+    }
+    stop() {
+        if (this.interval) {
+            clearInterval(this.interval);
+            this.interval = undefined;
+        }
+    }
+    checkAllSessions() {
+        return __awaiter(this, void 0, void 0, function* () {
+            // 1. Local memory check (fast path)
+            const localSessions = SessionManager_1.SESSION_MANAGER.getAllSessions();
+            const activeSessionIds = new Set(localSessions.map((s) => s.getId()));
+            // Cleanup failure counts for closed sessions
+            for (const sid of this.failureCounts.keys()) {
+                if (!activeSessionIds.has(sid)) {
+                    this.failureCounts.delete(sid);
+                }
+            }
+            for (const session of localSessions) {
+                // Principal Grace Period: If a session is already stopping, give it 60 seconds to finish
+                // its own cleanup (video processing, etc.) before we force it out.
+                if (session.isStopping && session.stoppedAt && Date.now() - session.stoppedAt < 60000) {
+                    continue;
+                }
+                const sessionId = session.getId();
+                try {
+                    const result = yield session.checkHealth();
+                    if (result.isHealthy) {
+                        // Recovery: Reset failure count and restore HEALTHY state
+                        if (this.failureCounts.get(sessionId) || 0 > 0) {
+                            this.log.info(`✨ Session ${sessionId} recovered and is now HEALTHY.`);
+                        }
+                        this.failureCounts.set(sessionId, 0);
+                        session.healthState = XenonSession_1.SessionHealthState.HEALTHY;
+                        continue;
+                    }
+                    const count = (this.failureCounts.get(sessionId) || 0) + 1;
+                    this.failureCounts.set(sessionId, count);
+                    // State Machine Transition Logic
+                    if (count >= this.THRESHOLD_DEAD) {
+                        session.healthState = XenonSession_1.SessionHealthState.DEAD;
+                        this.log.error(`💔 Session ${sessionId} reached DEAD state after ${count} failures. Error: ${result.errorType} - ${result.message}`);
+                        const reason = session.isStopping
+                            ? 'Shutdown timeout (Cleanup hung for > 60s)'
+                            : `Session terminal failure (${result.errorType}: ${result.message})`;
+                        // Trigger terminal cleanup
+                        const lifecycleService = typedi_1.Container.get(SessionLifecycleService_1.SessionLifecycleService);
+                        yield lifecycleService.deleteSession(() => __awaiter(this, void 0, void 0, function* () {
+                            /* No-op driver deletion */
+                        }), sessionId, SessionStatus_1.SessionStatus.FAILED, reason);
+                        this.failureCounts.delete(sessionId);
+                    }
+                    else if (count >= this.THRESHOLD_SUSPECT) {
+                        session.healthState = XenonSession_1.SessionHealthState.SUSPECT;
+                        this.log.warn(`🕵️ Session ${sessionId} is SUSPECT (${count}/${this.THRESHOLD_DEAD}). Evidence: ${result.errorType} - ${result.message}`);
+                    }
+                    else if (count >= this.THRESHOLD_DEGRADED) {
+                        session.healthState = XenonSession_1.SessionHealthState.DEGRADED;
+                        this.log.debug(`⚠️ Session ${sessionId} is DEGRADED (${count}/${this.THRESHOLD_DEAD}).`);
+                    }
+                }
+                catch (err) {
+                    this.log.error(`Unexpected error checking health for session ${sessionId}: ${err.message}`);
+                }
+            }
+            // 2. Global Database Check (Senior Tier: Multi-Hub resilience)
+            try {
+                const zombieThreshold = new Date(Date.now() - 10 * 60 * 1000);
+                const zombies = yield prisma_1.prisma.session.findMany({
+                    where: {
+                        status: 'running',
+                        updatedAt: { lt: zombieThreshold },
+                    },
+                    include: { build: true },
+                });
+                if (zombies.length > 0) {
+                    this.log.info(`🕵️ Found ${zombies.length} global zombie sessions. Harmonizing fleet state...`);
+                    for (const zombie of zombies) {
+                        yield this.cleanupGlobalZombie(zombie);
+                    }
+                }
+            }
+            catch (err) {
+                this.log.error(`Global zombie cleanup failed: ${err.message}`);
+            }
+        });
+    }
+    cleanupGlobalZombie(sessionData) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const { id, device_udid, node_id } = sessionData;
+            try {
+                // Unblock device in DB
+                yield device_store_1.DeviceStoreFactory.getStore().updateDevice(device_udid, node_id, {
+                    busy: false,
+                    session_id: null,
+                });
+                // Mark session as failed in DB
+                yield prisma_1.prisma.session.update({
+                    where: { id },
+                    data: {
+                        status: 'failed',
+                        endTime: new Date(),
+                        failure_reason: 'Session orphaned (Hub instance timeout)',
+                    },
+                });
+                this.log.info(`✅ Global zombie ${id} neutralized and device ${device_udid} released.`);
+            }
+            catch (err) {
+                this.log.error(`Failed to cleanup global zombie ${id}: ${err.message}`);
+            }
+        });
+    }
+};
+exports.SessionHeartbeatService = SessionHeartbeatService;
+exports.SessionHeartbeatService = SessionHeartbeatService = __decorate([
+    (0, typedi_1.Service)()
+], SessionHeartbeatService);