@xenon-device-management/xenon 1.2.0 → 1.3.0

package/lib/src/services/DeviceReconciler.js

@@ -0,0 +1,102 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+var DeviceReconciler_1;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DeviceReconciler = void 0;
+const typedi_1 = require("typedi");
+const logger_1 = __importDefault(require("../logger"));
+const device_store_1 = require("../data-service/device-store");
+const SessionManager_1 = require("../sessions/SessionManager");
+const device_service_1 = require("../data-service/device-service");
+// Closes a gap left by OrphanSweeper (which operates on prisma.session + the
+// Prisma device table via a heartbeat cutoff) and releaseBlockedDevices
+// (which only triggers on newCommandTimeout after a session has actually run
+// a command). Neither covers the case where:
+//   - blockDevice() marks a device busy with session_id = X
+//   - but X never makes it into SESSION_MANAGER (driver crash during startup,
+//     allocation failed after the lock but before registration, Loki store
+//     drift, etc.)
+//   - the session also never reaches Prisma with a heartbeat, so OrphanSweeper
+//     doesn't see it; lastCmdExecutedAt stays undefined, so
+//     releaseBlockedDevices skips it too.
+// Result: a ghost "busy" device that blocks allocation forever. This sweep
+// cross-references the device store against SESSION_MANAGER and unblocks
+// those ghosts.
+let DeviceReconciler = DeviceReconciler_1 = class DeviceReconciler {
+    constructor() {
+        this.logger = logger_1.default.scope('DeviceReconciler');
+        this.orphansFreed = 0;
+    }
+    reconcile() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const store = device_store_1.DeviceStoreFactory.getStore();
+            let devices;
+            try {
+                devices = yield store.getAllDevices();
+            }
+            catch (err) {
+                this.logger.error(`getAllDevices failed: ${err.message}`);
+                return;
+            }
+            const now = Date.now();
+            const orphans = devices.filter((d) => {
+                if (!d.busy || !d.session_id)
+                    return false;
+                // Device is busy but isn't tied to an active session at the driver
+                // layer. A known session in SESSION_MANAGER means a real driver is
+                // holding it — leave those alone.
+                if (SessionManager_1.SESSION_MANAGER.isValidSession(d.session_id))
+                    return false;
+                // Respect the bootstrap grace window: if sessionStartTime is 0 or
+                // null we don't know when allocation happened, so fall back to
+                // requiring SESSION_MANAGER knowledge (i.e. don't touch it).
+                if (!d.sessionStartTime || d.sessionStartTime <= 0)
+                    return false;
+                return now - d.sessionStartTime >= DeviceReconciler_1.MIN_BUSY_AGE_MS;
+            });
+            if (orphans.length === 0)
+                return;
+            this.logger.warn(`Found ${orphans.length} ghost device(s); releasing: ${orphans
+                .map((d) => `${d.udid}@${d.host}(session=${d.session_id})`)
+                .join(', ')}`);
+            for (const device of orphans) {
+                try {
+                    yield (0, device_service_1.unblockDevice)(device.udid, device.host);
+                    this.orphansFreed++;
+                }
+                catch (err) {
+                    this.logger.error(`Failed to unblock ghost device ${device.udid}: ${err.message}`);
+                }
+            }
+        });
+    }
+    getOrphansFreedCount() {
+        return this.orphansFreed;
+    }
+};
+exports.DeviceReconciler = DeviceReconciler;
+// Grace window after sessionStartTime within which we trust the session
+// is still bootstrapping (driver.createSession can legitimately take
+// 30s+ for simulator boot). Below this age we skip the device even if
+// SESSION_MANAGER doesn't know it yet.
+DeviceReconciler.MIN_BUSY_AGE_MS = 60000;
+exports.DeviceReconciler = DeviceReconciler = DeviceReconciler_1 = __decorate([
+    (0, typedi_1.Service)()
+], DeviceReconciler);

package/lib/src/services/MetricsService.js

@@ -24,6 +24,20 @@ const SessionManager_1 = require("../sessions/SessionManager");
 const device_store_1 = require("../data-service/device-store");
 const prisma_1 = require("../prisma");
 const logger_1 = __importDefault(require("../logger"));
+const HealingMetrics_1 = require("./healing/HealingMetrics");
+const CircuitBreaker_1 = require("./CircuitBreaker");
+const DeviceReconciler_1 = require("./DeviceReconciler");
+const ProcessMetricsService_1 = require("./ProcessMetricsService");
+// Label values must escape backslash, double-quote, and newline per the
+// Prometheus exposition format. Breaker keys contain colons which are fine.
+function escapeLabel(value) {
+    return value.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, '\\n');
+}
+const BREAKER_STATE_CODE = {
+    closed: 0,
+    half_open: 1,
+    open: 2,
+};
 let MetricsService = class MetricsService {
     constructor() {
         this.CONFIG_ID = 'metrics';
@@ -88,6 +102,7 @@ let MetricsService = class MetricsService {
     }
     getMetrics() {
         return __awaiter(this, void 0, void 0, function* () {
+            var _a;
             const sessions = SessionManager_1.SESSION_MANAGER.getStats();
             const devices = yield device_store_1.DeviceStoreFactory.getStore().getAllDevices();
             const totalDevices = devices.length;
@@ -124,6 +139,69 @@ let MetricsService = class MetricsService {
                 `xenon_healing_total{status="attempt"} ${healingAttempts}`,
                 `xenon_healing_total{status="success"} ${healingSuccesses}`,
             ];
+            // Per-tier healing metrics (in-process; resets on restart — fine for Prom
+            // scrape semantics). Tells us which tier is actually earning its compute.
+            const tiers = HealingMetrics_1.HEALING_METRICS.snapshot();
+            if (tiers.length > 0) {
+                lines.push('# HELP xenon_heal_tier_attempts_total Healing attempts per tier', '# TYPE xenon_heal_tier_attempts_total counter');
+                for (const t of tiers) {
+                    const labels = `tier="${t.tier}",name="${escapeLabel(t.name)}"`;
+                    lines.push(`xenon_heal_tier_attempts_total{${labels}} ${t.attempts}`);
+                }
+                lines.push('# HELP xenon_heal_tier_successes_total Healing successes per tier', '# TYPE xenon_heal_tier_successes_total counter');
+                for (const t of tiers) {
+                    const labels = `tier="${t.tier}",name="${escapeLabel(t.name)}"`;
+                    lines.push(`xenon_heal_tier_successes_total{${labels}} ${t.successes}`);
+                }
+                lines.push('# HELP xenon_heal_tier_failures_total Healing failures per tier', '# TYPE xenon_heal_tier_failures_total counter');
+                for (const t of tiers) {
+                    const labels = `tier="${t.tier}",name="${escapeLabel(t.name)}"`;
+                    lines.push(`xenon_heal_tier_failures_total{${labels}} ${t.failures}`);
+                }
+                lines.push('# HELP xenon_heal_tier_duration_seconds_sum Cumulative time spent in each tier', '# TYPE xenon_heal_tier_duration_seconds_sum counter');
+                for (const t of tiers) {
+                    const labels = `tier="${t.tier}",name="${escapeLabel(t.name)}"`;
+                    const seconds = (t.durationMsSum / 1000).toFixed(3);
+                    lines.push(`xenon_heal_tier_duration_seconds_sum{${labels}} ${seconds}`);
+                }
+            }
+            lines.push('# HELP xenon_heal_all_tiers_failed_total Healing calls where no tier matched', '# TYPE xenon_heal_all_tiers_failed_total counter', `xenon_heal_all_tiers_failed_total ${HealingMetrics_1.HEALING_METRICS.getAllTiersFailedCount()}`);
+            // Tier-advised skips. A rising trend on tier 3 usually means context
+            // collection (page source + screenshot) is failing upstream — the
+            // remaining tiers are being shed correctly but the root cause lives in
+            // the driver, not the heal pipeline.
+            const skips = HealingMetrics_1.HEALING_METRICS.skipSnapshot();
+            if (skips.length > 0) {
+                lines.push('# HELP xenon_heal_tier_skipped_remaining_total Times a tier short-circuited the remaining tiers', '# TYPE xenon_heal_tier_skipped_remaining_total counter');
+                for (const s of skips) {
+                    const labels = `tier="${s.tier}",name="${escapeLabel(s.name)}"`;
+                    lines.push(`xenon_heal_tier_skipped_remaining_total{${labels}} ${s.count}`);
+                }
+            }
+            // Device reconciliation: non-zero = ghost devices were being leaked and
+            // the reconciler caught them. A steady climb usually points at a bug in
+            // session allocation or shutdown.
+            lines.push('# HELP xenon_device_reconciler_orphans_freed_total Ghost devices released by the reconciler', '# TYPE xenon_device_reconciler_orphans_freed_total counter', `xenon_device_reconciler_orphans_freed_total ${typedi_1.Container.get(DeviceReconciler_1.DeviceReconciler).getOrphansFreedCount()}`);
+            // Hub process health. Heap growing while xenon_sessions_active is flat
+            // means a leak; event-loop lag climbing while CPU is idle means some
+            // handler is doing sync work on the hot path.
+            const proc = typedi_1.Container.get(ProcessMetricsService_1.ProcessMetricsService).snapshot();
+            lines.push('# HELP xenon_process_memory_bytes Node process memoryUsage breakdown', '# TYPE xenon_process_memory_bytes gauge', `xenon_process_memory_bytes{type="rss"} ${proc.memory.rss}`, `xenon_process_memory_bytes{type="heap_used"} ${proc.memory.heapUsed}`, `xenon_process_memory_bytes{type="heap_total"} ${proc.memory.heapTotal}`, `xenon_process_memory_bytes{type="external"} ${proc.memory.external}`, `xenon_process_memory_bytes{type="array_buffers"} ${proc.memory.arrayBuffers}`, '# HELP xenon_process_event_loop_lag_ms Most recent observed event-loop lag sample', '# TYPE xenon_process_event_loop_lag_ms gauge', `xenon_process_event_loop_lag_ms ${proc.eventLoop.lagMs}`, '# HELP xenon_process_event_loop_lag_max_ms Max event-loop lag since last scrape', '# TYPE xenon_process_event_loop_lag_max_ms gauge', `xenon_process_event_loop_lag_max_ms ${proc.eventLoop.maxLagSinceScrapeMs}`, '# HELP xenon_session_commands_processed_total All Appium commands intercepted (includes errors)', '# TYPE xenon_session_commands_processed_total counter', `xenon_session_commands_processed_total ${proc.commands.processed}`, '# HELP xenon_session_command_duration_ms_sum Cumulative command wall time', '# TYPE xenon_session_command_duration_ms_sum counter', `xenon_session_command_duration_ms_sum ${proc.commands.durationMsSum}`);
+            // Circuit breaker state — makes it obvious from a dashboard alert when
+            // an AI provider or any future wrapped dependency is shedding traffic.
+            // State encoded as int so Grafana can threshold on it easily:
+            //   0=closed (healthy), 1=half_open (probing), 2=open (shedding).
+            const breakers = CircuitBreaker_1.CIRCUIT_BREAKERS.snapshot();
+            if (breakers.length > 0) {
+                lines.push('# HELP xenon_circuit_breaker_state 0=closed, 1=half_open, 2=open', '# TYPE xenon_circuit_breaker_state gauge');
+                for (const b of breakers) {
+                    lines.push(`xenon_circuit_breaker_state{key="${escapeLabel(b.key)}"} ${(_a = BREAKER_STATE_CODE[b.state]) !== null && _a !== void 0 ? _a : 0}`);
+                }
+                lines.push('# HELP xenon_circuit_breaker_consecutive_failures Current failure streak', '# TYPE xenon_circuit_breaker_consecutive_failures gauge');
+                for (const b of breakers) {
+                    lines.push(`xenon_circuit_breaker_consecutive_failures{key="${escapeLabel(b.key)}"} ${b.consecutiveFailures}`);
+                }
+            }
             return lines.join('\n') + '\n';
         });
     }

package/lib/src/services/PortAllocator.js

@@ -54,6 +54,19 @@ let PortAllocator = class PortAllocator {
             const ttlMs = (_a = opts.ttlMs) !== null && _a !== void 0 ? _a : 60 * 60 * 1000;
             const now = Date.now();
             yield prisma_1.prisma.portLease.deleteMany({ where: { expiresAt: { lt: now } } });
+            const existing = yield prisma_1.prisma.portLease.findFirst({
+                where: { purpose, leasedToUdid: udid, port: { gte: start, lte: end } },
+                select: { port: true },
+            });
+            if (existing) {
+                yield prisma_1.prisma.portLease
+                    .update({
+                    where: { port: existing.port },
+                    data: { leasedAt: now, expiresAt: now + ttlMs, leasedToPid: opts.pid },
+                })
+                    .catch(() => undefined);
+                return existing.port;
+            }
             const active = yield prisma_1.prisma.portLease.findMany({
                 where: { purpose, port: { gte: start, lte: end } },
                 select: { port: true },

package/lib/src/services/ProcessMetricsService.js

@@ -0,0 +1,99 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ProcessMetricsService = void 0;
+const typedi_1 = require("typedi");
+const logger_1 = __importDefault(require("../logger"));
+// Hub-process health metrics. Individual sessions don't spawn child
+// processes (Appium is one Node process, sessions are logical constructs
+// inside it), so true per-session RSS doesn't make sense. What DOES matter
+// is whether the hub process itself is getting starved:
+//
+//   - heap growing while session count is flat = memory leak
+//   - event-loop lag climbing while CPU is unsaturated = blocking work
+//     sneaking into the hot path (sync fs, bad JSON, etc)
+//
+// Memory is cheap to read on demand in getMetrics(). Event-loop lag needs
+// a periodic sampler because by the time /metrics is scraped, the blocking
+// moment is gone — we report the max lag observed since the last scrape.
+let ProcessMetricsService = class ProcessMetricsService {
+    constructor() {
+        this.logger = logger_1.default.scope('ProcessMetrics');
+        this.sampleIntervalMs = 1000;
+        this.maxLagSinceScrapeMs = 0;
+        this.lastLagMs = 0;
+        this.timer = null;
+        this.commandsProcessed = 0;
+        this.commandDurationMsSum = 0;
+    }
+    start(intervalMs = 1000) {
+        if (this.timer)
+            return;
+        this.sampleIntervalMs = intervalMs;
+        this.scheduleNextSample();
+        this.logger.info(`Event-loop lag sampling every ${intervalMs}ms`);
+    }
+    stop() {
+        if (this.timer) {
+            clearTimeout(this.timer);
+            this.timer = null;
+        }
+    }
+    scheduleNextSample() {
+        var _a, _b;
+        const scheduledAt = Date.now();
+        this.timer = setTimeout(() => {
+            // Lag = actual delay beyond the scheduled interval. A healthy loop
+            // wakes up within a few ms of the deadline; sustained double-digit
+            // ms means something is blocking.
+            const actualDelay = Date.now() - scheduledAt;
+            const lag = Math.max(0, actualDelay - this.sampleIntervalMs);
+            this.lastLagMs = lag;
+            if (lag > this.maxLagSinceScrapeMs)
+                this.maxLagSinceScrapeMs = lag;
+            this.scheduleNextSample();
+        }, this.sampleIntervalMs);
+        // Don't keep the process alive just for this sampler.
+        (_b = (_a = this.timer).unref) === null || _b === void 0 ? void 0 : _b.call(_a);
+    }
+    recordCommand(durationMs) {
+        this.commandsProcessed++;
+        this.commandDurationMsSum += durationMs;
+    }
+    snapshot() {
+        const mem = process.memoryUsage();
+        // Reading /metrics resets the max-since-scrape counter so Prometheus
+        // rate() sees actual per-interval peaks rather than a monotonic climb.
+        const maxLag = this.maxLagSinceScrapeMs;
+        this.maxLagSinceScrapeMs = 0;
+        return {
+            memory: {
+                rss: mem.rss,
+                heapUsed: mem.heapUsed,
+                heapTotal: mem.heapTotal,
+                external: mem.external,
+                arrayBuffers: mem.arrayBuffers,
+            },
+            eventLoop: {
+                lagMs: this.lastLagMs,
+                maxLagSinceScrapeMs: maxLag,
+            },
+            commands: {
+                processed: this.commandsProcessed,
+                durationMsSum: this.commandDurationMsSum,
+            },
+        };
+    }
+};
+exports.ProcessMetricsService = ProcessMetricsService;
+exports.ProcessMetricsService = ProcessMetricsService = __decorate([
+    (0, typedi_1.Service)()
+], ProcessMetricsService);

package/lib/src/services/ProcessRegistry.js

@@ -0,0 +1,123 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ProcessRegistry = void 0;
+const typedi_1 = require("typedi");
+const crypto_1 = require("crypto");
+const logger_1 = __importDefault(require("../logger"));
+let ProcessRegistry = class ProcessRegistry {
+    constructor() {
+        this.log = logger_1.default.scope('ProcessRegistry');
+        this.processes = new Map();
+    }
+    track(opts) {
+        const id = (0, crypto_1.randomUUID)();
+        const entry = {
+            id,
+            sessionId: opts.sessionId,
+            udid: opts.udid,
+            kind: opts.kind,
+            pid: opts.process.pid || -1,
+            process: opts.process,
+            startedAt: Date.now(),
+        };
+        this.processes.set(id, entry);
+        opts.process.once('exit', () => this.processes.delete(id));
+        return id;
+    }
+    untrack(id) {
+        this.processes.delete(id);
+    }
+    snapshot() {
+        return Array.from(this.processes.values());
+    }
+    terminate(id_1) {
+        return __awaiter(this, arguments, void 0, function* (id, { gracefulMs = 5000 } = {}) {
+            const entry = this.processes.get(id);
+            if (!entry)
+                return;
+            const { process: child, pid, kind } = entry;
+            const exited = new Promise((resolve) => {
+                child.once('exit', () => resolve());
+            });
+            try {
+                if (process.platform === 'win32') {
+                    child.kill('SIGTERM');
+                }
+                else {
+                    try {
+                        process.kill(-pid, 'SIGTERM');
+                    }
+                    catch (_a) {
+                        child.kill('SIGTERM');
+                    }
+                }
+            }
+            catch (err) {
+                this.log.debug(`SIGTERM failed for ${kind}/${pid}: ${err.message}`);
+            }
+            const timedOut = yield Promise.race([
+                exited.then(() => false),
+                new Promise((r) => setTimeout(() => r(true), gracefulMs)),
+            ]);
+            if (timedOut) {
+                try {
+                    if (process.platform === 'win32') {
+                        child.kill('SIGKILL');
+                    }
+                    else {
+                        try {
+                            process.kill(-pid, 'SIGKILL');
+                        }
+                        catch (_b) {
+                            child.kill('SIGKILL');
+                        }
+                    }
+                }
+                catch (err) {
+                    this.log.warn(`SIGKILL failed for ${kind}/${pid}: ${err.message}`);
+                }
+            }
+            this.processes.delete(id);
+        });
+    }
+    terminateForSession(sessionId, opts) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const targets = this.snapshot().filter((p) => p.sessionId === sessionId);
+            yield Promise.all(targets.map((t) => this.terminate(t.id, opts)));
+        });
+    }
+    terminateForUdid(udid, opts) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const targets = this.snapshot().filter((p) => p.udid === udid);
+            yield Promise.all(targets.map((t) => this.terminate(t.id, opts)));
+        });
+    }
+    terminateAll(opts) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const targets = this.snapshot();
+            yield Promise.all(targets.map((t) => this.terminate(t.id, opts)));
+        });
+    }
+};
+exports.ProcessRegistry = ProcessRegistry;
+exports.ProcessRegistry = ProcessRegistry = __decorate([
+    (0, typedi_1.Service)()
+], ProcessRegistry);

package/lib/src/services/ServerManager.js

@@ -200,6 +200,9 @@ let ServerManager = ServerManager_1 = class ServerManager {
             const { runMigrations } = yield Promise.resolve().then(() => __importStar(require('../scripts/run-migrations')));
             yield runMigrations();
             yield device_store_1.DeviceStoreFactory.getStore().clearStorage();
+            const { ApiKeyService } = yield Promise.resolve().then(() => __importStar(require('./ApiKeyService')));
+            const { config: xenonConfig } = yield Promise.resolve().then(() => __importStar(require('../config')));
+            yield typedi_1.Container.get(ApiKeyService).bootstrapIfEmpty(xenonConfig.bootstrapKeyPath);
         });
     }
     registerRoutes(expressApp, cliArgs, pluginArgs) {
@@ -240,13 +243,13 @@ let ServerManager = ServerManager_1 = class ServerManager {
             const hubArgument = pluginArgs.hub;
             if (hubArgument !== undefined) {
                 this.logger.info(`📡 I'm a node and my hub is ${hubArgument}`);
-                yield (0, device_utils_1.setupCronUpdateDeviceList)(pluginArgs.bindHostOrIp, hubArgument, pluginArgs.sendNodeDevicesToHubIntervalMs, pluginArgs.tlsRejectUnauthorized);
+                yield (0, device_utils_1.setupCronUpdateDeviceList)(pluginArgs.bindHostOrIp, hubArgument, pluginArgs.sendNodeDevicesToHubIntervalMs, pluginArgs.tlsRejectUnauthorized, pluginArgs.nodeSecret);
                 // Handle graceful shutdown
                 ['SIGINT', 'SIGTERM'].forEach((signal) => {
                     process.once(signal, () => __awaiter(this, void 0, void 0, function* () {
                         logger_1.default.info(`Received ${signal}, unregistering node from hub...`);
                         try {
-                            yield new NodeDevices_1.default(hubArgument, pluginArgs.tlsRejectUnauthorized).unRegisterNode(pluginArgs.bindHostOrIp);
+                            yield new NodeDevices_1.default(hubArgument, pluginArgs.tlsRejectUnauthorized, pluginArgs.nodeSecret).unRegisterNode(pluginArgs.bindHostOrIp);
                         }
                         catch (err) {
                             logger_1.default.error(`Error during node unregistration: ${err}`);
@@ -295,6 +298,15 @@ let ServerManager = ServerManager_1 = class ServerManager {
                 typedi_1.Container.get(SessionHeartbeatService).start(pluginArgs);
                 // 8. Sweep orphaned sessions on a 30s cron
                 (0, device_utils_1.setupCronSweepOrphanSessions)(pluginArgs.sessionHeartbeatIntervalMs || 30000);
+                // 9. Reconcile device-store busy flags against SESSION_MANAGER every
+                //    60s to catch devices orphaned mid-allocation (driver crash before
+                //    session registration, etc.)
+                (0, device_utils_1.setupCronReconcileDevices)(60000);
+                // 10. Start event-loop lag sampler so xenon_process_event_loop_lag_ms
+                //     has real values by first scrape. Memory gauges are read on
+                //     demand; lag needs a running sampler.
+                const { ProcessMetricsService } = yield Promise.resolve().then(() => __importStar(require('./ProcessMetricsService')));
+                typedi_1.Container.get(ProcessMetricsService).start(1000);
             }
         });
     }

package/lib/src/services/SessionLifecycleService.js

@@ -78,15 +78,27 @@ const RemoteSession_1 = require("../sessions/RemoteSession");
 const SessionManager_1 = require("../sessions/SessionManager");
 const event_manager_1 = require("../dashboard/event-manager");
 const session_service_1 = require("../dashboard/services/session-service");
+const SessionStatus_1 = require("../types/SessionStatus");
 const SessionType_1 = __importDefault(require("../enums/SessionType"));
 const async_lock_1 = __importDefault(require("async-lock"));
 const commandsQueueGuard = new async_lock_1.default();
+// Serializes concurrent deleteSession cleanup for the same sessionId so
+// that onSessionStopped events don't fire twice if two clients race.
+const sessionCleanupLock = new async_lock_1.default();
 let SessionLifecycleService = class SessionLifecycleService {
     constructor() {
         this.logger = logger_1.default.scope('SessionLifecycleService');
     }
     createSession(next, driver, caps) {
         return __awaiter(this, void 0, void 0, function* () {
+            // Fail fast during graceful shutdown so clients get a clear error instead
+            // of their request hanging until the process dies. Lazy import to avoid
+            // a cycle (ShutdownCoordinator -> SessionLifecycleService -> this file).
+            const { ShutdownCoordinator } = yield Promise.resolve().then(() => __importStar(require('./ShutdownCoordinator')));
+            if (typedi_1.Container.get(ShutdownCoordinator).isDraining) {
+                this.logger.warn('Rejecting new session: hub is draining for shutdown');
+                throw new Error('Hub is shutting down; please retry against a different node');
+            }
             const context = typedi_1.Container.get(PluginContext_1.PluginContext);
             const pluginArgs = context.pluginArgs;
             this.logger.debug(`📱 pluginArgs: ${JSON.stringify(pluginArgs)}`);
@@ -457,15 +469,20 @@ let SessionLifecycleService = class SessionLifecycleService {
     }
     deleteSession(next, sessionId, status, reason) {
         return __awaiter(this, void 0, void 0, function* () {
+            // Phase 1: device unblock + finalizeCleanup must be atomic. Without the lock,
+            // two racing deletes both observe isStopping=false and both run finalizeCleanup,
+            // which releases ports and archives video twice.
             if (sessionId) {
-                yield (0, device_service_1.unblockDeviceMatchingFilter)({ session_id: sessionId });
-                this.logger.info(`📱 Unblocking the device that is blocked for session ${sessionId}`);
-            }
-            const session = sessionId ? SessionManager_1.SESSION_MANAGER.getSession(sessionId) : undefined;
-            if (session) {
-                session.isStopping = true;
-                session.stoppedAt = Date.now();
-                yield this.finalizeCleanup(session, status, reason);
+                yield sessionCleanupLock.acquire(sessionId, () => __awaiter(this, void 0, void 0, function* () {
+                    yield (0, device_service_1.unblockDeviceMatchingFilter)({ session_id: sessionId });
+                    this.logger.info(`📱 Unblocking the device that is blocked for session ${sessionId}`);
+                    const session = SessionManager_1.SESSION_MANAGER.getSession(sessionId);
+                    if (session && !session.isStopping) {
+                        session.isStopping = true;
+                        session.stoppedAt = Date.now();
+                        yield this.finalizeCleanup(session, status, reason);
+                    }
+                }));
             }
             let timeoutId;
             try {
@@ -486,8 +503,12 @@ let SessionLifecycleService = class SessionLifecycleService {
             }
             finally {
                 if (sessionId) {
-                    const session = SessionManager_1.SESSION_MANAGER.getSession(sessionId);
-                    if (session) {
+                    yield sessionCleanupLock.acquire(sessionId, () => __awaiter(this, void 0, void 0, function* () {
+                        const session = SessionManager_1.SESSION_MANAGER.getSession(sessionId);
+                        if (!session) {
+                            // Another concurrent deleteSession already cleaned up.
+                            return;
+                        }
                         const device = session.getDevice();
                         try {
                             const { NetworkConditioningService } = yield Promise.resolve().then(() => __importStar(require('./NetworkConditioningService')));
@@ -496,20 +517,20 @@ let SessionLifecycleService = class SessionLifecycleService {
                         catch (resetErr) {
                             this.logger.warn(`⚠️ NetworkConditioningService.reset failed for session ${sessionId}: ${resetErr.message}`);
                         }
-                    }
-                    yield event_manager_1.DASHBORD_EVENT_MANAGER.onSessionStopped(sessionId, status, reason);
-                    SessionManager_1.SESSION_MANAGER.removeSession(sessionId);
-                    try {
-                        const { getSessionById } = yield Promise.resolve().then(() => __importStar(require('../dashboard/services/session-service')));
-                        const sessionData = yield getSessionById(sessionId);
-                        if (sessionData && (sessionData.status === 'failed' || sessionData.failure_reason)) {
-                            const { NotificationService } = yield Promise.resolve().then(() => __importStar(require('./NotificationService')));
-                            yield typedi_1.Container.get(NotificationService).dispatchEvent('session_failed', sessionData);
+                        yield event_manager_1.DASHBORD_EVENT_MANAGER.onSessionStopped(sessionId, status, reason);
+                        SessionManager_1.SESSION_MANAGER.removeSession(sessionId);
+                        try {
+                            const { getSessionById } = yield Promise.resolve().then(() => __importStar(require('../dashboard/services/session-service')));
+                            const sessionData = yield getSessionById(sessionId);
+                            if (sessionData && (sessionData.status === 'failed' || sessionData.failure_reason)) {
+                                const { NotificationService } = yield Promise.resolve().then(() => __importStar(require('./NotificationService')));
+                                yield typedi_1.Container.get(NotificationService).dispatchEvent('session_failed', sessionData);
+                            }
                         }
-                    }
-                    catch (err) {
-                        /* ignore notification errors */
-                    }
+                        catch (err) {
+                            /* ignore notification errors */
+                        }
+                    }));
                 }
             }
         });
@@ -572,6 +593,42 @@ let SessionLifecycleService = class SessionLifecycleService {
             }
         });
     }
+    // Shutdown-path counterpart to deleteSession. Runs the same cleanup (unblock
+    // device, archive video, release ports, mark failed, emit stopped) WITHOUT
+    // needing Appium's `next()` driver shutdown — during process shutdown we
+    // don't have it, and any leftover driver state dies with the process
+    // anyway. Shares sessionCleanupLock with deleteSession so a racing client
+    // delete can't double-archive video.
+    stopSessionForShutdown(sessionId, reason) {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield sessionCleanupLock.acquire(sessionId, () => __awaiter(this, void 0, void 0, function* () {
+                try {
+                    yield (0, device_service_1.unblockDeviceMatchingFilter)({ session_id: sessionId });
+                }
+                catch (err) {
+                    this.logger.warn(`[shutdown] unblock failed for ${sessionId}: ${err.message}`);
+                }
+                const session = SessionManager_1.SESSION_MANAGER.getSession(sessionId);
+                if (session && !session.isStopping) {
+                    session.isStopping = true;
+                    session.stoppedAt = Date.now();
+                    try {
+                        yield this.finalizeCleanup(session, SessionStatus_1.SessionStatus.FAILED, reason);
+                    }
+                    catch (err) {
+                        this.logger.warn(`[shutdown] finalizeCleanup failed for ${sessionId}: ${err.message}`);
+                    }
+                }
+                try {
+                    yield event_manager_1.DASHBORD_EVENT_MANAGER.onSessionStopped(sessionId, SessionStatus_1.SessionStatus.FAILED, reason);
+                }
+                catch (err) {
+                    this.logger.warn(`[shutdown] onSessionStopped failed for ${sessionId}: ${err.message}`);
+                }
+                SessionManager_1.SESSION_MANAGER.removeSession(sessionId);
+            }));
+        });
+    }
     isHub(args) {
         return !args.hub;
     }