@xenon-device-management/xenon 2.14.0 → 2.16.0

package/lib/test/unit/tracing-export-routing.spec.js

@@ -0,0 +1,137 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+require("reflect-metadata");
+const chai_1 = require("chai");
+const sinon_1 = __importDefault(require("sinon"));
+const api_1 = require("@opentelemetry/api");
+const api_logs_1 = require("@opentelemetry/api-logs");
+const TracingService_1 = require("../../src/services/TracingService");
+const logger_1 = require("../../src/logger");
+const otlp_probe_1 = require("../helpers/otlp-probe");
+const ENV_KEYS = [
+    'OTEL_SDK_DISABLED',
+    'OTEL_LOGS_ENABLED',
+    'OTEL_TRACES_ENABLED',
+    'OTEL_METRICS_ENABLED',
+    'OTEL_EXPORTER_OTLP_ENDPOINT',
+    'OTEL_EXPORTER_OTLP_LOGS_ENDPOINT',
+    'OTEL_EXPORTER_OTLP_METRICS_ENDPOINT',
+    'XENON_OTEL_DEBUG',
+];
+/**
+ * With tracing on, Xenon builds an OTel NodeSDK for its spans. A NodeSDK
+ * that isn't handed log processors or metric readers builds its own from the
+ * standard OTEL_* variables and registers them globally before Xenon could.
+ * Xenon's own logger and meter providers were then refused, so:
+ * - logs went out as protobuf through the SDK's exporter, not Xenon's;
+ * - metrics went to the trace URL + /v1/metrics, even with
+ *   OTEL_METRICS_ENABLED=false;
+ * - only traces on also exported logs, to the trace URL + /v1/logs;
+ * - XENON_OTEL_DEBUG's console log and metric exporters never printed.
+ *
+ * A probe collector shows where each signal went.
+ */
+describe('TracingService — where logs and metrics go with tracing on', function () {
+    this.timeout(20000);
+    let probe;
+    let service;
+    let saved;
+    beforeEach(() => __awaiter(this, void 0, void 0, function* () {
+        saved = Object.fromEntries(ENV_KEYS.map((k) => [k, process.env[k]]));
+        for (const k of ENV_KEYS)
+            delete process.env[k];
+        (0, otlp_probe_1.resetOtelGlobals)();
+        logger_1.XenonLogger.setOtlpReady(false);
+        probe = yield (0, otlp_probe_1.startOtlpProbe)();
+        process.env.OTEL_EXPORTER_OTLP_ENDPOINT = probe.url('/v1/traces');
+    }));
+    afterEach(() => __awaiter(this, void 0, void 0, function* () {
+        yield (service === null || service === void 0 ? void 0 : service.shutdown());
+        service = undefined;
+        sinon_1.default.restore();
+        (0, otlp_probe_1.resetOtelGlobals)();
+        logger_1.XenonLogger.setOtlpReady(false);
+        yield probe.close();
+        for (const k of ENV_KEYS) {
+            if (saved[k] === undefined)
+                delete process.env[k];
+            else
+                process.env[k] = saved[k];
+        }
+    }));
+    function start() {
+        service = new TracingService_1.TracingService();
+        service.initialize({ isHub: true });
+        return service;
+    }
+    function countProbeMetric() {
+        api_1.metrics.getMeter('probe').createCounter('xenon.probe.count').add(1);
+    }
+    function emitProbeLog() {
+        api_logs_1.logs.getLogger('probe').emit({ body: 'probe line', severityNumber: api_logs_1.SeverityNumber.INFO });
+    }
+    it('exports no metrics with OTEL_METRICS_ENABLED=false', () => __awaiter(this, void 0, void 0, function* () {
+        process.env.OTEL_METRICS_ENABLED = 'false';
+        start();
+        countProbeMetric();
+        yield (0, otlp_probe_1.flushOtelGlobals)();
+        (0, chai_1.expect)(probe.requests.map((r) => r.path).filter((p) => p.includes('metrics'))).to.deep.equal([]);
+    }));
+    it('exports no metrics when no metrics endpoint is set, and never to the trace URL', () => __awaiter(this, void 0, void 0, function* () {
+        start();
+        countProbeMetric();
+        yield (0, otlp_probe_1.flushOtelGlobals)();
+        (0, chai_1.expect)(probe.requests.map((r) => r.path)).to.not.include('/v1/traces/v1/metrics');
+    }));
+    it("sends metrics to Xenon's metrics endpoint as OTLP/JSON", () => __awaiter(this, void 0, void 0, function* () {
+        process.env.OTEL_EXPORTER_OTLP_METRICS_ENDPOINT = probe.url('/xenon-metrics');
+        start();
+        countProbeMetric();
+        yield (0, otlp_probe_1.flushOtelGlobals)();
+        const sent = probe.requests.filter((r) => r.path === '/xenon-metrics');
+        (0, chai_1.expect)(sent, 'metrics export').to.have.length.greaterThan(0);
+        (0, chai_1.expect)(sent.every((r) => r.contentType.includes('application/json'))).to.equal(true);
+        (0, chai_1.expect)(sent.map((r) => r.body.toString('utf8')).join()).to.include('xenon.probe.count');
+    }));
+    it('exports no logs when only traces are configured', () => __awaiter(this, void 0, void 0, function* () {
+        start();
+        emitProbeLog();
+        yield (0, otlp_probe_1.flushOtelGlobals)();
+        (0, chai_1.expect)(probe.requests.map((r) => r.path).filter((p) => p.includes('logs'))).to.deep.equal([]);
+    }));
+    it("sends logs to Xenon's log endpoint as OTLP/JSON", () => __awaiter(this, void 0, void 0, function* () {
+        process.env.OTEL_EXPORTER_OTLP_LOGS_ENDPOINT = probe.url('/xenon-logs');
+        start();
+        emitProbeLog();
+        yield (0, otlp_probe_1.flushOtelGlobals)();
+        const sent = probe.requests.filter((r) => r.path === '/xenon-logs');
+        (0, chai_1.expect)(sent, 'logs export').to.have.length.greaterThan(0);
+        (0, chai_1.expect)(sent.every((r) => r.contentType.includes('application/json'))).to.equal(true);
+        (0, chai_1.expect)(sent.map((r) => r.body.toString('utf8')).join()).to.include('probe line');
+    }));
+    it('prints logs and metrics to the console with XENON_OTEL_DEBUG=true', () => __awaiter(this, void 0, void 0, function* () {
+        const printed = sinon_1.default.stub(console, 'dir');
+        process.env.XENON_OTEL_DEBUG = 'true';
+        process.env.OTEL_EXPORTER_OTLP_LOGS_ENDPOINT = probe.url('/xenon-logs');
+        start();
+        emitProbeLog();
+        countProbeMetric();
+        yield (0, otlp_probe_1.flushOtelGlobals)();
+        printed.restore();
+        const shown = printed.getCalls().map((c) => JSON.stringify(c.args[0]));
+        (0, chai_1.expect)(shown.some((s) => s.includes('probe line')), 'console log record').to.equal(true);
+        (0, chai_1.expect)(shown.some((s) => s.includes('xenon.probe.count')), 'console metric').to.equal(true);
+    }));
+});

package/lib/test/unit/tracing-session-spans.spec.js

@@ -0,0 +1,337 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+require("reflect-metadata");
+const chai_1 = require("chai");
+const http_1 = __importDefault(require("http"));
+const sinon_1 = __importDefault(require("sinon"));
+const typedi_1 = require("typedi");
+const TracingService_1 = require("../../src/services/TracingService");
+const CommandInterceptor_1 = require("../../src/interceptors/CommandInterceptor");
+const plugin_1 = require("../../src/plugin");
+const DeviceService = __importStar(require("../../src/data-service/device-service"));
+const ActiveLeases = __importStar(require("../../src/services/lease/activeLeases"));
+const device_utils_1 = require("../../src/device-utils");
+const SessionLifecycleService_1 = require("../../src/services/SessionLifecycleService");
+const SessionMetricsService_1 = require("../../src/services/metrics/SessionMetricsService");
+const OrphanSweeper_1 = require("../../src/services/OrphanSweeper");
+const PhoneNetworkRestore_1 = require("../../src/services/network/PhoneNetworkRestore");
+const event_manager_1 = require("../../src/dashboard/event-manager");
+const SessionManager_1 = require("../../src/sessions/SessionManager");
+const SessionStatus_1 = require("../../src/types/SessionStatus");
+const logger_1 = require("../../src/logger");
+const PluginContext_1 = require("../../src/PluginContext");
+const IPluginArgs_1 = require("../../src/interfaces/IPluginArgs");
+const container_registration_1 = require("../helpers/container-registration");
+const scratch_database_1 = require("../helpers/scratch-database");
+const otlp_probe_1 = require("../helpers/otlp-probe");
+const ENV_KEYS = [
+    'OTEL_SDK_DISABLED',
+    'OTEL_LOGS_ENABLED',
+    'OTEL_TRACES_ENABLED',
+    'OTEL_METRICS_ENABLED',
+    'OTEL_EXPORTER_OTLP_ENDPOINT',
+    'OTEL_EXPORTER_OTLP_LOGS_ENDPOINT',
+    'OTEL_EXPORTER_OTLP_METRICS_ENDPOINT',
+    'XENON_OTEL_DEBUG',
+];
+const ERROR = 2;
+const OK = 1;
+/**
+ * A hub starts a span for every session it creates, a node's and a cloud
+ * provider's included, and its commands' spans are its children. The only
+ * place that ended it was Appium's new-command timeout with the dashboard on,
+ * so every other session's span stayed open in memory for the life of the
+ * process and was never exported, and its commands showed no parent. A
+ * failed command's span also ended OK.
+ */
+describe('TracingService — session and command spans', function () {
+    this.timeout(30000);
+    const scratch = (0, scratch_database_1.useScratchDatabase)();
+    let probe;
+    let tracing;
+    let restoreRegs;
+    let saved;
+    beforeEach(() => __awaiter(this, void 0, void 0, function* () {
+        saved = Object.fromEntries(ENV_KEYS.map((k) => [k, process.env[k]]));
+        for (const k of ENV_KEYS)
+            delete process.env[k];
+        (0, otlp_probe_1.resetOtelGlobals)();
+        logger_1.XenonLogger.setOtlpReady(false);
+        probe = yield (0, otlp_probe_1.startOtlpProbe)();
+        process.env.OTEL_EXPORTER_OTLP_ENDPOINT = probe.url('/v1/traces');
+        restoreRegs = (0, container_registration_1.saveRegistrations)(TracingService_1.TracingService, PhoneNetworkRestore_1.PhoneNetworkRestore, SessionMetricsService_1.SessionMetricsService);
+        tracing = new TracingService_1.TracingService();
+        tracing.initialize({ isHub: true });
+        typedi_1.Container.set(TracingService_1.TracingService, tracing);
+        typedi_1.Container.set(PhoneNetworkRestore_1.PhoneNetworkRestore, {
+            restoreSession: sinon_1.default.stub().resolves(),
+            restoreAll: sinon_1.default.stub().resolves(),
+        });
+        typedi_1.Container.set(SessionMetricsService_1.SessionMetricsService, { stop: sinon_1.default.stub().resolves() });
+        sinon_1.default.stub(DeviceService, 'releaseSessionDevices').resolves();
+        sinon_1.default.stub(event_manager_1.DASHBORD_EVENT_MANAGER, 'onSessionStopped').resolves();
+    }));
+    afterEach(() => __awaiter(this, void 0, void 0, function* () {
+        yield tracing.shutdown();
+        sinon_1.default.restore();
+        restoreRegs();
+        (0, otlp_probe_1.resetOtelGlobals)();
+        yield probe.close();
+        for (const k of ENV_KEYS) {
+            if (saved[k] === undefined)
+                delete process.env[k];
+            else
+                process.env[k] = saved[k];
+        }
+    }));
+    /** The spans exported once the tracer has sent everything it holds. */
+    function exported() {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield tracing.shutdown();
+            return probe.spans();
+        });
+    }
+    function sessionSpans(sessionId) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return (yield exported()).filter((s) => s.name === `Session: ${sessionId}`);
+        });
+    }
+    describe('a session span ends once, however the session ends', () => {
+        it('when the test deletes a session Xenon keeps no record of (dashboard off)', () => __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            tracing.startSessionSpan('s-delete', 's-delete');
+            (0, chai_1.expect)(SessionManager_1.SESSION_MANAGER.getSession('s-delete')).to.equal(undefined);
+            yield typedi_1.Container.get(SessionLifecycleService_1.SessionLifecycleService).deleteSession(sinon_1.default.stub().resolves(), 's-delete');
+            const spans = yield sessionSpans('s-delete');
+            (0, chai_1.expect)(spans).to.have.length(1);
+            (0, chai_1.expect)((_a = spans[0].status) === null || _a === void 0 ? void 0 : _a.code).to.equal(OK);
+        }));
+        it('as failed when a session ends failed (a node no longer has it)', () => __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            tracing.startSessionSpan('s-gone', 's-gone');
+            yield typedi_1.Container.get(SessionLifecycleService_1.SessionLifecycleService).deleteSession(() => __awaiter(this, void 0, void 0, function* () { return undefined; }), 's-gone', SessionStatus_1.SessionStatus.FAILED, 'Session terminal failure (gone)');
+            const [span] = yield sessionSpans('s-gone');
+            (0, chai_1.expect)((_a = span.status) === null || _a === void 0 ? void 0 : _a.code).to.equal(ERROR);
+            (0, chai_1.expect)((0, otlp_probe_1.attributeOf)(span.attributes, 'xenon.session.stop_reason')).to.equal('Session terminal failure (gone)');
+        }));
+        it('when Appium ends the session itself, with the dashboard off', () => __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            tracing.startSessionSpan('s-timeout', 's-timeout');
+            const plugin = { pluginArgs: {}, xenonLog: { withSession: () => ({ info() { } }) } };
+            yield plugin_1.XenonPlugin.prototype.onUnexpectedShutdown.call(plugin, { sessionId: 's-timeout', caps: { udid: 'R5CT' } }, new Error('New command timeout'));
+            const spans = yield sessionSpans('s-timeout');
+            (0, chai_1.expect)(spans).to.have.length(1);
+            (0, chai_1.expect)((_a = spans[0].status) === null || _a === void 0 ? void 0 : _a.code).to.equal(ERROR);
+        }));
+        it("when Xenon's idle check releases the phone", () => __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            tracing.startSessionSpan('s-idle', 's-idle');
+            sinon_1.default.stub(DeviceService, 'getAllDevices').resolves([
+                {
+                    udid: 'R5CT',
+                    host: 'http://127.0.0.1:4723',
+                    busy: true,
+                    userBlocked: false,
+                    session_id: 's-idle',
+                    claimSessionId: 's-idle',
+                    lastCmdExecutedAt: Date.now() - 10 * 60 * 1000,
+                },
+            ]);
+            sinon_1.default.stub(ActiveLeases, 'activeLeasesByDevice').resolves(new Map());
+            sinon_1.default.stub(DeviceService, 'releaseSessionDevice').resolves(true);
+            yield (0, device_utils_1.releaseBlockedDevices)(60);
+            const [span] = yield sessionSpans('s-idle');
+            (0, chai_1.expect)(span, 'idle session span').to.not.equal(undefined);
+            (0, chai_1.expect)((_a = span.status) === null || _a === void 0 ? void 0 : _a.code).to.equal(ERROR);
+        }));
+        it('when a graceful shutdown drains the session', () => __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            tracing.startSessionSpan('s-down', 's-down');
+            yield typedi_1.Container.get(SessionLifecycleService_1.SessionLifecycleService).stopSessionForShutdown('s-down', 'shutdown');
+            const [span] = yield sessionSpans('s-down');
+            (0, chai_1.expect)(span, 'drained session span').to.not.equal(undefined);
+            (0, chai_1.expect)((_a = span.status) === null || _a === void 0 ? void 0 : _a.code).to.equal(ERROR);
+        }));
+        it("when a session's heartbeat goes stale", () => __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            tracing.startSessionSpan('s-stale', 's-stale');
+            yield scratch.db.session.deleteMany({});
+            yield scratch.db.session.create({
+                data: {
+                    id: 's-stale',
+                    status: 'running',
+                    desired_capabilities: '{}',
+                    session_capabilities: '{}',
+                    node_id: 'n',
+                    has_live_video: false,
+                    device_udid: 'R5CT',
+                    device_platform: 'android',
+                    device_version: '14',
+                    last_heartbeat_at: new Date(Date.now() - 60 * 60 * 1000),
+                },
+            });
+            yield new OrphanSweeper_1.OrphanSweeper().sweep({ heartbeatIntervalMs: 1000 });
+            const [span] = yield sessionSpans('s-stale');
+            (0, chai_1.expect)(span, 'stale session span').to.not.equal(undefined);
+            (0, chai_1.expect)((_a = span.status) === null || _a === void 0 ? void 0 : _a.code).to.equal(ERROR);
+        }));
+        it('when the server shuts down with the session still running elsewhere', () => __awaiter(this, void 0, void 0, function* () {
+            tracing.startSessionSpan('s-on-node', 's-on-node');
+            const [span] = yield sessionSpans('s-on-node');
+            (0, chai_1.expect)(span, 'open session span at shutdown').to.not.equal(undefined);
+            (0, chai_1.expect)((0, otlp_probe_1.attributeOf)(span.attributes, 'xenon.session.stop_reason')).to.equal('Xenon shut down while the session was running');
+        }));
+        it('within its budget when the collector never answers', () => __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            const silent = http_1.default.createServer(() => undefined);
+            yield new Promise((resolve) => silent.listen(0, '127.0.0.1', () => resolve()));
+            try {
+                yield tracing.shutdown();
+                (0, otlp_probe_1.resetOtelGlobals)();
+                process.env.OTEL_EXPORTER_OTLP_ENDPOINT = `http://127.0.0.1:${silent.address().port}/v1/traces`;
+                tracing = new TracingService_1.TracingService();
+                tracing.initialize({ isHub: true });
+                tracing.startSessionSpan('s-hung', 's-hung');
+                const started = Date.now();
+                yield tracing.shutdownWithin(300);
+                (0, chai_1.expect)(Date.now() - started).to.be.lessThan(2000);
+            }
+            finally {
+                (_a = silent.closeAllConnections) === null || _a === void 0 ? void 0 : _a.call(silent);
+                silent.close();
+            }
+        }));
+        it('when Xenon fails to finish creating it', () => __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            const context = typedi_1.Container.get(PluginContext_1.PluginContext);
+            const savedContext = Object.assign({}, context);
+            context.setContext(Object.assign({}, IPluginArgs_1.DefaultPluginArgs), 4723, 'hub-1', '');
+            sinon_1.default.stub(DeviceService, 'claimDeviceForSession').rejects(new Error('database is locked'));
+            try {
+                const finishing = typedi_1.Container.get(SessionLifecycleService_1.SessionLifecycleService).finalizeSession({ value: ['s-half', {}] }, { udid: 'R5CT', platform: 'android', host: 'http://127.0.0.1:4723', nodeId: 'hub-1' }, { alwaysMatch: {}, firstMatch: [{}] }, {}, false);
+                yield finishing.then(() => chai_1.expect.fail('the create was expected to fail'), () => undefined);
+            }
+            finally {
+                Object.assign(context, savedContext);
+            }
+            const [span] = yield sessionSpans('s-half');
+            (0, chai_1.expect)(span, 'half-created session span').to.not.equal(undefined);
+            (0, chai_1.expect)((_a = span.status) === null || _a === void 0 ? void 0 : _a.code).to.equal(ERROR);
+            (0, chai_1.expect)((0, otlp_probe_1.attributeOf)(span.attributes, 'xenon.session.stop_reason')).to.include('database is locked');
+        }));
+        it('only once when two endings race', () => __awaiter(this, void 0, void 0, function* () {
+            tracing.startSessionSpan('s-twice', 's-twice');
+            const lifecycle = typedi_1.Container.get(SessionLifecycleService_1.SessionLifecycleService);
+            yield Promise.all([
+                lifecycle.deleteSession(() => __awaiter(this, void 0, void 0, function* () { return undefined; }), 's-twice'),
+                lifecycle.stopSessionForShutdown('s-twice', 'shutdown'),
+            ]);
+            (0, chai_1.expect)(yield sessionSpans('s-twice')).to.have.length(1);
+        }));
+    });
+    describe("a command's span", () => {
+        beforeEach(() => {
+            sinon_1.default.stub(DeviceService, 'updateCmdExecutedTime').resolves();
+            sinon_1.default.stub(event_manager_1.DASHBORD_EVENT_MANAGER, 'beforeSessionCommand').resolves(true);
+        });
+        function run(next) {
+            return typedi_1.Container.get(CommandInterceptor_1.CommandInterceptor).handle(next, { sessionId: 's-cmd', caps: {} }, 'getPageSource', [], {}, true);
+        }
+        it('is a child of its session span', () => __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            tracing.startSessionSpan('s-cmd', 's-cmd');
+            yield run(() => __awaiter(this, void 0, void 0, function* () { return '<xml/>'; }));
+            tracing.endSessionSpan('s-cmd');
+            const spans = yield exported();
+            const session = spans.find((s) => s.name === 'Session: s-cmd');
+            const command = spans.find((s) => s.name === 'getPageSource');
+            (0, chai_1.expect)(session, 'session span').to.not.equal(undefined);
+            (0, chai_1.expect)(command, 'command span').to.not.equal(undefined);
+            (0, chai_1.expect)(command === null || command === void 0 ? void 0 : command.parentSpanId).to.equal(session === null || session === void 0 ? void 0 : session.spanId);
+            (0, chai_1.expect)((_a = command === null || command === void 0 ? void 0 : command.status) === null || _a === void 0 ? void 0 : _a.code).to.equal(OK);
+        }));
+        it('records the error and ends ERROR when the command fails', () => __awaiter(this, void 0, void 0, function* () {
+            var _a, _b, _c;
+            tracing.startSessionSpan('s-cmd', 's-cmd');
+            let thrown;
+            try {
+                yield run(() => __awaiter(this, void 0, void 0, function* () {
+                    throw new Error('An unknown server-side error occurred: boom');
+                }));
+            }
+            catch (err) {
+                thrown = err;
+            }
+            (0, chai_1.expect)(thrown).to.be.instanceOf(Error);
+            const command = (yield exported()).find((s) => s.name === 'getPageSource');
+            (0, chai_1.expect)((_a = command === null || command === void 0 ? void 0 : command.status) === null || _a === void 0 ? void 0 : _a.code).to.equal(ERROR);
+            (0, chai_1.expect)((_b = command === null || command === void 0 ? void 0 : command.status) === null || _b === void 0 ? void 0 : _b.message).to.equal('An unknown server-side error occurred: boom');
+            const exception = (_c = command === null || command === void 0 ? void 0 : command.events) === null || _c === void 0 ? void 0 : _c.find((e) => e.name === 'exception');
+            (0, chai_1.expect)((0, otlp_probe_1.attributeOf)(exception === null || exception === void 0 ? void 0 : exception.attributes, 'exception.message')).to.equal('An unknown server-side error occurred: boom');
+        }));
+        it("keeps each of two concurrent same-named commands' outcome on its own span", () => __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            // The Inspector reads the page source while the test does: both spans
+            // are found by `<session>:getPageSource`, the second over the first.
+            tracing.startSessionSpan('s-cmd', 's-cmd');
+            let finishB;
+            const b = run(() => new Promise((resolve) => (finishB = resolve)));
+            const a = run(() => __awaiter(this, void 0, void 0, function* () {
+                throw new Error('A failed');
+            }));
+            yield a.catch(() => undefined);
+            finishB('<xml/>');
+            yield b;
+            const commands = (yield exported()).filter((s) => s.name === 'getPageSource');
+            (0, chai_1.expect)(commands, 'both command spans').to.have.length(2);
+            (0, chai_1.expect)(commands.map((c) => { var _a; return (_a = c.status) === null || _a === void 0 ? void 0 : _a.code; }).sort()).to.deep.equal([OK, ERROR]);
+            const failed = commands.find((c) => { var _a; return ((_a = c.status) === null || _a === void 0 ? void 0 : _a.code) === ERROR; });
+            (0, chai_1.expect)((_a = failed === null || failed === void 0 ? void 0 : failed.status) === null || _a === void 0 ? void 0 : _a.message).to.equal('A failed');
+        }));
+    });
+});

package/lib/test/unit/unexpected-shutdown-reason.spec.js

@@ -0,0 +1,163 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+require("reflect-metadata");
+const chai_1 = require("chai");
+const sinon_1 = __importDefault(require("sinon"));
+const typedi_1 = require("typedi");
+const plugin_1 = require("../../src/plugin");
+const DeviceService = __importStar(require("../../src/data-service/device-service"));
+const device_store_1 = require("../../src/data-service/device-store");
+const SessionManager_1 = require("../../src/sessions/SessionManager");
+const SessionMetricsService_1 = require("../../src/services/metrics/SessionMetricsService");
+const PhoneNetworkRestore_1 = require("../../src/services/network/PhoneNetworkRestore");
+const SocketServer_1 = require("../../src/services/SocketServer");
+const MetricsService_1 = require("../../src/services/MetricsService");
+const NotificationService_1 = require("../../src/services/NotificationService");
+const AIService_1 = require("../../src/services/AIService");
+const shutdownReason_1 = require("../../src/services/session/shutdownReason");
+const prisma_1 = require("../../src/prisma");
+const container_registration_1 = require("../helpers/container-registration");
+const scratch_database_1 = require("../helpers/scratch-database");
+/**
+ * A session Appium ends by itself reaches `XenonPlugin.onUnexpectedShutdown`
+ * with a cause. For an idle session that is Appium's new-command timeout.
+ * The plugin threw the cause away and recorded "Driver shut down
+ * unexpectedly", which no failure pattern matches, so an idle session on this
+ * server's own phones was filed UNKNOWN and its "Why it failed" card opened
+ * the general runbook rather than the Timeout one.
+ */
+// What @appium/base-driver 10.2 passes when a session sits idle too long.
+const IDLE = "New Command Timeout of 60 seconds expired. Try customizing the timeout using the 'newCommandTimeout' desired capability";
+describe('the reason of a session Appium ends by itself', () => {
+    it("is Appium's cause: its new-command timeout", () => {
+        (0, chai_1.expect)((0, shutdownReason_1.unexpectedShutdownReason)(new Error(IDLE))).to.equal(IDLE);
+    });
+    it("is a driver's own account of giving up, which still reads as one", () => {
+        (0, chai_1.expect)((0, shutdownReason_1.unexpectedShutdownReason)(new Error('The driver was unexpectedly shut down!'))).to.equal('The driver was unexpectedly shut down!');
+    });
+    it('is the old fixed reason when there is no cause to tell', () => {
+        for (const cause of [undefined, null, new Error(''), new Error('   '), {}, 42]) {
+            (0, chai_1.expect)((0, shutdownReason_1.unexpectedShutdownReason)(cause), String(cause)).to.equal(shutdownReason_1.UNEXPECTED_SHUTDOWN_REASON);
+        }
+        // The umbrella's placeholder when a driver gives none.
+        (0, chai_1.expect)((0, shutdownReason_1.unexpectedShutdownReason)(new Error('Unknown error'))).to.equal(shutdownReason_1.UNEXPECTED_SHUTDOWN_REASON);
+    });
+    it('is the message alone, never a stack', () => {
+        const err = new Error(IDLE);
+        (0, chai_1.expect)((0, shutdownReason_1.unexpectedShutdownReason)(err)).not.to.contain('    at ');
+    });
+});
+describe('a session Appium ends for being idle, on a hub that records sessions', function () {
+    this.timeout(30000);
+    const scratch = (0, scratch_database_1.useScratchDatabase)();
+    let restoreRegs;
+    let wasHub;
+    const plugin = {
+        pluginArgs: { enableDashboard: true },
+        xenonLog: { withSession: () => ({ info() { } }) },
+    };
+    const seed = (id) => scratch.db.session.create({
+        data: {
+            id,
+            status: 'running',
+            desired_capabilities: '{}',
+            session_capabilities: '{}',
+            node_id: 'n-1',
+            has_live_video: false,
+            device_udid: 'R5CT',
+            device_platform: 'android',
+            device_version: '14',
+        },
+    });
+    const shutDown = (id, cause) => plugin_1.XenonPlugin.prototype.onUnexpectedShutdown.call(plugin, { sessionId: id, caps: { udid: 'R5CT' } }, cause);
+    beforeEach(() => __awaiter(this, void 0, void 0, function* () {
+        restoreRegs = (0, container_registration_1.saveRegistrations)(PhoneNetworkRestore_1.PhoneNetworkRestore, SessionMetricsService_1.SessionMetricsService, SocketServer_1.SocketServer, MetricsService_1.MetricsService, NotificationService_1.NotificationService);
+        typedi_1.Container.set(PhoneNetworkRestore_1.PhoneNetworkRestore, { restoreSession: sinon_1.default.stub().resolves() });
+        typedi_1.Container.set(SessionMetricsService_1.SessionMetricsService, { stop: sinon_1.default.stub().resolves() });
+        typedi_1.Container.set(SocketServer_1.SocketServer, {
+            emitToDashboard: sinon_1.default.stub(),
+            emitToDashboardForDevices: sinon_1.default.stub().resolves(),
+            hasScopedDashboard: sinon_1.default.stub().returns(false),
+        });
+        typedi_1.Container.set(MetricsService_1.MetricsService, {
+            incrementSessionSuccess: sinon_1.default.stub(),
+            incrementSessionFailure: sinon_1.default.stub(),
+        });
+        typedi_1.Container.set(NotificationService_1.NotificationService, { notifySessionFailed: sinon_1.default.stub().resolves() });
+        sinon_1.default.stub(DeviceService, 'releaseSessionDevices').resolves();
+        sinon_1.default
+            .stub(device_store_1.DeviceStoreFactory, 'getStore')
+            .returns({ getDevices: () => __awaiter(this, void 0, void 0, function* () { return []; }), findDevices: () => __awaiter(this, void 0, void 0, function* () { return []; }) });
+        sinon_1.default.stub(SessionManager_1.SESSION_MANAGER, 'getSession').returns(undefined);
+        sinon_1.default.stub(AIService_1.AI_SERVICE, 'isEnabled').returns(false);
+        wasHub = plugin_1.XenonPlugin.IS_HUB;
+        plugin_1.XenonPlugin.IS_HUB = true;
+        yield scratch.db.session.deleteMany({});
+    }));
+    afterEach(() => {
+        plugin_1.XenonPlugin.IS_HUB = wasHub;
+        sinon_1.default.restore();
+        restoreRegs();
+    });
+    it("is failed with Appium's reason and filed TIMEOUT", () => __awaiter(this, void 0, void 0, function* () {
+        yield seed('s-idle');
+        yield shutDown('s-idle', new Error(IDLE));
+        const row = yield prisma_1.prisma.session.findUnique({ where: { id: 's-idle' } });
+        (0, chai_1.expect)(row === null || row === void 0 ? void 0 : row.status).to.equal('failed');
+        (0, chai_1.expect)(row === null || row === void 0 ? void 0 : row.failure_reason).to.equal(IDLE);
+        (0, chai_1.expect)(row === null || row === void 0 ? void 0 : row.failure_category).to.equal('TIMEOUT');
+    }));
+    it('keeps the old reason, filed as a lost session, when Appium gives no cause', () => __awaiter(this, void 0, void 0, function* () {
+        yield seed('s-crash');
+        yield shutDown('s-crash', undefined);
+        const row = yield prisma_1.prisma.session.findUnique({ where: { id: 's-crash' } });
+        (0, chai_1.expect)(row === null || row === void 0 ? void 0 : row.status).to.equal('failed');
+        (0, chai_1.expect)(row === null || row === void 0 ? void 0 : row.failure_reason).to.equal(shutdownReason_1.UNEXPECTED_SHUTDOWN_REASON);
+        (0, chai_1.expect)(row === null || row === void 0 ? void 0 : row.failure_category).to.equal('SESSION_LOST');
+    }));
+});