@xenon-device-management/xenon 2.0.0 → 2.1.0

package/lib/test/unit/hub-node-race.spec.js

@@ -0,0 +1,401 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+require("reflect-metadata");
+const chai_1 = require("chai");
+const sinon_1 = __importDefault(require("sinon"));
+const express_1 = __importDefault(require("express"));
+const loopbackRequest_1 = __importDefault(require("../helpers/loopbackRequest"));
+const typedi_1 = require("typedi");
+const prisma_1 = require("../../src/prisma");
+const grid_1 = __importDefault(require("../../src/app/routers/grid"));
+const prisma_store_1 = require("../../src/data-service/prisma-store");
+const device_store_1 = require("../../src/data-service/device-store");
+const device_utils_1 = require("../../src/device-utils");
+const localDeviceHosts_1 = require("../../src/device-managers/localDeviceHosts");
+const SessionLifecycleService_1 = require("../../src/services/SessionLifecycleService");
+const device_managers_1 = require("../../src/device-managers");
+const PluginContext_1 = require("../../src/PluginContext");
+const IPluginArgs_1 = require("../../src/interfaces/IPluginArgs");
+const SocketServer_1 = require("../../src/services/SocketServer");
+const NotificationService_1 = require("../../src/services/NotificationService");
+const event_manager_1 = require("../../src/dashboard/event-manager");
+const helpers = __importStar(require("../../src/helpers"));
+const container_registration_1 = require("../helpers/container-registration");
+const scratch_database_1 = require("../helpers/scratch-database");
+/**
+ * The hub/node busy race, on a real SQLite database with the migrations
+ * applied. A hub keeps its nodes' phones in its own table, from each node's
+ * report (`POST /xenon/api/register`, every 30 s by default). The report used
+ * to be written whole: a node that had not yet been handed the hub's session
+ * said `busy: false` and freed the phone for a second client, and its copy of
+ * the team, tags, reservation and block replaced the hub's.
+ */
+const IP = '192.168.0.104';
+const HUB = `http://${IP}:4724`;
+const NODE = `http://${IP}:4725`;
+const HUB_ID = 'hub-process';
+const NODE_ID = 'node-process';
+const T0 = Date.UTC(2026, 8, 29, 12, 0, 0);
+const nodePhone = (udid, over = {}) => (Object.assign({ udid, host: NODE, nodeId: NODE_ID, name: 'Galaxy S9+', platform: 'android', deviceType: 'real', realDevice: true, state: 'device', sdk: '10', offline: false, userBlocked: false }, over));
+const caps = (udid) => ({
+    alwaysMatch: { platformName: 'Android', 'appium:udid': udid },
+    firstMatch: [{}],
+});
+describe('The hub/node busy race', function () {
+    this.timeout(60000);
+    const scratch = (0, scratch_database_1.useScratchDatabase)();
+    let restoreContainer;
+    let savedStore;
+    let context;
+    let savedContext;
+    let clock;
+    const pluginArgs = Object.assign({}, IPluginArgs_1.DefaultPluginArgs);
+    const app = () => {
+        const a = (0, express_1.default)();
+        a.use(express_1.default.json());
+        a.use((req, _res, next) => {
+            req.auth = {
+                kind: 'api-key',
+                userId: 'node-user',
+                role: 'ADMIN',
+                scopes: 'devices',
+                rateLimit: 0,
+            };
+            next();
+        });
+        grid_1.default.register(a, pluginArgs);
+        return a;
+    };
+    /** The node's periodic report: its whole row for each phone. */
+    const report = (...phones) => __awaiter(this, void 0, void 0, function* () {
+        const res = yield (0, loopbackRequest_1.default)(app()).post('/register').query({ type: 'add' }).send(phones);
+        (0, chai_1.expect)(res.status, JSON.stringify(res.body)).to.equal(200);
+    });
+    const row = (udid_1, ...args_1) => __awaiter(this, [udid_1, ...args_1], void 0, function* (udid, host = NODE) { return (yield scratch.db.device.findFirst({ where: { udid, host } })); });
+    /** The hub allocates a phone for a create, as prepareSession does. */
+    const allocate = (udid) => (0, device_utils_1.allocateDeviceForSession)(caps(udid), 2000, 50, pluginArgs);
+    /** The create succeeded: the session is recorded on its phone. */
+    const finalize = (device, sessionId) => __awaiter(this, void 0, void 0, function* () {
+        const svc = new SessionLifecycleService_1.SessionLifecycleService();
+        sinon_1.default.stub(svc, 'createSessionInstance').returns({});
+        sinon_1.default.stub(svc, 'applyPostSessionLogic').resolves();
+        yield svc.finalizeSession({ protocol: 'W3C', value: [sessionId, { platformName: 'Android' }, 'W3C'] }, device, caps(device.udid), {}, false);
+    });
+    const deleteSession = (sessionId) => new SessionLifecycleService_1.SessionLifecycleService().deleteSession(() => __awaiter(this, void 0, void 0, function* () { return null; }), sessionId);
+    /** Let the sweepers' fire-and-forget releases land before looking. */
+    const settle = () => new Promise((resolve) => setTimeout(resolve, 150));
+    beforeEach(() => __awaiter(this, void 0, void 0, function* () {
+        restoreContainer = (0, container_registration_1.saveRegistrations)(device_managers_1.XenonManager, SocketServer_1.SocketServer, NotificationService_1.NotificationService, 'LocalStorage');
+        typedi_1.Container.set({
+            id: device_managers_1.XenonManager,
+            value: {
+                getMaxSessionCount: () => undefined,
+                deviceInstances: () => __awaiter(this, void 0, void 0, function* () { return []; }),
+                getDevices: () => __awaiter(this, void 0, void 0, function* () { return []; }),
+            },
+        });
+        typedi_1.Container.set({
+            id: SocketServer_1.SocketServer,
+            value: {
+                emitToDashboard: () => undefined,
+                emitToDashboardForDevices: () => __awaiter(this, void 0, void 0, function* () { return undefined; }),
+                hasScopedDashboard: () => false,
+            },
+        });
+        typedi_1.Container.set({ id: NotificationService_1.NotificationService, value: { dispatchEvent: () => __awaiter(this, void 0, void 0, function* () { return undefined; }) } });
+        typedi_1.Container.set({
+            id: 'LocalStorage',
+            value: { getItem: () => 0, setItem: () => __awaiter(this, void 0, void 0, function* () { return undefined; }) },
+        });
+        sinon_1.default.stub(event_manager_1.DASHBORD_EVENT_MANAGER, 'onSessionStopped').resolves();
+        context = typedi_1.Container.get(PluginContext_1.PluginContext);
+        savedContext = { nodeId: context.nodeId, pluginArgs: context.pluginArgs };
+        context.nodeId = HUB_ID;
+        context.pluginArgs = pluginArgs;
+        savedStore = device_store_1.DeviceStoreFactory._deviceStore;
+        device_store_1.DeviceStoreFactory._deviceStore = new prisma_store_1.PrismaDeviceStore();
+        yield scratch.db.device.deleteMany({});
+        yield scratch.db.team.deleteMany({});
+    }));
+    afterEach(() => {
+        clock === null || clock === void 0 ? void 0 : clock.restore();
+        clock = undefined;
+        device_store_1.DeviceStoreFactory._deviceStore = savedStore;
+        Object.assign(context, savedContext);
+        sinon_1.default.restore();
+        restoreContainer();
+    });
+    describe("a node's report while the hub holds a claim", () => {
+        it('leaves the phone busy while the session is created, and while it runs', () => __awaiter(this, void 0, void 0, function* () {
+            yield scratch.db.device.create({ data: nodePhone('s9') });
+            const device = yield allocate('s9');
+            // The node hasn't been handed the create yet: free, as far as it knows.
+            yield report(nodePhone('s9', { busy: false }));
+            (0, chai_1.expect)(yield row('s9')).to.include({ busy: true });
+            yield finalize(device, 'hub-session-1');
+            yield report(nodePhone('s9', { busy: false, session_id: null }));
+            (0, chai_1.expect)(yield row('s9')).to.include({ busy: true, session_id: 'hub-session-1' });
+            // A second client can't be given the phone.
+            let second;
+            try {
+                second = yield (0, device_utils_1.allocateDeviceForSession)(caps('s9'), 200, 50, pluginArgs);
+            }
+            catch (err) {
+                second = err;
+            }
+            (0, chai_1.expect)(second).to.be.an('error');
+        }));
+        it('keeps the phone busy after the hub ends its claim while the node still reports it busy', () => __awaiter(this, void 0, void 0, function* () {
+            yield scratch.db.device.create({ data: nodePhone('s9') });
+            const device = yield allocate('s9');
+            yield finalize(device, 'hub-session-1');
+            yield report(nodePhone('s9', { busy: true, session_id: 'hub-session-1' }));
+            yield deleteSession('hub-session-1');
+            (0, chai_1.expect)(yield row('s9')).to.include({ busy: true, nodeBusy: true, session_id: null });
+            yield report(nodePhone('s9', { busy: false, session_id: null }));
+            (0, chai_1.expect)(yield row('s9')).to.include({ busy: false, nodeBusy: false });
+        }));
+        it('keeps a hold the hub took (a preview, a recording) when the node reports the phone free', () => __awaiter(this, void 0, void 0, function* () {
+            yield scratch.db.device.create({
+                data: nodePhone('s9', { busy: true, session_id: 'manual_u1_s9' }),
+            });
+            yield report(nodePhone('s9', { busy: false, session_id: null }));
+            (0, chai_1.expect)(yield row('s9')).to.include({
+                busy: true,
+                nodeBusy: false,
+                session_id: 'manual_u1_s9',
+            });
+        }));
+        it("counts a node's own busy with no claim, and frees it when the node does", () => __awaiter(this, void 0, void 0, function* () {
+            yield scratch.db.device.create({ data: nodePhone('s9') });
+            yield report(nodePhone('s9', { busy: true, session_id: 'node-local' }));
+            (0, chai_1.expect)(yield row('s9')).to.include({ busy: true, nodeBusy: true });
+            yield report(nodePhone('s9', { busy: false, session_id: null }));
+            (0, chai_1.expect)(yield row('s9')).to.include({ busy: false, nodeBusy: false });
+        }));
+    });
+    describe("a node's report and the settings the hub owns", () => {
+        it('never overwrites the team, tags, reservation or block', () => __awaiter(this, void 0, void 0, function* () {
+            const team = yield scratch.db.team.create({ data: { name: 'Payments' } });
+            const reservedUntil = Date.now() + 3600000;
+            yield scratch.db.device.create({
+                data: Object.assign(Object.assign({}, nodePhone('s9')), { teamId: team.id, tags: JSON.stringify(['smoke']), userBlocked: true, reservedBy: 'alice', reservedUntil, reservationReason: 'release check' }),
+            });
+            // The node's copy of the row: none of the hub's settings.
+            yield report(nodePhone('s9', {
+                busy: false,
+                sdk: '11',
+                teamId: null,
+                tags: [],
+                userBlocked: false,
+                reservedBy: null,
+                reservedUntil: null,
+                reservationReason: null,
+            }));
+            (0, chai_1.expect)(yield row('s9')).to.include({
+                sdk: '11',
+                teamId: team.id,
+                tags: JSON.stringify(['smoke']),
+                userBlocked: true,
+                reservedBy: 'alice',
+                reservedUntil,
+                reservationReason: 'release check',
+            });
+        }));
+    });
+    describe('a release for an ended session', () => {
+        it("doesn't free a phone another session has claimed since", () => __awaiter(this, void 0, void 0, function* () {
+            yield scratch.db.device.create({ data: nodePhone('s9') });
+            const svc = new SessionLifecycleService_1.SessionLifecycleService();
+            // Create A takes the phone, and stalls.
+            const a = yield allocate('s9');
+            // Its claim is given up (a timeout), and create B takes the phone.
+            yield scratch.db.device.updateMany({
+                where: { udid: 's9' },
+                data: { busy: false, lastCmdExecutedAt: null },
+            });
+            const b = yield allocate('s9');
+            yield finalize(b, 'session-b');
+            // Create A fails at last and gives its phone back.
+            yield svc.releaseAllocation({ device: a, pendingSessionId: 'pending-a' });
+            (0, chai_1.expect)(yield row('s9')).to.include({ busy: true, session_id: 'session-b' });
+        }));
+        it("doesn't free a same-machine hub's row for the same udid", () => __awaiter(this, void 0, void 0, function* () {
+            yield scratch.db.device.create({ data: nodePhone('s9') });
+            yield scratch.db.device.create({
+                data: Object.assign(Object.assign({}, nodePhone('s9')), { host: HUB, nodeId: HUB_ID }),
+            });
+            const svc = new SessionLifecycleService_1.SessionLifecycleService();
+            // Two creates for the udid: one gets each row.
+            const running = yield allocate('s9');
+            yield finalize(running, 'session-running');
+            const failing = yield allocate('s9');
+            (0, chai_1.expect)([running.host, failing.host]).to.have.members([HUB, NODE]);
+            // The second create fails and gives its phone back.
+            yield svc.releaseAllocation({ device: failing, pendingSessionId: 'pending' });
+            (0, chai_1.expect)(yield row('s9', failing.host)).to.include({ busy: false });
+            (0, chai_1.expect)(yield row('s9', running.host)).to.include({
+                busy: true,
+                session_id: 'session-running',
+            });
+        }));
+    });
+    describe('a claim still being created, and the idle sweeper', () => {
+        it('is not idle at the new-command timeout, and is freed at its own timeout', () => __awaiter(this, void 0, void 0, function* () {
+            clock = sinon_1.default.useFakeTimers({ now: T0, toFake: ['Date'] });
+            yield scratch.db.device.create({ data: nodePhone('s9') });
+            yield allocate('s9');
+            // A first driver install: two minutes, past the 60 s new-command timeout.
+            clock.setSystemTime(T0 + 120000);
+            yield (0, device_utils_1.releaseBlockedDevices)(60);
+            yield settle();
+            (0, chai_1.expect)(yield row('s9')).to.include({ busy: true });
+            // A create that never finished.
+            clock.setSystemTime(T0 + 11 * 60000);
+            yield (0, device_utils_1.releaseBlockedDevices)(60);
+            yield settle();
+            (0, chai_1.expect)(yield row('s9')).to.include({ busy: false });
+        }));
+        it('frees a created session that went idle, as before', () => __awaiter(this, void 0, void 0, function* () {
+            clock = sinon_1.default.useFakeTimers({ now: T0, toFake: ['Date'] });
+            yield scratch.db.device.create({ data: nodePhone('s9') });
+            yield finalize(yield allocate('s9'), 'idle-session');
+            clock.setSystemTime(T0 + 120000);
+            yield (0, device_utils_1.releaseBlockedDevices)(60);
+            yield settle();
+            (0, chai_1.expect)(yield row('s9')).to.include({ busy: false, session_id: null });
+        }));
+    });
+    describe('a release racing a new claim', () => {
+        it('a release that read the phone before its next claim leaves that claim alone', () => __awaiter(this, void 0, void 0, function* () {
+            clock = sinon_1.default.useFakeTimers({ now: T0, toFake: ['Date'] });
+            yield scratch.db.device.create({ data: nodePhone('s9') });
+            yield finalize(yield allocate('s9'), 'session-a');
+            clock.setSystemTime(T0 + 120000);
+            // The idle sweeper reads the phone with session A on it. Before it
+            // writes, A is deleted and session B claims the phone.
+            const findMany = prisma_1.prisma.device.findMany;
+            let interleaved = false;
+            findMany.callsFake((args) => __awaiter(this, void 0, void 0, function* () {
+                const rows = yield scratch.db.device.findMany(args);
+                if (!interleaved && !(args === null || args === void 0 ? void 0 : args.where)) {
+                    interleaved = true;
+                    yield deleteSession('session-a');
+                    yield finalize(yield allocate('s9'), 'session-b');
+                }
+                return rows;
+            }));
+            yield (0, device_utils_1.releaseBlockedDevices)(60);
+            yield settle();
+            (0, chai_1.expect)(interleaved).to.equal(true);
+            (0, chai_1.expect)(yield row('s9')).to.include({ busy: true, session_id: 'session-b' });
+        }));
+        it('two releases of one session leave the next claim alone', () => __awaiter(this, void 0, void 0, function* () {
+            yield scratch.db.device.create({ data: nodePhone('s9') });
+            yield finalize(yield allocate('s9'), 'session-a');
+            yield Promise.all([deleteSession('session-a'), deleteSession('session-a')]);
+            yield finalize(yield allocate('s9'), 'session-b');
+            yield deleteSession('session-a');
+            (0, chai_1.expect)(yield row('s9')).to.include({ busy: true, session_id: 'session-b' });
+        }));
+    });
+    describe('stale cleanup of node phones', () => {
+        // The hub also files iPhones under a bare remoteMachineProxyIP, and so
+        // does the node: the node's iPhone has a host the hub calls its own.
+        const hubHosts = (0, localDeviceHosts_1.localDeviceHosts)({ bindHostOrIp: IP, remoteMachineProxyIP: '203.0.113.7' }, 4724);
+        const nodeIphone = () => nodePhone('iphone', { host: '203.0.113.7', platform: 'ios', deviceType: 'real' });
+        it("the hub's local sync never prunes a phone carrying the node's id", () => __awaiter(this, void 0, void 0, function* () {
+            yield scratch.db.device.create({ data: nodeIphone() });
+            yield scratch.db.device.create({
+                data: Object.assign(Object.assign({}, nodePhone('gone')), { host: HUB, nodeId: HUB_ID }),
+            });
+            yield (0, device_utils_1.updateDeviceList)(hubHosts);
+            (0, chai_1.expect)(yield row('iphone', '203.0.113.7')).to.include({ nodeId: NODE_ID });
+            (0, chai_1.expect)(yield row('gone', HUB)).to.equal(null);
+        }));
+        it("drops the node's phones with the node, keyed on its id", () => __awaiter(this, void 0, void 0, function* () {
+            yield scratch.db.device.create({ data: nodeIphone() });
+            yield scratch.db.device.create({ data: nodePhone('s9') });
+            yield scratch.db.device.create({
+                data: Object.assign(Object.assign({}, nodePhone('hub-phone')), { host: HUB, nodeId: HUB_ID }),
+            });
+            const alive = sinon_1.default.stub(helpers, 'isXenonRunning').callsFake((host) => __awaiter(this, void 0, void 0, function* () { return host === NODE; }));
+            yield (0, device_utils_1.removeStaleDevices)(hubHosts);
+            (0, chai_1.expect)(yield row('iphone', '203.0.113.7'), 'kept while its node answers').to.not.equal(null);
+            (0, chai_1.expect)(yield row('s9')).to.not.equal(null);
+            alive.callsFake(() => __awaiter(this, void 0, void 0, function* () { return false; }));
+            yield (0, device_utils_1.removeStaleDevices)(hubHosts);
+            (0, chai_1.expect)(yield row('iphone', '203.0.113.7'), 'dropped with its node').to.equal(null);
+            (0, chai_1.expect)(yield row('s9')).to.equal(null);
+            (0, chai_1.expect)(yield row('hub-phone', HUB), "the hub's own").to.not.equal(null);
+            (0, chai_1.expect)(alive.calledWith(HUB)).to.equal(false);
+        }));
+    });
+    describe('a standalone server', () => {
+        it('allocates, runs and frees its own phone as before', () => __awaiter(this, void 0, void 0, function* () {
+            yield scratch.db.device.create({
+                data: Object.assign(Object.assign({}, nodePhone('pixel')), { host: HUB, nodeId: HUB_ID }),
+            });
+            const device = yield allocate('pixel');
+            (0, chai_1.expect)(yield row('pixel', HUB)).to.include({ busy: true });
+            yield finalize(device, 'local-session');
+            (0, chai_1.expect)(yield row('pixel', HUB)).to.include({ busy: true, session_id: 'local-session' });
+            yield deleteSession('local-session');
+            (0, chai_1.expect)(yield row('pixel', HUB)).to.include({
+                busy: false,
+                session_id: null,
+                lastCmdExecutedAt: null,
+                sessionStartTime: 0,
+            });
+            (0, chai_1.expect)(yield row('pixel', HUB)).to.not.include({ nodeBusy: true });
+            // And it can be allocated again.
+            yield allocate('pixel');
+            (0, chai_1.expect)(yield row('pixel', HUB)).to.include({ busy: true });
+        }));
+    });
+});