@xenon-device-management/xenon 2.13.2 → 2.15.0

package/lib/test/unit/token-mint-limits.spec.js

@@ -0,0 +1,165 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+require("reflect-metadata");
+const chai_1 = require("chai");
+const jose = __importStar(require("jose"));
+const typedi_1 = require("typedi");
+const fs_1 = __importDefault(require("fs"));
+const os_1 = __importDefault(require("os"));
+const path_1 = __importDefault(require("path"));
+const JwtKeyService_1 = require("../../src/services/token/JwtKeyService");
+const auth_1 = require("../../src/app/routers/auth");
+const container_registration_1 = require("../helpers/container-registration");
+/**
+ * A token POST /auth/token mints never carries more than the credential
+ * minting it: not more scopes, and not a longer life.
+ *
+ * - The `sessionToken` beside a xenon-mcp token creates Appium sessions, so
+ *   it is minted only when the grant includes `appium:use` (which needs the
+ *   `sessions` scope), and carries the scopes a session is judged by. Any
+ *   credential used to get one, a read-only key included.
+ * - A token minted with a Bearer token, or an API key that expires, ends no
+ *   later than that credential. A 1-hour token used to mint a fresh 1-hour
+ *   token, for ever.
+ */
+describe('POST /auth/token mints nothing its creator does not have', () => {
+    let dir;
+    let restoreContainer;
+    beforeEach(() => __awaiter(void 0, void 0, void 0, function* () {
+        restoreContainer = (0, container_registration_1.saveRegistrations)(JwtKeyService_1.JwtKeyService);
+        dir = fs_1.default.mkdtempSync(path_1.default.join(os_1.default.tmpdir(), 'xenon-mint-limits-'));
+        const svc = new JwtKeyService_1.JwtKeyService();
+        yield svc.init(dir);
+        typedi_1.Container.set(JwtKeyService_1.JwtKeyService, svc);
+    }));
+    afterEach(() => {
+        restoreContainer();
+        fs_1.default.rmSync(dir, { recursive: true, force: true });
+    });
+    const base = { kind: 'api-key', userId: 'u1', role: 'MEMBER', teamId: null, rateLimit: 300 };
+    const auth = (extra) => (Object.assign(Object.assign({}, base), extra));
+    describe('the session token', () => {
+        it('is not minted for a read-only credential', () => __awaiter(void 0, void 0, void 0, function* () {
+            const out = yield (0, auth_1.issueToken)(auth({ scopes: 'read' }), { audience: 'xenon-mcp' });
+            (0, chai_1.expect)(out.token).to.be.a('string');
+            (0, chai_1.expect)(out.sessionToken).to.equal(undefined);
+        }));
+        it('is not minted when the grant asked for leaves out appium:use', () => __awaiter(void 0, void 0, void 0, function* () {
+            const out = yield (0, auth_1.issueToken)(auth({ scopes: 'devices,sessions,read' }), {
+                audience: 'xenon-mcp',
+                scopes: ['xenon:devices:read'],
+            });
+            (0, chai_1.expect)(out.sessionToken).to.equal(undefined);
+        }));
+        it('carries `sessions`, and `admin` only when its creator has it', () => __awaiter(void 0, void 0, void 0, function* () {
+            const member = yield (0, auth_1.issueToken)(auth({ scopes: 'devices,sessions,read' }), {
+                audience: 'xenon-mcp',
+            });
+            (0, chai_1.expect)(jose.decodeJwt(String(member.sessionToken)).scopes).to.equal('sessions');
+            const dashboardAdmin = yield (0, auth_1.issueToken)(auth({ kind: 'user-session', role: 'ADMIN', scopes: 'admin,devices,sessions,read' }), { audience: 'xenon-mcp' });
+            (0, chai_1.expect)(jose.decodeJwt(String(dashboardAdmin.sessionToken)).scopes).to.equal('admin,sessions');
+        }));
+    });
+    // `admin` lets a session take over a lease someone else made. A token asked
+    // for with narrower MCP scopes doesn't get it from an admin key's breadth.
+    describe("the session token's admin scope follows the grant", () => {
+        const adminKey = () => auth({ scopes: 'admin' });
+        const scopesOf = (t) => jose.decodeJwt(String(t)).scopes;
+        it('is left out when narrower scopes are asked for', () => __awaiter(void 0, void 0, void 0, function* () {
+            const out = yield (0, auth_1.issueToken)(adminKey(), { audience: 'xenon-mcp', scopes: ['appium:use'] });
+            (0, chai_1.expect)(scopesOf(out.sessionToken)).to.equal('sessions');
+        }));
+        it('is given for the default grant, and for a full-admin one', () => __awaiter(void 0, void 0, void 0, function* () {
+            const byDefault = yield (0, auth_1.issueToken)(adminKey(), { audience: 'xenon-mcp' });
+            (0, chai_1.expect)(scopesOf(byDefault.sessionToken)).to.equal('admin,sessions');
+            const full = yield (0, auth_1.issueToken)(adminKey(), {
+                audience: 'xenon-mcp',
+                scopes: [
+                    'appium:use',
+                    'xenon:analytics:read',
+                    'xenon:devices:lock',
+                    'xenon:devices:read',
+                    'xenon:recordings',
+                ],
+            });
+            (0, chai_1.expect)(scopesOf(full.sessionToken)).to.equal('admin,sessions');
+        }));
+    });
+    describe('its lifetime', () => {
+        const now = () => Math.floor(Date.now() / 1000);
+        for (const audience of ['xenon-rest', 'xenon-mcp']) {
+            it(`a ${audience} token ends no later than the credential that minted it`, () => __awaiter(void 0, void 0, void 0, function* () {
+                const creatorExp = now() + 600;
+                const out = yield (0, auth_1.issueToken)(auth({ kind: 'bearer', scopes: 'sessions,read', credentialExpiresAt: creatorExp * 1000 }), { audience });
+                const p = jose.decodeJwt(out.token);
+                (0, chai_1.expect)(p.exp).to.be.at.most(creatorExp);
+                (0, chai_1.expect)(out.expiresIn).to.be.at.most(600);
+                (0, chai_1.expect)(out.expiresIn).to.be.at.least(598);
+                if (out.sessionToken) {
+                    (0, chai_1.expect)(jose.decodeJwt(out.sessionToken).exp).to.be.at.most(creatorExp);
+                }
+            }));
+        }
+        it('keeps its own lifetime when the credential outlives it', () => __awaiter(void 0, void 0, void 0, function* () {
+            const out = yield (0, auth_1.issueToken)(auth({ scopes: 'sessions', credentialExpiresAt: Date.now() + 30 * 86400000 }), { audience: 'xenon-rest' });
+            (0, chai_1.expect)(out.expiresIn).to.equal(3600);
+        }));
+        it('is refused when the credential has already expired', () => __awaiter(void 0, void 0, void 0, function* () {
+            let caught;
+            try {
+                yield (0, auth_1.issueToken)(auth({ scopes: 'sessions', credentialExpiresAt: Date.now() - 1000 }), {
+                    audience: 'xenon-rest',
+                });
+            }
+            catch (err) {
+                caught = err;
+            }
+            (0, chai_1.expect)(caught === null || caught === void 0 ? void 0 : caught.message).to.match(/expired/);
+        }));
+    });
+});

package/lib/test/unit/unexpected-shutdown-reason.spec.js

@@ -0,0 +1,163 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+require("reflect-metadata");
+const chai_1 = require("chai");
+const sinon_1 = __importDefault(require("sinon"));
+const typedi_1 = require("typedi");
+const plugin_1 = require("../../src/plugin");
+const DeviceService = __importStar(require("../../src/data-service/device-service"));
+const device_store_1 = require("../../src/data-service/device-store");
+const SessionManager_1 = require("../../src/sessions/SessionManager");
+const SessionMetricsService_1 = require("../../src/services/metrics/SessionMetricsService");
+const PhoneNetworkRestore_1 = require("../../src/services/network/PhoneNetworkRestore");
+const SocketServer_1 = require("../../src/services/SocketServer");
+const MetricsService_1 = require("../../src/services/MetricsService");
+const NotificationService_1 = require("../../src/services/NotificationService");
+const AIService_1 = require("../../src/services/AIService");
+const shutdownReason_1 = require("../../src/services/session/shutdownReason");
+const prisma_1 = require("../../src/prisma");
+const container_registration_1 = require("../helpers/container-registration");
+const scratch_database_1 = require("../helpers/scratch-database");
+/**
+ * A session Appium ends by itself reaches `XenonPlugin.onUnexpectedShutdown`
+ * with a cause. For an idle session that is Appium's new-command timeout.
+ * The plugin threw the cause away and recorded "Driver shut down
+ * unexpectedly", which no failure pattern matches, so an idle session on this
+ * server's own phones was filed UNKNOWN and its "Why it failed" card opened
+ * the general runbook rather than the Timeout one.
+ */
+// What @appium/base-driver 10.2 passes when a session sits idle too long.
+const IDLE = "New Command Timeout of 60 seconds expired. Try customizing the timeout using the 'newCommandTimeout' desired capability";
+describe('the reason of a session Appium ends by itself', () => {
+    it("is Appium's cause: its new-command timeout", () => {
+        (0, chai_1.expect)((0, shutdownReason_1.unexpectedShutdownReason)(new Error(IDLE))).to.equal(IDLE);
+    });
+    it("is a driver's own account of giving up, which still reads as one", () => {
+        (0, chai_1.expect)((0, shutdownReason_1.unexpectedShutdownReason)(new Error('The driver was unexpectedly shut down!'))).to.equal('The driver was unexpectedly shut down!');
+    });
+    it('is the old fixed reason when there is no cause to tell', () => {
+        for (const cause of [undefined, null, new Error(''), new Error('   '), {}, 42]) {
+            (0, chai_1.expect)((0, shutdownReason_1.unexpectedShutdownReason)(cause), String(cause)).to.equal(shutdownReason_1.UNEXPECTED_SHUTDOWN_REASON);
+        }
+        // The umbrella's placeholder when a driver gives none.
+        (0, chai_1.expect)((0, shutdownReason_1.unexpectedShutdownReason)(new Error('Unknown error'))).to.equal(shutdownReason_1.UNEXPECTED_SHUTDOWN_REASON);
+    });
+    it('is the message alone, never a stack', () => {
+        const err = new Error(IDLE);
+        (0, chai_1.expect)((0, shutdownReason_1.unexpectedShutdownReason)(err)).not.to.contain('    at ');
+    });
+});
+describe('a session Appium ends for being idle, on a hub that records sessions', function () {
+    this.timeout(30000);
+    const scratch = (0, scratch_database_1.useScratchDatabase)();
+    let restoreRegs;
+    let wasHub;
+    const plugin = {
+        pluginArgs: { enableDashboard: true },
+        xenonLog: { withSession: () => ({ info() { } }) },
+    };
+    const seed = (id) => scratch.db.session.create({
+        data: {
+            id,
+            status: 'running',
+            desired_capabilities: '{}',
+            session_capabilities: '{}',
+            node_id: 'n-1',
+            has_live_video: false,
+            device_udid: 'R5CT',
+            device_platform: 'android',
+            device_version: '14',
+        },
+    });
+    const shutDown = (id, cause) => plugin_1.XenonPlugin.prototype.onUnexpectedShutdown.call(plugin, { sessionId: id, caps: { udid: 'R5CT' } }, cause);
+    beforeEach(() => __awaiter(this, void 0, void 0, function* () {
+        restoreRegs = (0, container_registration_1.saveRegistrations)(PhoneNetworkRestore_1.PhoneNetworkRestore, SessionMetricsService_1.SessionMetricsService, SocketServer_1.SocketServer, MetricsService_1.MetricsService, NotificationService_1.NotificationService);
+        typedi_1.Container.set(PhoneNetworkRestore_1.PhoneNetworkRestore, { restoreSession: sinon_1.default.stub().resolves() });
+        typedi_1.Container.set(SessionMetricsService_1.SessionMetricsService, { stop: sinon_1.default.stub().resolves() });
+        typedi_1.Container.set(SocketServer_1.SocketServer, {
+            emitToDashboard: sinon_1.default.stub(),
+            emitToDashboardForDevices: sinon_1.default.stub().resolves(),
+            hasScopedDashboard: sinon_1.default.stub().returns(false),
+        });
+        typedi_1.Container.set(MetricsService_1.MetricsService, {
+            incrementSessionSuccess: sinon_1.default.stub(),
+            incrementSessionFailure: sinon_1.default.stub(),
+        });
+        typedi_1.Container.set(NotificationService_1.NotificationService, { notifySessionFailed: sinon_1.default.stub().resolves() });
+        sinon_1.default.stub(DeviceService, 'releaseSessionDevices').resolves();
+        sinon_1.default
+            .stub(device_store_1.DeviceStoreFactory, 'getStore')
+            .returns({ getDevices: () => __awaiter(this, void 0, void 0, function* () { return []; }), findDevices: () => __awaiter(this, void 0, void 0, function* () { return []; }) });
+        sinon_1.default.stub(SessionManager_1.SESSION_MANAGER, 'getSession').returns(undefined);
+        sinon_1.default.stub(AIService_1.AI_SERVICE, 'isEnabled').returns(false);
+        wasHub = plugin_1.XenonPlugin.IS_HUB;
+        plugin_1.XenonPlugin.IS_HUB = true;
+        yield scratch.db.session.deleteMany({});
+    }));
+    afterEach(() => {
+        plugin_1.XenonPlugin.IS_HUB = wasHub;
+        sinon_1.default.restore();
+        restoreRegs();
+    });
+    it("is failed with Appium's reason and filed TIMEOUT", () => __awaiter(this, void 0, void 0, function* () {
+        yield seed('s-idle');
+        yield shutDown('s-idle', new Error(IDLE));
+        const row = yield prisma_1.prisma.session.findUnique({ where: { id: 's-idle' } });
+        (0, chai_1.expect)(row === null || row === void 0 ? void 0 : row.status).to.equal('failed');
+        (0, chai_1.expect)(row === null || row === void 0 ? void 0 : row.failure_reason).to.equal(IDLE);
+        (0, chai_1.expect)(row === null || row === void 0 ? void 0 : row.failure_category).to.equal('TIMEOUT');
+    }));
+    it('keeps the old reason, filed UNKNOWN, when Appium gives no cause', () => __awaiter(this, void 0, void 0, function* () {
+        yield seed('s-crash');
+        yield shutDown('s-crash', undefined);
+        const row = yield prisma_1.prisma.session.findUnique({ where: { id: 's-crash' } });
+        (0, chai_1.expect)(row === null || row === void 0 ? void 0 : row.status).to.equal('failed');
+        (0, chai_1.expect)(row === null || row === void 0 ? void 0 : row.failure_reason).to.equal(shutdownReason_1.UNEXPECTED_SHUTDOWN_REASON);
+        (0, chai_1.expect)(row === null || row === void 0 ? void 0 : row.failure_category).to.equal('UNKNOWN');
+    }));
+});

package/lib/test/unit/verify-credential.spec.js

@@ -24,6 +24,7 @@ const UserService_1 = require("../../src/services/UserService");
 const ApiKeyService_1 = require("../../src/services/ApiKeyService");
 const verifyCredential_1 = require("../../src/middleware/verifyCredential");
 const container_registration_1 = require("../helpers/container-registration");
+const commandCaller_1 = require("../../src/middleware/commandCaller");
 /**
  * The credential checks authMiddleware applies to the header pair and to a
  * Bearer JWT, pulled out so the WebDriver per-command check verifies exactly
@@ -80,6 +81,38 @@ describe('verifyCredential', () => {
             const token = yield keys.sign({ sub: 'u1' }, { audience: 'xenon-rest', ttlSeconds: 60 });
             (0, chai_1.expect)(yield (0, verifyCredential_1.verifyBearerCredential)(token)).to.equal(null);
         }));
+        // A token's scopes are fixed when it is minted, for up to a day. An
+        // admin demoted to member since keeps no `admin` from it: REST
+        // (resolveActor, scopeGuard) and per-command auth's override read these.
+        describe("the scopes, by the user's role now", () => {
+            const mint = (scopes, audience = 'xenon-rest') => keys.sign({ sub: 'u1', scopes }, { audience, ttlSeconds: 60 });
+            const withRole = (role) => typedi_1.Container.get(UserService_1.UserService).findById.resolves(Object.assign(Object.assign({}, active), { role }));
+            it('drops admin from the token of a user who is now a member', () => __awaiter(void 0, void 0, void 0, function* () {
+                withRole('MEMBER');
+                for (const audience of ['xenon-rest', 'xenon-mcp']) {
+                    const out = yield (0, verifyCredential_1.verifyBearerCredential)(yield mint('admin,devices,sessions,read', audience));
+                    (0, chai_1.expect)(out === null || out === void 0 ? void 0 : out.payload.scopes, audience).to.equal('devices,sessions,read');
+                }
+            }));
+            it('keeps admin for an ADMIN or SUPER_ADMIN', () => __awaiter(void 0, void 0, void 0, function* () {
+                for (const role of ['ADMIN', 'SUPER_ADMIN']) {
+                    withRole(role);
+                    const out = yield (0, verifyCredential_1.verifyBearerCredential)(yield mint('admin,sessions'));
+                    (0, chai_1.expect)(out === null || out === void 0 ? void 0 : out.payload.scopes, role).to.equal('admin,sessions');
+                }
+            }));
+            it('gives a demoted admin no override in per-command auth', () => __awaiter(void 0, void 0, void 0, function* () {
+                withRole('MEMBER');
+                const verdict = yield new commandCaller_1.CommandCallerVerifier().verify({
+                    kind: 'bearer',
+                    token: yield mint('admin,sessions'),
+                });
+                (0, chai_1.expect)(verdict).to.deep.equal({
+                    valid: true,
+                    caller: { userId: 'u1', overrideAdmin: false },
+                });
+            }));
+        });
         it('throws when the signing key is not available, rather than calling the token invalid', () => __awaiter(void 0, void 0, void 0, function* () {
             typedi_1.Container.set(JwtKeyService_1.JwtKeyService, new JwtKeyService_1.JwtKeyService()); // never initialised
             const token = yield keys.sign({ sub: 'u1' }, { audience: 'xenon-rest', ttlSeconds: 60 });

package/lib/test/unit/vision-assertion.spec.js

@@ -0,0 +1,119 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+require("reflect-metadata");
+const chai_1 = require("chai");
+const sinon_1 = __importDefault(require("sinon"));
+const VisionAssertionService_1 = require("../../src/services/omni-vision/VisionAssertionService");
+const AIService_1 = require("../../src/services/AIService");
+/**
+ * `xenon: assertVisualState` answered `{ result: true, message: 'Assertion
+ * placeholder' }` whenever the AI step gave it nothing usable: with no AI
+ * provider, when the provider failed, and usually even with one, because the
+ * condition was wrapped in a "find the centre coordinates, answer {x, y}"
+ * prompt. A test asserting a screen state passed without anything being
+ * checked.
+ *
+ * Now it asks the provider a yes/no question of the screenshot and answers
+ * with the provider's verdict. Whenever there is no verdict (no provider, a
+ * failed call, an answer that isn't true or false, no screenshot, no
+ * condition) it fails with an error the test sees, never `true` and never
+ * `false`: a `false` would pass a test that asserts a state is absent.
+ */
+describe('xenon: assertVisualState', () => {
+    const SCREEN = 'c2NyZWVuc2hvdA==';
+    let service;
+    let driver;
+    beforeEach(() => {
+        service = new VisionAssertionService_1.VisionAssertionService();
+        driver = { getScreenshot: sinon_1.default.stub().resolves(SCREEN) };
+        // The real provider set-up reads the server's AI settings; these tests set the provider themselves.
+        sinon_1.default.stub(AIService_1.AI_SERVICE, 'initializeProvider');
+    });
+    afterEach(() => sinon_1.default.restore());
+    const failureOf = (p) => p.then((value) => {
+        throw new Error(`expected a failure, got ${JSON.stringify(value)}`);
+    }, (e) => e);
+    function withProvider(answer) {
+        sinon_1.default.stub(AIService_1.AI_SERVICE, 'provider').value({ analyze: () => __awaiter(this, void 0, void 0, function* () { return ''; }) });
+        const call = sinon_1.default.stub(AIService_1.AI_SERVICE, 'callProvider');
+        if (answer instanceof Error)
+            call.rejects(answer);
+        else
+            call.resolves(answer);
+        return call;
+    }
+    it('fails, rather than passing, when no AI provider is configured', () => __awaiter(void 0, void 0, void 0, function* () {
+        sinon_1.default.stub(AIService_1.AI_SERVICE, 'provider').value(null);
+        const err = yield failureOf(service.assertState(driver, 'The Login button is visible'));
+        (0, chai_1.expect)(err.message).to.match(/No AI provider is configured/);
+        (0, chai_1.expect)(err.message).to.match(/not checked/);
+    }));
+    it('asks a yes/no question about the condition, with the screenshot', () => __awaiter(void 0, void 0, void 0, function* () {
+        const call = withProvider('{"result": true, "reason": "A Login button is shown."}');
+        const answer = yield service.assertState(driver, 'The Login button is visible');
+        (0, chai_1.expect)(answer).to.deep.equal({ result: true, message: 'A Login button is shown.' });
+        (0, chai_1.expect)(call.calledOnce).to.equal(true);
+        const [prompt, image] = call.firstCall.args;
+        (0, chai_1.expect)(image).to.equal(SCREEN);
+        (0, chai_1.expect)(prompt).to.include('The Login button is visible');
+        (0, chai_1.expect)(prompt).to.match(/true or false/i);
+        (0, chai_1.expect)(prompt).to.not.match(/coordinates/i);
+    }));
+    it("answers false when the provider says the condition doesn't hold", () => __awaiter(void 0, void 0, void 0, function* () {
+        withProvider('```json\n{"result": false, "reason": "The cart shows 2 items."}\n```');
+        (0, chai_1.expect)(yield service.assertState(driver, 'The cart is empty')).to.deep.equal({
+            result: false,
+            message: 'The cart shows 2 items.',
+        });
+    }));
+    it("fails when the provider's answer is not a true/false verdict", () => __awaiter(void 0, void 0, void 0, function* () {
+        for (const answer of ['{"x": 120, "y": 340}', 'Yes, I can see it.', '{"result": "maybe"}']) {
+            sinon_1.default.restore();
+            sinon_1.default.stub(AIService_1.AI_SERVICE, 'initializeProvider');
+            driver = { getScreenshot: sinon_1.default.stub().resolves(SCREEN) };
+            withProvider(answer);
+            const err = yield failureOf(service.assertState(driver, 'The cart is empty'));
+            (0, chai_1.expect)(err.message, answer).to.match(/could not read/i);
+        }
+    }));
+    it('fails when the provider call fails', () => __awaiter(void 0, void 0, void 0, function* () {
+        withProvider(new Error('429 quota exceeded'));
+        const err = yield failureOf(service.assertState(driver, 'The cart is empty'));
+        (0, chai_1.expect)(err.message).to.match(/429 quota exceeded/);
+    }));
+    it('fails when the provider is rate-limited', () => __awaiter(void 0, void 0, void 0, function* () {
+        // Any provider's 429 (ai-rate-limit.spec.ts).
+        withProvider(new AIService_1.AIRateLimitedError('429 Too Many Requests'));
+        const err = yield failureOf(service.assertState(driver, 'The cart is empty'));
+        (0, chai_1.expect)(err.message).to.match(/rate-limited/i);
+        (0, chai_1.expect)(err.message).to.match(/not checked/);
+    }));
+    it('fails without asking when there is no condition', () => __awaiter(void 0, void 0, void 0, function* () {
+        const call = withProvider('{"result": true, "reason": "ok"}');
+        for (const empty of ['', '   ', undefined]) {
+            const err = yield failureOf(service.assertState(driver, empty));
+            (0, chai_1.expect)(err.message).to.match(/needs a condition/);
+        }
+        (0, chai_1.expect)(call.called).to.equal(false);
+        (0, chai_1.expect)(driver.getScreenshot.called).to.equal(false);
+    }));
+    it('fails when no screenshot can be taken', () => __awaiter(void 0, void 0, void 0, function* () {
+        const call = withProvider('{"result": true, "reason": "ok"}');
+        driver.getScreenshot.rejects(new Error('WDA is not running'));
+        const err = yield failureOf(service.assertState(driver, 'The cart is empty'));
+        (0, chai_1.expect)(err.message).to.match(/WDA is not running/);
+        (0, chai_1.expect)(call.called).to.equal(false);
+    }));
+});

package/lib/test/unit/wda-client-shell.spec.js

@@ -0,0 +1,133 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+require("reflect-metadata");
+const chai_1 = require("chai");
+const fs_1 = __importDefault(require("fs"));
+const os_1 = __importDefault(require("os"));
+const path_1 = __importDefault(require("path"));
+const sinon_1 = __importDefault(require("sinon"));
+const typedi_1 = require("typedi");
+const IOSStreamService_1 = __importDefault(require("../../src/device-managers/ios/IOSStreamService"));
+const IOSTunnels_1 = require("../../src/device-managers/ios/IOSTunnels");
+const WDAClient_1 = require("../../src/device-managers/ios/WDAClient");
+const device_store_1 = require("../../src/data-service/device-store");
+/**
+ * The Shell tab on iOS: simctl commands and a few generic ones on simulators,
+ * go-ios commands on real iPhones. Through 2.13.2 a real iPhone's command could
+ * carry its own `--udid` (another phone on the Mac), `list` named every device
+ * attached to the Mac, simulator commands took any path on this Mac, and the
+ * go-ios call ran without the phone's tunnel, so iOS 17+ phones never answered.
+ *
+ * `xcrun` and `ios` here are scripts that print the arguments they were given.
+ */
+const SIMULATOR = 'TEST-SIM-0000-AAAA';
+const IPHONE = '00008110-00084CE80E51401E';
+const ECHO = `#!/bin/sh
+printf '%s|' "$@"
+printf 'port=%s' "$GO_IOS_AGENT_PORT"
+`;
+describe('WDAClient.executeShell', () => {
+    let dir;
+    let savedPath;
+    let realDevice;
+    const borrowed = [];
+    before(() => {
+        dir = fs_1.default.mkdtempSync(path_1.default.join(os_1.default.tmpdir(), 'xenon-fake-shell-'));
+        fs_1.default.writeFileSync(path_1.default.join(dir, 'xcrun'), ECHO, { mode: 0o755 });
+        fs_1.default.writeFileSync(path_1.default.join(dir, 'ios'), ECHO, { mode: 0o755 });
+    });
+    after(() => fs_1.default.rmSync(dir, { recursive: true, force: true }));
+    beforeEach(() => {
+        savedPath = process.env.PATH;
+        process.env.PATH = `${dir}:${process.env.PATH}`;
+        borrowed.length = 0;
+        const stream = { isGoIOSAvailable: () => __awaiter(void 0, void 0, void 0, function* () { return true; }), goIOSPath: path_1.default.join(dir, 'ios') };
+        const tunnels = {
+            borrow(udid) {
+                return __awaiter(this, void 0, void 0, function* () {
+                    borrowed.push(udid);
+                    return 12100;
+                });
+            },
+            envFor: () => (Object.assign(Object.assign({}, process.env), { GO_IOS_AGENT_PORT: '12100' })),
+        };
+        const real = typedi_1.Container.get.bind(typedi_1.Container);
+        sinon_1.default.stub(typedi_1.Container, 'get').callsFake((token) => {
+            if (token === IOSStreamService_1.default)
+                return stream;
+            if (token === IOSTunnels_1.IOSTunnels)
+                return tunnels;
+            return real(token);
+        });
+        sinon_1.default
+            .stub(device_store_1.DeviceStoreFactory, 'getStore')
+            .returns({ findDevice: () => __awaiter(void 0, void 0, void 0, function* () { return ({ realDevice }); }) });
+    });
+    afterEach(() => {
+        process.env.PATH = savedPath;
+        sinon_1.default.restore();
+    });
+    const run = (udid, command) => new WDAClient_1.WDAClient().executeShell(udid, command).then((out) => ({ out }), (err) => ({ err: err.message }));
+    describe('on a simulator', () => {
+        beforeEach(() => {
+            realDevice = false;
+        });
+        it('runs a simctl command for this simulator', () => __awaiter(void 0, void 0, void 0, function* () {
+            const { out } = (yield run(SIMULATOR, 'getenv HOME'));
+            (0, chai_1.expect)(out).to.match(new RegExp(`^simctl\\|getenv\\|${SIMULATOR}\\|HOME\\|`));
+        }));
+        it('spawns a generic command in this simulator, without arguments', () => __awaiter(void 0, void 0, void 0, function* () {
+            const { out } = (yield run(SIMULATOR, 'ls'));
+            (0, chai_1.expect)(out).to.match(new RegExp(`^simctl\\|spawn\\|${SIMULATOR}\\|ls\\|`));
+            (0, chai_1.expect)(yield run(SIMULATOR, 'ls /Users'))
+                .to.have.property('err')
+                .that.match(/not allowed/);
+        }));
+        it('refuses list, chaining and look-alikes', () => __awaiter(void 0, void 0, void 0, function* () {
+            for (const cmd of ['list', 'ls; id', 'lsof', 'getenv $HOME']) {
+                (0, chai_1.expect)(yield run(SIMULATOR, cmd), cmd)
+                    .to.have.property('err')
+                    .that.match(/not allowed/);
+            }
+        }));
+    });
+    describe('on a real iPhone', () => {
+        beforeEach(() => {
+            realDevice = true;
+        });
+        it('runs a go-ios command for this phone, through its tunnel', () => __awaiter(void 0, void 0, void 0, function* () {
+            const { out } = (yield run(IPHONE, 'apps --list'));
+            (0, chai_1.expect)(out).to.equal(`apps|--list|--udid|${IPHONE}|port=12100`);
+            (0, chai_1.expect)(borrowed).to.deep.equal([IPHONE]);
+        }));
+        it('refuses a command that names another device, and list', () => __awaiter(void 0, void 0, void 0, function* () {
+            for (const cmd of [
+                'apps --udid 00008101-OTHER',
+                'apps --udid=00008101-OTHER',
+                'list',
+                'info --json',
+            ]) {
+                (0, chai_1.expect)(yield run(IPHONE, cmd), cmd)
+                    .to.have.property('err')
+                    .that.match(/not allowed/);
+            }
+        }));
+        it('says a generic command needs a simulator', () => __awaiter(void 0, void 0, void 0, function* () {
+            (0, chai_1.expect)(yield run(IPHONE, 'ls'))
+                .to.have.property('err')
+                .that.match(/real/i);
+        }));
+    });
+});