@xenon-device-management/xenon 2.16.0 → 2.17.0

package/lib/src/services/SessionLifecycleService.js

@@ -79,6 +79,7 @@ const appVisibility_1 = require("./device-access/appVisibility");
 const LiveSessionOwners_1 = require("./device-access/LiveSessionOwners");
 const sessionMemory_1 = require("../sessions/sessionMemory");
 const SessionMetricsService_1 = require("./metrics/SessionMetricsService");
+const SessionDeviceLogs_1 = require("./logcat/SessionDeviceLogs");
 const appCapability_1 = require("./session/appCapability");
 const CircuitBreaker_1 = require("../data-service/CircuitBreaker");
 const nodeWebDriverUrl_1 = require("../gateway/nodeWebDriverUrl");
@@ -904,15 +905,26 @@ let SessionLifecycleService = class SessionLifecycleService {
             if (sessionInstance instanceof LocalSession_1.LocalSession) {
                 typedi_1.Container.get(LiveSessionOwners_1.LiveSessionOwners).record(sessionId, userId);
             }
-            // A node samples its own phones for its hub, whatever its dashboard
-            // setting (the hub collects the figures). A hub and a standalone server
-            // start sampling in EventManager.onSessionStarted, once the row exists.
+            // A node samples its own phones for its hub, and records their device
+            // log unless the session turned it off, whatever its dashboard setting
+            // (the hub collects both). A hub and a standalone server start them in
+            // EventManager.onSessionStarted, once the row exists.
             if (!this.isHub(context.pluginArgs) && sessionInstance instanceof LocalSession_1.LocalSession) {
                 typedi_1.Container.get(SessionMetricsService_1.SessionMetricsService).start({
                     sessionId,
                     device: freshDevice,
                     capabilities: sessionResponse,
                 });
+                if (xenonCapabilities[XenonCapabilityManager_1.XENON_CAPABILITIES.SAVE_DEVICE_LOGS] !== false) {
+                    void typedi_1.Container.get(SessionDeviceLogs_1.SessionDeviceLogs).start({
+                        sessionId,
+                        device: freshDevice,
+                        since: allocatedAt,
+                        // An iPhone's or simulator's: the log its driver captures.
+                        driverLog: sessionInstance.deviceLog(),
+                        appUnderTest: sessionInstance.appUnderTest(),
+                    });
+                }
             }
             yield this.applyPostSessionLogic(sessionInstance, xenonCapabilities, freshDevice);
         });
@@ -1218,9 +1230,11 @@ let SessionLifecycleService = class SessionLifecycleService {
                         reason,
                     });
                     // Before the lock's "still in memory?" check, which a dashboard-off
-                    // node fails. On a node this ends the figures it holds for its hub;
-                    // on a hub it collects a node session's last ones. Idempotent.
+                    // node fails. On a node this ends the figures and device log it
+                    // holds for its hub; on a hub it collects a node session's last ones,
+                    // before onSessionStopped's failure analysis reads the log. Idempotent.
                     yield typedi_1.Container.get(SessionMetricsService_1.SessionMetricsService).stop(sessionId);
+                    yield typedi_1.Container.get(SessionDeviceLogs_1.SessionDeviceLogs).stop(sessionId);
                     yield sessionCleanupLock.acquire(sessionId, () => __awaiter(this, void 0, void 0, function* () {
                         const session = SessionManager_1.SESSION_MANAGER.getSession(sessionId);
                         if (!session) {
@@ -1338,6 +1352,7 @@ let SessionLifecycleService = class SessionLifecycleService {
                 (0, sessionMemory_1.forgetSessionMemory)(sessionId);
                 typedi_1.Container.get(TracingService_1.TracingService).endSessionSpan(sessionId, { failed: true, reason });
                 yield typedi_1.Container.get(SessionMetricsService_1.SessionMetricsService).stop(sessionId);
+                yield typedi_1.Container.get(SessionDeviceLogs_1.SessionDeviceLogs).stop(sessionId);
             }));
         });
     }

package/lib/src/services/interceptor/CertManager.js

@@ -41,13 +41,19 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.CertManager = void 0;
+exports.caSerialNumber = caSerialNumber;
+exports.subjectHashOld = subjectHashOld;
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
-const child_process_1 = require("child_process");
 const crypto_1 = require("crypto");
 const node_forge_1 = require("node-forge");
+const logger_1 = __importDefault(require("../../logger"));
+const logger = logger_1.default.scope('CertManager');
 class CertManager {
     constructor(dir) {
         this.dir = dir;
@@ -71,6 +77,12 @@ class CertManager {
             if (!fs.existsSync(this.certPath) || !fs.existsSync(this.keyPath)) {
                 this.generateCa();
             }
+            else if (!loadsInOpenSsl(this.certPath)) {
+                // Through 2.15 the serial had illegal leading zeros, so OpenSSL, Go and
+                // a phone's BoringSSL refused the CA. It never worked; make a new one.
+                logger.warn(`Replacing the interceptor CA at ${this.certPath}: it can't be loaded`);
+                this.generateCa();
+            }
             const subjectHash = this.computeAndroidHash(this.certPath);
             return { certPath: this.certPath, keyPath: this.keyPath, subjectHash };
         });
@@ -83,7 +95,7 @@ class CertManager {
         const keys = node_forge_1.pki.rsa.generateKeyPair(2048);
         const cert = node_forge_1.pki.createCertificate();
         cert.publicKey = keys.publicKey;
-        cert.serialNumber = Date.now().toString(16).padStart(16, '0');
+        cert.serialNumber = caSerialNumber();
         const now = new Date();
         cert.validity.notBefore = new Date(now.getTime() - 24 * 60 * 60 * 1000);
         cert.validity.notAfter = new Date(now.getTime() + 10 * 365 * 24 * 60 * 60 * 1000);
@@ -112,20 +124,42 @@ class CertManager {
         fs.writeFileSync(this.publicKeyPath, node_forge_1.pki.publicKeyToPem(keys.publicKey), 'utf8');
     }
     computeAndroidHash(certPath) {
-        try {
-            const out = (0, child_process_1.execSync)(`openssl x509 -inform PEM -subject_hash_old -noout -in "${certPath}"`, {
-                encoding: 'utf8',
-                timeout: 5000,
-                stdio: ['ignore', 'pipe', 'ignore'],
-            }).trim();
-            if (/^[0-9a-f]{8}$/.test(out))
-                return out;
-        }
-        catch (e) {
-            /* openssl unavailable — fall through */
-        }
-        const pem = fs.readFileSync(certPath, 'utf8');
-        return (0, crypto_1.createHash)('sha1').update(pem).digest('hex').slice(0, 8);
+        return subjectHashOld(fs.readFileSync(certPath, 'utf8'));
     }
 }
 exports.CertManager = CertManager;
+/**
+ * 16 random bytes, positive and minimally encoded as DER wants: the first
+ * byte is 0x01..0x7f (no leading zero, no sign bit), as node-forge writes the
+ * hex as the INTEGER's bytes unchanged.
+ */
+function caSerialNumber() {
+    const bytes = (0, crypto_1.randomBytes)(16);
+    bytes[0] = bytes[0] & 0x7f || 0x01;
+    return bytes.toString('hex');
+}
+function loadsInOpenSsl(certPath) {
+    try {
+        new crypto_1.X509Certificate(fs.readFileSync(certPath));
+        return true;
+    }
+    catch (_a) {
+        return false;
+    }
+}
+/**
+ * OpenSSL's `x509 -subject_hash_old`, the name Android files a system CA
+ * under: MD5 of the subject's DER as the certificate holds it, the first four
+ * bytes read little-endian.
+ */
+function subjectHashOld(pem) {
+    const der = node_forge_1.pki.pemToDer(pem).getBytes();
+    const tbs = node_forge_1.asn1.fromDer(der).value[0];
+    const fields = tbs.value;
+    const hasVersion = fields[0].tagClass === node_forge_1.asn1.Class.CONTEXT_SPECIFIC;
+    const subject = fields[hasVersion ? 5 : 4];
+    const digest = (0, crypto_1.createHash)('md5')
+        .update(Buffer.from(node_forge_1.asn1.toDer(subject).getBytes(), 'binary'))
+        .digest();
+    return digest.readUInt32LE(0).toString(16).padStart(8, '0');
+}

package/lib/src/services/logcat/NodeDeviceLogStore.js

@@ -0,0 +1,99 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.NodeDeviceLogStore = void 0;
+const typedi_1 = require("typedi");
+const nodeDeviceLogs_1 = require("./nodeDeviceLogs");
+/**
+ * On a node: each recorded session's device log lines, held in memory until
+ * the hub collects them. A node has no Session row for a session its hub
+ * created, so it can't write them to Log. The session's DeviceLogBook bounds
+ * how many there can be; what the hub has collected is dropped; an ended
+ * session is kept NODE_DEVICE_LOGS_KEEP_AFTER_END_MS, then forgotten,
+ * whether or not a hub asked.
+ */
+let NodeDeviceLogStore = class NodeDeviceLogStore {
+    constructor() {
+        this.now = () => Date.now();
+        this.entries = new Map();
+    }
+    begin(sessionId) {
+        this.prune();
+        this.entries.set(sessionId, {
+            lines: [],
+            nextSeq: 1,
+            state: 'recording',
+            claimed: false,
+            endedAt: null,
+        });
+    }
+    add(sessionId, rows) {
+        const entry = this.entries.get(sessionId);
+        if (!entry)
+            return;
+        for (const row of rows) {
+            entry.lines.push({
+                seq: entry.nextSeq++,
+                message: row.message,
+                timestamp: row.timestamp.getTime(),
+            });
+        }
+    }
+    end(sessionId) {
+        var _a;
+        const entry = this.entries.get(sessionId);
+        if (!entry)
+            return;
+        entry.state = 'ended';
+        entry.endedAt = this.now();
+        // Forgotten on time even if nothing else happens on this node: an ended
+        // session can hold the book's whole limit.
+        const timer = setTimeout(() => this.prune(), nodeDeviceLogs_1.NODE_DEVICE_LOGS_KEEP_AFTER_END_MS);
+        (_a = timer.unref) === null || _a === void 0 ? void 0 : _a.call(timer);
+    }
+    /** Drops the session at once: it isn't recorded any more. */
+    forget(sessionId) {
+        this.entries.delete(sessionId);
+    }
+    /** Whether a hub has asked about the session. */
+    claimed(sessionId) {
+        var _a;
+        return ((_a = this.entries.get(sessionId)) === null || _a === void 0 ? void 0 : _a.claimed) === true;
+    }
+    /**
+     * Up to `limit` lines after `after` (from the first, without it). Those at
+     * or before it the hub has, so they go.
+     */
+    read(sessionId, after, limit = nodeDeviceLogs_1.NODE_DEVICE_LOG_PAGE) {
+        this.prune();
+        const entry = this.entries.get(sessionId);
+        if (!entry)
+            return { state: 'off', lines: [], more: false };
+        entry.claimed = true;
+        if (after !== null) {
+            const keep = entry.lines.findIndex((l) => l.seq > after);
+            entry.lines.splice(0, keep === -1 ? entry.lines.length : keep);
+        }
+        return {
+            state: entry.state,
+            lines: entry.lines.slice(0, limit),
+            more: entry.lines.length > limit,
+        };
+    }
+    prune() {
+        const cutoff = this.now() - nodeDeviceLogs_1.NODE_DEVICE_LOGS_KEEP_AFTER_END_MS;
+        for (const [id, entry] of this.entries) {
+            if (entry.endedAt !== null && entry.endedAt <= cutoff)
+                this.entries.delete(id);
+        }
+    }
+};
+exports.NodeDeviceLogStore = NodeDeviceLogStore;
+exports.NodeDeviceLogStore = NodeDeviceLogStore = __decorate([
+    (0, typedi_1.Service)()
+], NodeDeviceLogStore);

package/lib/src/services/logcat/NodeDeviceLogsCollector.js

@@ -0,0 +1,164 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.NodeDeviceLogsCollector = exports.OUTAGE_AFTER_FAILURES = exports.MAX_PAGES_PER_ROUND = exports.COLLECT_INTERVAL_MS = void 0;
+/** How often the hub asks a node for a session's new lines. */
+exports.COLLECT_INTERVAL_MS = 10000;
+/** Answers asked for in one round while the node says it holds more. */
+exports.MAX_PAGES_PER_ROUND = 10;
+/**
+ * Failed asks in a row after which the node is taken to be gone, and not
+ * asked once more when the session ends. One failure is a busy node, often at
+ * the end of a test, and the last ask brings the test's last lines.
+ */
+exports.OUTAGE_AFTER_FAILURES = 2;
+const rowOf = (l) => ({
+    message: l.message,
+    timestamp: new Date(l.timestamp),
+});
+/**
+ * On a hub: a session on a node's phone, its device log lines collected by
+ * asking the node (GET /node/sessions/:id/device-logs) at once, then every
+ * COLLECT_INTERVAL_MS, instead of reading the phone. Asking at once tells the
+ * node a hub wants the session's lines (it stops recording a session nobody
+ * asks about). While the node holds more than one answer, it is asked again at
+ * once. Asks never overlap. An unreachable node is asked again; whether the
+ * session is alive is the heartbeat's call. When the session ends it asks
+ * once more, for the last lines, unless the node stopped answering
+ * (OUTAGE_AFTER_FAILURES).
+ */
+class NodeDeviceLogsCollector {
+    constructor(o) {
+        var _a;
+        this.o = o;
+        /** The newest line's `seq` the hub has. */
+        this.after = null;
+        this.done = false;
+        this.asking = null;
+        this.outage = false;
+        /** Failed asks in a row. */
+        this.failures = 0;
+        this.storedTail = (_a = o.storedTail) !== null && _a !== void 0 ? _a : [];
+    }
+    start() {
+        var _a, _b, _c;
+        const origin = this.o.source.nodeOrigin();
+        if (!origin || !this.o.support.shouldAsk(origin)) {
+            this.finish();
+            return;
+        }
+        this.timer = setInterval(() => void this.tick(), (_a = this.o.intervalMs) !== null && _a !== void 0 ? _a : exports.COLLECT_INTERVAL_MS);
+        (_c = (_b = this.timer).unref) === null || _c === void 0 ? void 0 : _c.call(_b);
+        void this.tick();
+    }
+    stop() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this.timer)
+                clearInterval(this.timer);
+            if (this.asking)
+                yield this.asking;
+            // No last ask of a node that stopped answering: ending each of a dark
+            // node's sessions would wait out another timeout for nothing.
+            if (!this.done && this.failures < exports.OUTAGE_AFTER_FAILURES)
+                yield this.round();
+            this.done = true;
+        });
+    }
+    tick() {
+        if (this.done || this.asking)
+            return undefined;
+        this.asking = this.round().finally(() => {
+            this.asking = null;
+        });
+        return this.asking;
+    }
+    /** Asks until the node has nothing more to give, at most MAX_PAGES_PER_ROUND times. */
+    round() {
+        return __awaiter(this, void 0, void 0, function* () {
+            for (let page = 0; page < exports.MAX_PAGES_PER_ROUND; page++) {
+                if (!(yield this.ask()))
+                    return;
+            }
+        });
+    }
+    /** One ask; true when the node holds more, to ask for at once. */
+    ask() {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a, _b;
+            const reply = yield this.o.source.nodeDeviceLogs(this.after);
+            switch (reply.kind) {
+                case 'answer': {
+                    this.failures = 0;
+                    if (this.outage) {
+                        this.outage = false;
+                        this.o.logger.info(`${this.o.source.nodeOrigin()} answers again`);
+                    }
+                    const after = this.after;
+                    const lines = reply.answer.lines.filter((l) => after === null || l.seq > after);
+                    for (const l of lines)
+                        this.after = Math.max((_a = this.after) !== null && _a !== void 0 ? _a : 0, l.seq);
+                    const fresh = this.pastStored(lines);
+                    if (fresh.length > 0)
+                        yield this.o.onLines(fresh.map(rowOf));
+                    if (reply.answer.more && lines.length > 0)
+                        return true;
+                    if (reply.answer.state !== 'recording')
+                        this.finish();
+                    return false;
+                }
+                case 'unsupported':
+                    this.o.support.unsupported((_b = this.o.source.nodeOrigin()) !== null && _b !== void 0 ? _b : '', reply.status);
+                    this.finish();
+                    return false;
+                case 'refused':
+                    this.finish();
+                    return false;
+                case 'unavailable':
+                    this.failures += 1;
+                    if (!this.outage) {
+                        this.outage = true;
+                        this.o.logger.warn(`Can't collect device logs from ${this.o.source.nodeOrigin()}: ${reply.reason}. Asking again.`);
+                    }
+                    return false;
+            }
+        });
+    }
+    /**
+     * After a hub restart the node still holds the last page the hub received
+     * (it drops lines only when the next ask names them), so the first answer
+     * can repeat lines the hub stored. They end where the answer's lines match
+     * the stored tail, line for line back from there: logcat often prints the
+     * same line twice in one millisecond, so one line alone can't say which
+     * copy the hub has.
+     */
+    pastStored(lines) {
+        const tail = this.storedTail;
+        if (tail.length === 0)
+            return lines;
+        this.storedTail = [];
+        const same = (l, t) => l.message === t.message && l.timestamp === t.timestamp;
+        for (let end = lines.length - 1; end >= 0; end -= 1) {
+            const span = Math.min(tail.length, end + 1);
+            let k = 0;
+            while (k < span && same(lines[end - k], tail[tail.length - 1 - k]))
+                k += 1;
+            if (k === span)
+                return lines.slice(end + 1);
+        }
+        return lines;
+    }
+    finish() {
+        this.done = true;
+        if (this.timer)
+            clearInterval(this.timer);
+    }
+}
+exports.NodeDeviceLogsCollector = NodeDeviceLogsCollector;