@leera.io/qa-runner 1.0.0 → 1.0.2

package/dist/src/cli/commands/doctor.js

@@ -3,16 +3,18 @@ import { formatDoctor, runDoctor } from '../../doctor.js';
 import { isPlatform } from '../../devices/types.js';
 import { createRegistry } from '../../drivers/registry.js';
 import { ExitCode } from '../../errors.js';
+import { envSecrets, toHealthReport } from '../../health.js';
 import { homePaths } from '../../home.js';
 import { printJson, usageError } from '../output.js';
 import { installBrowsers } from './setup.js';
-const usage = `Usage: leera-qa-runner doctor [--json] [--fix] [--platform web|android|ios|electron|tauri]
+const usage = `Usage: leera-qa-runner doctor [--json] [--health] [--fix] [--platform web|android|ios|electron|tauri]
 
 Checks that this machine can run jobs: Node.js, the runner home, the server URL and
 token, and each platform's tools. Exits 4 when something needs fixing.
 
 Options:
   --json              Print the report as JSON
+  --health            Print the report the runner sends to the server (JSON)
   --fix               Fix what can be fixed safely (permissions, installing Chromium)
   --platform NAME     Only check one platform
 `;
@@ -25,6 +27,7 @@ export const command = {
             strict: true,
             options: {
                 json: { type: 'boolean' },
+                health: { type: 'boolean' },
                 fix: { type: 'boolean' },
                 platform: { type: 'string' },
                 offline: { type: 'boolean' },
@@ -44,11 +47,14 @@ export const command = {
             paths,
             registry,
             fix: values.fix ?? false,
-            offline: values.offline ?? false,
+            // `--health` prints exactly what `start` sends, and that report never uses the network.
+            offline: (values.offline ?? false) || (values.health ?? false),
             ...(platform ? { platform } : {}),
             installBrowsers: async () => (await installBrowsers(io, paths)) === 0,
         });
-        if (values.json)
+        if (values.health)
+            printJson(io, toHealthReport(report, { secrets: envSecrets(io.env) }));
+        else if (values.json)
             printJson(io, report);
         else
             io.stdout(formatDoctor(report));

package/dist/src/client.js

@@ -49,8 +49,11 @@ export class RunnerClient {
             return { result: 'unexpected', status: error.status, message: error.message };
         }
     }
-    async heartbeat(jobId) {
-        const response = await this.json('/api/v1/qa/runner/heartbeat', { job_id: jobId });
+    async heartbeat(jobId, progress) {
+        const response = await this.json('/api/v1/qa/runner/heartbeat', {
+            job_id: jobId,
+            ...(progress ? { progress } : {}),
+        });
         return response?.continue ?? false;
     }
     async uploadEvidence(jobId, name, bytes, contentType = 'image/png') {

package/dist/src/executor.js

@@ -76,9 +76,9 @@ export async function createDriver(registry, ctx, shared) {
     return factory.create(ctx, shared);
 }
 /** Runs a claimed job's script on the driver its platform maps to (web, android, …). */
-export async function executeClaimedJob(registry, ctx, shared, shouldStop) {
+export async function executeClaimedJob(registry, ctx, shared, shouldStop, onProgress) {
     const driver = await createDriver(registry, ctx, shared);
-    return executeSteps(driver, ctx.job.automation, { jobTimeoutMs: ctx.timeouts.jobMs }, shouldStop, ctx.redact);
+    return executeSteps(driver, ctx.job.automation, { jobTimeoutMs: ctx.timeouts.jobMs, ...(onProgress ? { onProgress } : {}) }, shouldStop, ctx.redact);
 }
 /**
  * Runs a script's steps in index order on a driver: stops at the first failed or
@@ -93,6 +93,23 @@ export async function executeSteps(driver, script, options, shouldStop, redact)
     let videoPath;
     try {
         const ordered = [...script.steps].sort((a, b) => a.index - b.index);
+        const report = (progress) => {
+            if (!options.onProgress)
+                return;
+            const passed = steps.filter((outcome) => outcome.result === 'passed').length;
+            try {
+                options.onProgress({
+                    ...progress,
+                    step_count: ordered.length,
+                    steps_passed: passed,
+                    steps_failed: steps.length - passed,
+                });
+            }
+            catch {
+                // Progress is a courtesy to whoever is watching; it never fails the job.
+            }
+        };
+        report({ phase: 'launching' });
         try {
             await driver.launch();
         }
@@ -104,8 +121,9 @@ export async function executeSteps(driver, script, options, shouldStop, redact)
             const outcome = { index: first.index, result: 'blocked', note: redact(errorMessage(error)), screenshots: [] };
             return { steps: [outcome], elapsedSecs: Math.max(1, Math.round((Date.now() - started) / 1000)), stopped: false };
         }
-        for (const step of ordered) {
+        for (const [position, step] of ordered.entries()) {
             const outcome = { index: step.index, result: 'passed', screenshots: [] };
+            report({ phase: 'running', step_index: step.index, step_number: position + 1 });
             driver.beginStep(step.index);
             try {
                 for (const action of step.actions) {

package/dist/src/health.js

@@ -0,0 +1,309 @@
+import { TOKEN_PREFIX } from './config.js';
+import { runDoctor } from './doctor.js';
+import { isPlatform } from './devices/types.js';
+import { messageOf } from './errors.js';
+import { redactor } from './template.js';
+/** Contracts §O limits. The server refuses a report that exceeds any of them. */
+export const MAX_HEALTH_CHECKS = 60;
+export const MAX_HEALTH_BYTES = 16 * 1024;
+export const MAX_HEALTH_DETAIL = 300;
+export const MAX_HEALTH_LABEL = 80;
+export const MAX_HEALTH_FIX_COMMAND = 200;
+export const MAX_HEALTH_FIX_URL = 300;
+const HEALTH_ID = /^[a-z][a-z0-9._-]{0,60}$/;
+/**
+ * Checks that depend on the network. They are left out of the report: a five-second outage
+ * or a proxy hiccup would otherwise show up on the runners page as a setup problem the
+ * person cannot fix, and the page already knows whether the runner is reachable (it is
+ * talking to the server whenever a report arrives at all).
+ */
+export const NETWORK_CHECK_IDS = new Set(['server']);
+/** `doctor` titles the bare platform checks with the platform name; the UI wants a label. */
+const PLATFORM_LABELS = {
+    web: 'Web jobs',
+    android: 'Android jobs',
+    ios: 'iOS jobs',
+    electron: 'Electron jobs',
+    tauri: 'Tauri jobs',
+    macos: 'macOS app jobs',
+    windows: 'Windows app jobs',
+    linux: 'Linux app jobs',
+};
+/** First words that make a fix line a command someone can paste, rather than prose. */
+const COMMAND_WORDS = new Set([
+    'leera-qa-runner',
+    'npm',
+    'npx',
+    'node',
+    'sdkmanager',
+    'avdmanager',
+    'adb',
+    'brew',
+    'winget',
+    'choco',
+    'sudo',
+    'apt',
+    'apt-get',
+    'cargo',
+    'pip',
+    'pip3',
+    'gem',
+    'xcode-select',
+    'xcrun',
+    'xcodebuild',
+    'softwareupdate',
+    'automationmodetool',
+    'sh',
+    'bash',
+    'zsh',
+    'powershell',
+    'msiexec',
+    'defaults',
+    'docker',
+    'git',
+    'chmod',
+]);
+const TOKEN_PATTERN = new RegExp(`${TOKEN_PREFIX}[A-Za-z0-9_.-]+`, 'g');
+/** `scheme://user:password@host` — a URL somebody configured with credentials in it. */
+const URL_USERINFO = /([a-z][a-z0-9+.-]*:\/\/)[^\s/@]+@/gi;
+export function truncate(text, max) {
+    const trimmed = text.trim();
+    return trimmed.length <= max ? trimmed : `${trimmed.slice(0, max - 1).trimEnd()}…`;
+}
+/** The token sources a report could otherwise quote back. */
+export function envSecrets(env) {
+    return [env.RUNNER_TOKEN, env.LEERA_RUNNER_TOKEN, env.RUNNER_WEBHOOK_SECRET]
+        .map((value) => value?.trim())
+        .filter((value) => !!value);
+}
+/**
+ * Removes every known secret, anything shaped like a runner token, and the user:password
+ * part of a URL. Paths are deliberately kept (contracts §O).
+ */
+export function scrubber(secrets) {
+    const redact = redactor(secrets);
+    return (text) => redact(text).replace(TOKEN_PATTERN, `${TOKEN_PREFIX}[redacted]`).replace(URL_USERINFO, '$1[redacted]@');
+}
+/** Makes an id the server accepts (`^[a-z][a-z0-9._-]{0,60}$`) without changing ids that already do. */
+export function healthCheckId(id) {
+    if (HEALTH_ID.test(id))
+        return id;
+    const cleaned = id
+        .toLowerCase()
+        .replace(/[^a-z0-9._-]+/g, '-')
+        .replace(/^[^a-z]+/, '')
+        .slice(0, 61);
+    return cleaned.length > 0 ? cleaned : 'check';
+}
+function platformOf(id) {
+    const head = id.split('.')[0] ?? '';
+    return isPlatform(head) ? head : undefined;
+}
+/**
+ * Turns a `doctor` fix line into `{command, url}`. A line inside backticks is the command
+ * (`… then run \`leera-qa-runner setup ios\``); a bare line that starts with a known tool is
+ * itself the command; anything else is advice and stays in `detail` so nothing is lost.
+ */
+export function parseFix(text) {
+    const line = text.trim();
+    if (!line)
+        return {};
+    const fix = {};
+    const quoted = /`([^`]+)`/.exec(line)?.[1]?.trim();
+    let command = quoted;
+    let prose = quoted ? undefined : line;
+    if (!command) {
+        const head = line.split(/\s+/)[0] ?? '';
+        if (COMMAND_WORDS.has(head)) {
+            // "leera-qa-runner config set tauri.macos_plugin true (only when …)" — drop the aside.
+            command = (line.split(' (')[0] ?? line).trim();
+            prose = undefined;
+        }
+    }
+    if (command && command.length <= MAX_HEALTH_FIX_COMMAND)
+        fix.command = command;
+    else if (command)
+        prose = line;
+    const url = /https?:\/\/[^\s)`'"<>]+/.exec(line)?.[0];
+    if (url && url.length <= MAX_HEALTH_FIX_URL)
+        fix.url = url;
+    return {
+        ...(fix.command || fix.url ? { fix } : {}),
+        ...(prose ? { prose } : {}),
+    };
+}
+const WORST = { ok: 0, warn: 1, fail: 2 };
+export function deriveStatus(checks) {
+    if (checks.some((check) => check.status === 'fail'))
+        return 'action_needed';
+    if (checks.some((check) => check.status === 'warn'))
+        return 'warning';
+    return 'ok';
+}
+function toHealthCheck(check, scrub) {
+    const id = healthCheckId(check.id);
+    const platform = platformOf(id);
+    const parsed = check.fix && check.status !== 'ok' ? parseFix(scrub(check.fix)) : {};
+    const detail = [check.detail ? scrub(check.detail) : '', parsed.prose ? `fix: ${parsed.prose}` : '']
+        .filter((part) => part.length > 0)
+        .join(' — ');
+    return {
+        id,
+        label: truncate(PLATFORM_LABELS[check.id] ?? check.title, MAX_HEALTH_LABEL),
+        ...(platform ? { platform } : {}),
+        status: check.status,
+        ...(detail ? { detail: truncate(detail, MAX_HEALTH_DETAIL) } : {}),
+        ...(parsed.fix ? { fix: parsed.fix } : {}),
+    };
+}
+/** Keeps one check per id: the worst status wins, so a later failure is never hidden. */
+function dedupe(checks) {
+    const byId = new Map();
+    for (const check of checks) {
+        const seen = byId.get(check.id);
+        if (!seen || WORST[check.status] > WORST[seen.status])
+            byId.set(check.id, check);
+    }
+    return [...byId.values()];
+}
+function capChecks(checks) {
+    if (checks.length <= MAX_HEALTH_CHECKS)
+        return checks;
+    const problems = checks.filter((check) => check.status !== 'ok');
+    const rest = checks.filter((check) => check.status === 'ok');
+    return [...problems, ...rest].slice(0, MAX_HEALTH_CHECKS);
+}
+export function serializedBytes(report) {
+    return Buffer.byteLength(JSON.stringify(report), 'utf8');
+}
+function withoutDetail(check) {
+    return {
+        id: check.id,
+        label: check.label,
+        ...(check.platform ? { platform: check.platform } : {}),
+        status: check.status,
+        ...(check.fix ? { fix: check.fix } : {}),
+    };
+}
+/** Ways to shrink an oversized report, tried in order; problems and their fixes go last. */
+const SHRINK = [
+    (checks) => checks.map((check) => (check.status === 'ok' ? withoutDetail(check) : check)),
+    (checks) => checks.filter((check) => check.status !== 'ok'),
+    (checks) => checks.map((check) => (check.detail ? { ...check, detail: truncate(check.detail, 120) } : check)),
+    (checks) => checks.map(withoutDetail),
+];
+function fit(report) {
+    let current = report;
+    for (const shrink of SHRINK) {
+        if (serializedBytes(current) <= MAX_HEALTH_BYTES)
+            return current;
+        current = { ...current, checks: shrink(current.checks) };
+    }
+    while (serializedBytes(current) > MAX_HEALTH_BYTES && current.checks.length > 0) {
+        current = { ...current, checks: current.checks.slice(0, Math.floor(current.checks.length / 2)) };
+    }
+    return current;
+}
+/** Builds the report the server stores from a `doctor` run (contracts §O). */
+export function toHealthReport(report, options = {}) {
+    const scrub = scrubber(options.secrets ?? []);
+    const checks = dedupe(report.checks.filter((check) => !NETWORK_CHECK_IDS.has(check.id)).map((check) => toHealthCheck(check, scrub)));
+    // The summary covers every check, including any the size limits drop below.
+    return fit({ generated_at: (options.now ?? (() => new Date()))().toISOString(), status: deriveStatus(checks), checks: capChecks(checks) });
+}
+/**
+ * Runs the `doctor` checks and turns them into a health report. Never fixes anything and
+ * never touches the network (`offline`), so it is safe to run while jobs are in flight.
+ */
+export async function healthReport(options) {
+    const report = await runDoctor({ env: options.env, paths: options.paths, registry: options.registry, fix: false, offline: true });
+    return toHealthReport(report, {
+        secrets: [...envSecrets(options.env), ...(options.secrets ?? [])],
+        ...(options.now ? { now: options.now } : {}),
+    });
+}
+/** How often a report may be recomputed, whatever asks for it (contracts §O). */
+export const HEALTH_MIN_INTERVAL_MS = 60_000;
+/** How long to wait for the checks before giving up on this round and keeping the last report. */
+export const HEALTH_TIMEOUT_MS = 120_000;
+function defaultTimer(fn, ms) {
+    const timer = setTimeout(fn, ms);
+    timer.unref?.();
+    return { cancel: () => clearTimeout(timer) };
+}
+/**
+ * Keeps the last health report and recomputes it in the background. Registration never waits
+ * for it: the first register carries no report, the next one does. At most one computation
+ * runs at a time (the checks shell out to adb, xcrun and java) and at most one a minute.
+ */
+export class HealthMonitor {
+    deps;
+    report = null;
+    running = null;
+    lastStartedAt = null;
+    constructor(deps) {
+        this.deps = deps;
+    }
+    /** The last computed report, or null while none has finished yet. */
+    current() {
+        return this.report;
+    }
+    /** Resolves when a computation in flight has been stored or timed out (tests). */
+    settled() {
+        return this.running ?? Promise.resolve();
+    }
+    /**
+     * Starts a computation unless one is already running or one started less than a minute ago.
+     * The returned promise is for tests; callers fire and forget.
+     */
+    refresh(trigger) {
+        const now = (this.deps.now ?? Date.now)();
+        if (this.running)
+            return this.running;
+        const minInterval = this.deps.minIntervalMs ?? HEALTH_MIN_INTERVAL_MS;
+        if (this.lastStartedAt !== null && now - this.lastStartedAt < minInterval)
+            return Promise.resolve();
+        this.lastStartedAt = now;
+        let raw;
+        try {
+            raw = this.deps.compute();
+        }
+        catch (error) {
+            this.deps.log.warn('health checks failed', { trigger, error: messageOf(error) });
+            return Promise.resolve();
+        }
+        // The lock is held until the checks really finish, even if we stopped waiting for them,
+        // so a slow adb or xcrun is never started a second time alongside itself.
+        const lock = raw.then(() => undefined, () => undefined);
+        this.running = lock;
+        void lock.then(() => {
+            if (this.running === lock)
+                this.running = null;
+        });
+        return this.store(raw, trigger);
+    }
+    async store(raw, trigger) {
+        const timeoutMs = this.deps.timeoutMs ?? HEALTH_TIMEOUT_MS;
+        const timer = this.deps.setTimer ?? defaultTimer;
+        const handles = [];
+        const expired = new Promise((resolve) => {
+            handles.push(timer(() => resolve('timeout'), timeoutMs));
+        });
+        try {
+            const outcome = await Promise.race([raw, expired]);
+            if (outcome === 'timeout') {
+                this.deps.log.warn('health checks are taking too long; keeping the previous report', { trigger, timeout_ms: timeoutMs });
+                return;
+            }
+            this.report = outcome;
+            this.deps.log.debug('health report updated', { trigger, status: outcome.status, checks: outcome.checks.length });
+        }
+        catch (error) {
+            this.deps.log.warn('health checks failed', { trigger, error: messageOf(error) });
+        }
+        finally {
+            for (const handle of handles)
+                handle.cancel();
+        }
+    }
+}

package/dist/src/progress.js

@@ -0,0 +1,32 @@
+/**
+ * Sends progress one request at a time, keeping only the newest update while one
+ * is in flight: a script with fast steps must not queue a request per step, and
+ * the run page only needs where the job is now, not every place it has been.
+ */
+export function progressReporter(send) {
+    let pending;
+    let inFlight = false;
+    let stopped = false;
+    const pump = async () => {
+        inFlight = true;
+        while (pending && !stopped) {
+            const next = pending;
+            pending = undefined;
+            await send(next).catch(() => undefined);
+        }
+        inFlight = false;
+    };
+    return {
+        report(progress) {
+            if (stopped)
+                return;
+            pending = progress;
+            if (!inFlight)
+                void pump();
+        },
+        stop() {
+            stopped = true;
+            pending = undefined;
+        },
+    };
+}

package/dist/src/runner.js

@@ -9,7 +9,9 @@ import { createRegistry } from './drivers/registry.js';
 import { BrowserPool } from './drivers/web-playwright.js';
 import { ExitCode, ExitError, messageOf } from './errors.js';
 import { createDriver, executeClaimedJob, jobContext, sessionContext } from './executor.js';
+import { HealthMonitor, healthReport } from './health.js';
 import { homePaths, readJsonFile, writePrivateFile } from './home.js';
+import { progressReporter } from './progress.js';
 import { redactor } from './template.js';
 import { runSession } from './session/loop.js';
 import { isSessionClaim } from './types.js';
@@ -170,6 +172,7 @@ export class Runner {
         const platform = job.platform ?? 'web';
         const lease = isPlatform(platform) ? devices.lease(platform, job.device?.id, { holder: `job:${job.job.id}` }) : null;
         if (!lease) {
+            void this.deps.health?.refresh('job_failed');
             await client
                 .complete({ job_id: job.job.id, error: `this runner has no free ${platform} device for the job` })
                 .catch((error) => log.error('could not report the failure', { job_id: job.job.id, error: messageOf(error) }));
@@ -188,6 +191,7 @@ export class Runner {
             ? devices.lease(session.platform, session.device?.id, { holder: `session:${session.id}` })
             : null;
         if (!lease) {
+            void this.deps.health?.refresh('job_failed');
             await client
                 .endSession(session.id, `this runner has no free ${session.platform} device for the session`)
                 .catch((error) => log.error('could not end the session', { session_id: session.id, error: messageOf(error) }));
@@ -255,7 +259,10 @@ export class Runner {
         let delay = 2_000;
         while (!this.stopping) {
             try {
-                const registration = await client.register(body);
+                // The report is whatever the background checks last produced: the first registration
+                // carries none, later ones do (contracts §O). Registration never waits for it.
+                const health = this.deps.health?.current();
+                const registration = await client.register({ ...body, ...(health ? { health } : {}) });
                 if (this.registration?.runner_id !== registration.runner_id) {
                     log.info('registered', {
                         runner_id: registration.runner_id,
@@ -266,6 +273,8 @@ export class Runner {
                 this.noteLatestVersion(registration.latest_version, body.version);
                 this.registration = registration;
                 this.writeState();
+                // Recompute for the next registration (device change or the 5-minute refresh).
+                void this.deps.health?.refresh('register');
                 return registration;
             }
             catch (error) {
@@ -318,6 +327,16 @@ export class Runner {
             })
                 .catch((error) => log.warn('heartbeat failed', { job_id: jobId, error: redact(messageOf(error)) }));
         }, job.job.heartbeat_seconds * 1_000);
+        const progress = progressReporter(async (update) => {
+            try {
+                if (!(await client.heartbeat(jobId, update)))
+                    cancelled = true;
+            }
+            catch (error) {
+                // An older server refuses the extra field or the network blipped; the timed heartbeat keeps the lease.
+                log.debug('progress report failed', { job_id: jobId, error: redact(messageOf(error)) });
+            }
+        });
         let videoPath;
         try {
             const ctx = jobContext(job, {
@@ -331,7 +350,7 @@ export class Runner {
                 redact,
                 log,
             });
-            const report = await executeClaimedJob(registry, ctx, shared, () => cancelled || this.stopping);
+            const report = await executeClaimedJob(registry, ctx, shared, () => cancelled || this.stopping, progress.report);
             videoPath = report.videoPath;
             if (cancelled) {
                 log.info('job cancelled', { job_id: jobId });
@@ -343,6 +362,13 @@ export class Runner {
                 this.jobFinished(job, 'error', 'the runner shut down before the job finished', [], report.elapsedSecs);
                 return;
             }
+            const passedSteps = report.steps.filter((step) => step.result === 'passed').length;
+            progress.report({
+                phase: 'uploading',
+                step_count: Math.max(job.item.steps.length, report.steps.length),
+                steps_passed: passedSteps,
+                steps_failed: report.steps.length - passedSteps,
+            });
             const stepResults = [];
             for (const step of report.steps) {
                 const attachments = [];
@@ -398,6 +424,9 @@ export class Runner {
         catch (error) {
             const reason = redact(messageOf(error));
             log.error('job could not run', { job_id: jobId, error: reason });
+            // A job that could not start often means the machine's setup changed (a driver, a
+            // device, a permission); refresh the report so the runners page says what broke.
+            void this.deps.health?.refresh('job_failed');
             await client
                 .complete({ job_id: jobId, error: reason })
                 .catch((e) => log.error('could not report the failure', { job_id: jobId, error: redact(messageOf(e)) }));
@@ -405,6 +434,7 @@ export class Runner {
         }
         finally {
             clearInterval(heartbeat);
+            progress.stop();
             this.running.delete(jobId);
             // Each job records into its own temporary directory.
             if (videoPath)
@@ -491,6 +521,10 @@ export async function createRunner(options) {
         host,
         shared,
         log: options.log,
+        health: new HealthMonitor({
+            compute: () => healthReport({ env, paths, registry, secrets: [config.token] }),
+            log: options.log,
+        }),
         ...(options.useHome === false ? {} : { paths }),
         ...(options.writeState ? { statePath: paths.state } : {}),
     });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leera.io/qa-runner",
-  "version": "1.0.0",
+  "version": "1.0.2",
   "description": "Runs recorded QA test scripts on your own machines and reports results back to your workspace.",
   "license": "SEE LICENSE IN LICENSE",
   "type": "module",