@leera.io/qa-runner 1.1.0 → 1.4.0

package/dist/src/files/store.js

@@ -0,0 +1,181 @@
+import { randomFillSync } from 'node:crypto';
+import { closeSync, constants, copyFileSync, linkSync, mkdirSync, mkdtempSync, openSync, rmSync, statSync, truncateSync, writeSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { basename, join } from 'node:path';
+import { BuildCache } from '../builds/cache.js';
+import { obtainBuild } from '../builds/download.js';
+/** The largest generated file the runner writes: the server's own limit. */
+export const MAX_GENERATED_BYTES = 2 * 1024 * 1024 * 1024;
+/** Test files stay small; 2 GB of them are kept between jobs by default. */
+export const DEFAULT_FILE_CACHE_BYTES = 2 * 1024 * 1024 * 1024;
+const WRITE_CHUNK = 1024 * 1024;
+export class TestFileError extends Error {
+}
+/** `~/.leera-qa-runner/cache/test-files`. */
+export function testFileCacheDir(paths) {
+    return join(paths.root, 'cache', 'test-files');
+}
+const caches = new Map();
+/** One cache object per directory per process, so pins are shared between jobs. */
+export function testFileCacheFor(paths, maxBytes = DEFAULT_FILE_CACHE_BYTES) {
+    const dir = testFileCacheDir(paths);
+    let cache = caches.get(dir);
+    if (!cache) {
+        cache = new BuildCache(dir, maxBytes);
+        caches.set(dir, cache);
+    }
+    return cache;
+}
+/**
+ * The name a file is written under: the server checked it is a plain file name every OS accepts;
+ * anything that still names a folder or a device is refused here too, so a job can never write
+ * outside its own folder.
+ */
+export function checkedFileName(fileName) {
+    const name = typeof fileName === 'string' ? fileName : '';
+    const plain = name.length > 0 && name.length <= 255 && basename(name) === name && !name.includes('\\') && !name.includes('/');
+    const control = [...name].some((ch) => ch.charCodeAt(0) < 0x20);
+    if (!plain || control || name === '.' || name === '..' || /[<>:"|?*]/.test(name)) {
+        throw new TestFileError(`the test file name '${name}' cannot be written on this runner`);
+    }
+    return name;
+}
+/**
+ * A job's test files on this machine: `<job folder>/<name>/<file name>`, so each keeps the name
+ * the app should see even when two share it. Stored files come through the digest-checked cache;
+ * generated files are written here. Disposing removes the folder.
+ */
+export class JobFiles {
+    dir;
+    files;
+    releases;
+    picks = 0;
+    constructor(dir, files, releases) {
+        this.dir = dir;
+        this.files = files;
+        this.releases = releases;
+    }
+    /** No files: a job whose script hands none over. */
+    static empty() {
+        return new JobFiles('', {}, []);
+    }
+    static async prepare(options) {
+        const entries = Object.entries(options.files ?? {});
+        if (entries.length === 0)
+            return JobFiles.empty();
+        const dir = mkdtempSync(join(options.root ?? tmpdir(), 'qa-files-'));
+        const files = {};
+        const releases = [];
+        const store = new JobFiles(dir, files, releases);
+        try {
+            for (const [name, file] of entries) {
+                if (!/^[a-z][a-z0-9_]{0,39}$/.test(name))
+                    throw new TestFileError(`'${name}' is not a test file name`);
+                const fileName = checkedFileName(file.file_name);
+                const folder = join(dir, name);
+                mkdirSync(folder, { recursive: true, mode: 0o700 });
+                const target = join(folder, fileName);
+                if (file.kind === 'generated') {
+                    writeGenerated(target, file);
+                }
+                else if (file.kind === 'stored') {
+                    const obtained = await obtainBuild({
+                        cache: options.cache,
+                        build: {
+                            id: file.id,
+                            version_name: null,
+                            build_number: null,
+                            file_name: fileName,
+                            sha256: file.sha256 ?? '',
+                            size_bytes: file.size_bytes,
+                        },
+                        getUrl: () => options.downloadUrl(file.id),
+                        what: `test file ${name}`,
+                        ...(options.fetchImpl ? { fetchImpl: options.fetchImpl } : {}),
+                    });
+                    releases.push(obtained.release);
+                    copyFileSync(obtained.path, target, constants.COPYFILE_FICLONE);
+                }
+                else {
+                    throw new TestFileError(`test file ${name} is of a kind this runner does not know (${file.kind}); update the runner`);
+                }
+                files[name] = { file_name: fileName, path: target };
+            }
+            options.cache.prune();
+            return store;
+        }
+        catch (error) {
+            await store.dispose();
+            throw error;
+        }
+    }
+    /**
+     * A new folder holding exactly these files under their own names, for an Open dialog that picks
+     * everything in one folder. Links where the filesystem allows, copies otherwise.
+     */
+    folderWith(paths) {
+        if (!this.dir)
+            throw new TestFileError('the job has no test files');
+        this.picks += 1;
+        const folder = join(this.dir, `.pick-${this.picks}`);
+        mkdirSync(folder, { recursive: true, mode: 0o700 });
+        const seen = new Set();
+        for (const path of paths) {
+            const name = basename(path);
+            if (seen.has(name.toLowerCase())) {
+                throw new TestFileError(`two of the files are named ${name}; an Open dialog cannot pick both`);
+            }
+            seen.add(name.toLowerCase());
+            const target = join(folder, name);
+            try {
+                linkSync(path, target);
+            }
+            catch {
+                copyFileSync(path, target, constants.COPYFILE_FICLONE);
+            }
+        }
+        return folder;
+    }
+    async dispose() {
+        for (const release of this.releases.splice(0))
+            release();
+        if (this.dir)
+            rmSync(this.dir, { recursive: true, force: true });
+    }
+}
+/** Writes a generated file: zeros (sparse where the filesystem allows) or random bytes. */
+export function writeGenerated(path, file) {
+    const size = file.size_bytes;
+    if (!Number.isSafeInteger(size) || size < 0 || size > MAX_GENERATED_BYTES) {
+        throw new TestFileError(`test file ${file.name} has a size this runner will not generate (${size} bytes)`);
+    }
+    const fill = file.fill ?? 'zeros';
+    if (fill === 'zeros') {
+        const fd = openSync(path, 'w', 0o600);
+        closeSync(fd);
+        truncateSync(path, size);
+    }
+    else if (fill === 'random') {
+        const fd = openSync(path, 'w', 0o600);
+        try {
+            const chunk = Buffer.allocUnsafe(Math.min(WRITE_CHUNK, Math.max(size, 1)));
+            let left = size;
+            while (left > 0) {
+                const length = Math.min(left, chunk.length);
+                randomFillSync(chunk, 0, length);
+                let offset = 0;
+                while (offset < length)
+                    offset += writeSync(fd, chunk, offset, length - offset);
+                left -= length;
+            }
+        }
+        finally {
+            closeSync(fd);
+        }
+    }
+    else {
+        throw new TestFileError(`test file ${file.name} has a fill this runner does not know (${fill}); update the runner`);
+    }
+    if (statSync(path).size !== size)
+        throw new TestFileError(`test file ${file.name} was not written in full`);
+}

package/dist/src/home.js

@@ -16,6 +16,7 @@ export function homePaths(env = process.env) {
         browsers: join(root, 'browsers'),
         jobs: join(root, 'jobs'),
         versions: join(root, 'versions'),
+        ffmpeg: join(root, 'ffmpeg'),
     };
 }
 const isWindows = process.platform === 'win32';

package/dist/src/runner.js

@@ -1,5 +1,6 @@
 import { readFile, rm, stat } from 'node:fs/promises';
-import { dirname } from 'node:path';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
 import { buildCapabilities, detect, hostInfo } from './capabilities.js';
 import { ApiError, RunnerClient } from './client.js';
 import { loadFileConfig, resolveConfig } from './config.js';
@@ -9,10 +10,11 @@ import { createRegistry, defaultFactories } from './drivers/registry.js';
 import { BrowserPool } from './drivers/web-playwright.js';
 import { ExitCode, ExitError, messageOf } from './errors.js';
 import { createDriver, executeClaimedJob, jobContext, sessionContext } from './executor.js';
+import { JobFiles, testFileCacheFor } from './files/store.js';
 import { HealthMonitor, healthReport } from './health.js';
 import { homePaths, readJsonFile, writePrivateFile } from './home.js';
 import { progressReporter } from './progress.js';
-import { redactor } from './template.js';
+import { credentialSecrets, redactor } from './template.js';
 import { runSession } from './session/loop.js';
 import { isSessionClaim } from './types.js';
 import { localDay } from './update/check.js';
@@ -221,7 +223,7 @@ export class Runner {
                     jobTimeoutMs: config.jobTimeoutMs,
                     headless: config.headless,
                     ...(config.proxy ? { proxy: config.proxy } : {}),
-                    redact: redactor([session.credential?.password]),
+                    redact: redactor(credentialSecrets(session.credential)),
                     log,
                 });
                 return createDriver(registry, ctx, shared);
@@ -314,7 +316,7 @@ export class Runner {
         const jobId = job.job.id;
         const attempt = job.job.attempt;
         const platform = lease.device.platform;
-        const redact = redactor([job.credential?.password]);
+        const redact = redactor(credentialSecrets(job.credential));
         let cancelled = false;
         this.running.set(jobId, { id: jobId, kind: 'test', platform, device_id: lease.device.id, started_at: new Date().toISOString() });
         this.writeState();
@@ -339,10 +341,18 @@ export class Runner {
             }
         });
         let videoPath;
+        let files = JobFiles.empty();
         try {
+            // Downloaded or generated before the app starts, so a step never waits on the network.
+            files = await JobFiles.prepare({
+                files: job.files,
+                cache: testFileCacheFor(this.deps.paths ?? { root: join(tmpdir(), 'leera-qa-runner') }),
+                downloadUrl: (testFileId) => client.testFileDownloadUrl(testFileId, jobId),
+            });
             const ctx = jobContext(job, {
                 lease,
                 app: this.appSettings(),
+                files,
                 buildDownloadUrl: (buildId) => client.buildDownloadUrl(buildId, { jobId }),
                 actionTimeoutMs: config.actionTimeoutMs,
                 jobTimeoutMs: config.jobTimeoutMs,
@@ -443,6 +453,7 @@ export class Runner {
             clearInterval(heartbeat);
             progress.stop();
             this.running.delete(jobId);
+            await files.dispose().catch(() => undefined);
             // Each job records into its own temporary directory.
             if (videoPath)
                 await rm(dirname(videoPath), { recursive: true, force: true }).catch(() => undefined);

package/dist/src/session/loop.js

@@ -1,6 +1,6 @@
 import { setTimeout as sleep } from 'node:timers/promises';
 import { ApiError } from '../client.js';
-import { redactor } from '../template.js';
+import { credentialSecrets, redactor } from '../template.js';
 import { DEFAULT_COMMAND_TIMEOUT_MS } from './commands.js';
 import { SessionExecutor } from './execute.js';
 /** The long-poll never asks for more than the server allows (plan §4). */
@@ -16,7 +16,7 @@ export async function runSession(session, deps) {
     const { client, log } = deps;
     const now = deps.now ?? Date.now;
     const retryDelayMs = deps.retryDelayMs ?? 2_000;
-    const redact = redactor([session.credential?.password]);
+    const redact = redactor(credentialSecrets(session.credential));
     const started = now();
     const idleMs = Math.max(1, session.idle_timeout_seconds) * 1_000;
     const maxMs = Math.max(1, session.max_seconds) * 1_000;

package/dist/src/template.js

@@ -13,6 +13,10 @@ export function resolveUrl(url, baseUrl) {
         return trimSlash(baseUrl) + url;
     return url;
 }
+/** The values of a credential that must never appear in logs, reports or debug output. */
+export function credentialSecrets(credential) {
+    return [credential?.password, credential?.otp];
+}
 /** Replaces every secret in `text`; secrets under 4 characters are skipped because they would mangle ordinary words. */
 export function redactor(secrets) {
     const values = secrets
@@ -34,6 +38,8 @@ export function redactDeep(value, redact) {
 function placeholderValue(name, ctx) {
     if (name.startsWith('data.'))
         return dataValue(name, ctx);
+    if (name.startsWith('file.'))
+        return testFile(name.slice('file.'.length), name, ctx).file_name;
     switch (name) {
         case 'env.base_url':
             return trimSlash(ctx.baseUrl);
@@ -47,6 +53,11 @@ function placeholderValue(name, ctx) {
             return present(ctx.username, name);
         case 'credential.password':
             return present(ctx.password, name);
+        case 'credential.otp':
+            present(ctx.password, name);
+            if (!ctx.otp)
+                throw new TemplateError(`the script uses {{${name}}} but the account has no one-time code`);
+            return ctx.otp;
         case 'run.id':
             return String(ctx.runId);
         case 'item.id':
@@ -98,6 +109,43 @@ function dataValue(name, ctx) {
     }
     return value;
 }
+/** A test file of the job by name; `placeholder` is how the script wrote it, for messages. */
+export function testFile(fileName, placeholder, ctx) {
+    if (ctx.session)
+        throw new TemplateError(`a device session has no test files; {{${placeholder}}} only works in a queued run`);
+    const files = ctx.files ?? {};
+    const file = /^[a-z][a-z0-9_]{0,39}$/.test(fileName) && Object.hasOwn(files, fileName) ? files[fileName] : undefined;
+    if (!file)
+        throw new TemplateError(`the script uses {{${placeholder}}} but the job has no test file named ${fileName}`);
+    return file;
+}
+/**
+ * The files an `upload`, `drop_files` or `add_files` action hands over, as paths on this machine.
+ * Each entry is `{{file.<name>}}`, or a data placeholder whose value names a test file of the job;
+ * anything else is refused, so a script can never point the runner at its own files.
+ */
+export function resolveFileEntries(entries, ctx) {
+    if (!Array.isArray(entries) || entries.length === 0)
+        throw new TemplateError('the action names no files');
+    return entries.map((entry, i) => {
+        const whole = /^\s*\{\{\s*([a-z][a-z0-9_.]*)\s*\}\}\s*$/.exec(typeof entry === 'string' ? entry : '');
+        const placeholder = whole?.[1];
+        let name;
+        if (placeholder?.startsWith('file.')) {
+            name = placeholder.slice('file.'.length);
+        }
+        else if (placeholder?.startsWith('data.')) {
+            name = dataValue(placeholder, ctx).trim();
+        }
+        else {
+            throw new TemplateError(`files[${i}] must be {{file.<name>}} or a test data placeholder naming a test file`);
+        }
+        const file = testFile(name, placeholder.startsWith('file.') ? placeholder : `file.${name}`, ctx);
+        if (!file.path)
+            throw new TemplateError(`test file ${name} is not ready on this runner`);
+        return { name, file, path: file.path };
+    });
+}
 /** A value only a queued job has: device sessions refuse it, and older servers may not send it. */
 function jobValue(ctx, name, value) {
     if (ctx.session)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leera.io/qa-runner",
-  "version": "1.1.0",
+  "version": "1.4.0",
   "description": "Runs recorded QA test scripts on your own machines and reports results back to your workspace.",
   "license": "SEE LICENSE IN LICENSE",
   "type": "module",
@@ -16,7 +16,7 @@
     "node": ">=22.12"
   },
   "scripts": {
-    "build": "tsc -p tsconfig.json && node -e \"for (const f of ['dom-locate.js', 'dom-snapshot.js']) require('node:fs').copyFileSync('src/drivers/webdriver/' + f, 'dist/src/drivers/webdriver/' + f)\"",
+    "build": "tsc -p tsconfig.json && node -e \"for (const f of ['dom-locate.js', 'dom-snapshot.js', 'dom-element.js']) require('node:fs').copyFileSync('src/drivers/webdriver/' + f, 'dist/src/drivers/webdriver/' + f)\"",
     "typecheck": "tsc -p tsconfig.json --noEmit",
     "lint": "biome lint --error-on-warnings src test packaging",
     "test": "npm run build && node --test \"dist/test/unit/**/*.test.js\"",