lynkeus 0.2.0 → 0.4.0

package/lib/cases/ahead.d.ts

@@ -0,0 +1,31 @@
+import type { Case } from './format.js';
+import type { Prepared } from './runner.js';
+/** Where setups made ahead wait for their cases: files for a preparer in another process, memory in the same one. */
+export type Store = {
+    start(c: Case): void;
+    started(c: Case): boolean;
+    keep(c: Case, prepared: Prepared): void;
+    take(c: Case): Prepared | undefined;
+    waiting(): number;
+};
+export declare class InMemory implements Store {
+    #private;
+    start(c: Case): void;
+    started(c: Case): boolean;
+    keep(c: Case, prepared: Prepared): void;
+    take(c: Case): Prepared | undefined;
+    waiting(): number;
+}
+export declare class Ahead implements Store {
+    #private;
+    private readonly dir;
+    constructor(dir: string);
+    clear(): void;
+    start(c: Case): void;
+    started(c: Case): boolean;
+    keep(c: Case, prepared: Prepared): void;
+    /** Setups made and not yet taken by a case that is still to start. */
+    waiting(): number;
+    /** The case's setup if it was made, is recent and is the setup the case has now. */
+    take(c: Case): Prepared | undefined;
+}

package/lib/cases/ahead.js

@@ -0,0 +1,90 @@
+/**
+ * Setups made ahead of their cases. `case prepare` runs, in the order a suite
+ * will, the setup of every case that needs only the project's fixtures, and
+ * leaves what it produced here, one file a case; `case run` takes the file
+ * when its case starts and goes straight to the steps.
+ *
+ * A case that starts before its setup is ready marks itself started, runs its
+ * own setup, and the preparer skips it. Files are taken with a rename, so two
+ * processes never get the same one.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+// What a backend held a quarter of an hour ago may have moved on (a session, a rate).
+const MAX_AGE_MS = 15 * 60_000;
+export class InMemory {
+    #started = new Set();
+    #kept = new Map();
+    start(c) {
+        this.#started.add(c.id);
+    }
+    started(c) {
+        return this.#started.has(c.id);
+    }
+    keep(c, prepared) {
+        if (!this.started(c))
+            this.#kept.set(c.id, prepared);
+    }
+    take(c) {
+        const prepared = this.#kept.get(c.id);
+        this.#kept.delete(c.id);
+        return prepared;
+    }
+    waiting() {
+        return this.#kept.size;
+    }
+}
+export class Ahead {
+    dir;
+    constructor(dir) {
+        this.dir = dir;
+    }
+    #file(c, suffix = '.json') {
+        return path.join(this.dir, `${c.id.replace(/[^\w.-]+/g, '_')}${suffix}`);
+    }
+    clear() {
+        fs.rmSync(this.dir, { recursive: true, force: true });
+        fs.mkdirSync(this.dir, { recursive: true });
+    }
+    start(c) {
+        fs.mkdirSync(this.dir, { recursive: true });
+        fs.writeFileSync(this.#file(c, '.started'), '');
+    }
+    started(c) {
+        return fs.existsSync(this.#file(c, '.started'));
+    }
+    keep(c, prepared) {
+        // Its case started while this was being made, and ran a setup of its own.
+        if (this.started(c))
+            return;
+        const file = this.#file(c);
+        const kept = { setup: JSON.stringify(c.setup), at: Date.now(), prepared };
+        fs.writeFileSync(`${file}.tmp`, JSON.stringify(kept));
+        fs.renameSync(`${file}.tmp`, file);
+    }
+    /** Setups made and not yet taken by a case that is still to start. */
+    waiting() {
+        return fs.readdirSync(this.dir).filter((f) => f.endsWith('.json') && !fs.existsSync(path.join(this.dir, f.replace(/\.json$/, '.started')))).length;
+    }
+    /** The case's setup if it was made, is recent and is the setup the case has now. */
+    take(c) {
+        const file = this.#file(c);
+        const mine = `${file}.${process.pid}`;
+        try {
+            fs.renameSync(file, mine);
+        }
+        catch {
+            return undefined;
+        }
+        try {
+            const kept = JSON.parse(fs.readFileSync(mine, 'utf8'));
+            return kept.setup === JSON.stringify(c.setup) && Date.now() - kept.at < MAX_AGE_MS ? kept.prepared : undefined;
+        }
+        catch {
+            return undefined;
+        }
+        finally {
+            fs.rmSync(mine, { force: true });
+        }
+    }
+}

package/lib/cases/fixtures.d.ts

@@ -5,9 +5,21 @@ export type FixturesConfig = {
     timeoutMs?: number;
 };
 export type Answer = Record<string, unknown>;
+/** The one command that is lynkeus's own and not the project's. */
+export declare const DESCRIBE = "lynkeus.describe";
+export type Description = {
+    commands: string[];
+    down?: string;
+};
+/** lynkeus's second own command: this connection works ahead of the cases. */
+export declare const AHEAD = "lynkeus.ahead";
 export declare class Fixtures {
     #private;
     constructor(config: FixturesConfig, root: string);
     call(command: string, params: Record<string, unknown>): Promise<Answer>;
+    /** What the server says it answers, or nothing when it does not say. */
+    describe(): Promise<Description | undefined>;
+    /** Says this connection works ahead of the cases being run. Whatever the server answers, it goes on. */
+    ahead(): Promise<void>;
     close(): void;
 }

package/lib/cases/fixtures.js

@@ -8,10 +8,30 @@
  * — and one line out, the step's answer: any JSON object, `"ok": false` and an
  * `error` when it failed. `params` is the step as written; `env` is the same
  * flattened to upper-case strings, for a server that hands them to a script.
+ *
+ * Before the first case lynkeus asks what the server can do —
+ *
+ *   {"command": "lynkeus.describe", "params": {}, "env": {}}
+ *
+ * — and a server that knows answers `{"commands": ["create_user", …]}`. With
+ * that, a case naming a fixture nobody answers is refused before anything
+ * runs. A server that answers `"ok": false`, as one written before this did,
+ * is taken to describe nothing, and its cases run unchecked. A server that is
+ * running but cannot serve (its database is down) adds `"down": "<why>"`, and
+ * no case starts: every one would fail at its first fixture.
+ *
+ * A connection that makes cases' setups ahead of time (`case prepare`) says so
+ * first, with `lynkeus.ahead`, so a server may give its work a lower priority
+ * than the cases running meanwhile. A server that does not know it is free to
+ * answer anything.
  */
 import { spawn } from 'node:child_process';
 import path from 'node:path';
 import readline from 'node:readline';
+/** The one command that is lynkeus's own and not the project's. */
+export const DESCRIBE = 'lynkeus.describe';
+/** lynkeus's second own command: this connection works ahead of the cases. */
+export const AHEAD = 'lynkeus.ahead';
 const envOf = (params) => Object.fromEntries(Object.entries(params)
     .filter(([, v]) => v !== undefined && v !== null)
     .map(([k, v]) => [k.replace(/([a-z])([A-Z])/g, '$1_$2').toUpperCase(), typeof v === 'object' ? JSON.stringify(v) : String(v)]));
@@ -83,6 +103,18 @@ export class Fixtures {
             throw new Error(`${command}: ${String(answer.error ?? answer.message ?? 'ok=false')}`);
         return answer;
     }
+    /** What the server says it answers, or nothing when it does not say. */
+    async describe() {
+        const answer = await this.call(DESCRIBE, {}).catch(() => undefined);
+        const commands = answer?.commands;
+        if (!Array.isArray(commands) || !commands.every((c) => typeof c === 'string'))
+            return undefined;
+        return typeof answer?.down === 'string' && answer.down ? { commands, down: answer.down } : { commands };
+    }
+    /** Says this connection works ahead of the cases being run. Whatever the server answers, it goes on. */
+    async ahead() {
+        await this.call(AHEAD, {}).catch(() => undefined);
+    }
     close() {
         const child = this.#child;
         this.#child = null;

package/lib/cases/flakes.d.ts

@@ -0,0 +1,28 @@
+import type { CaseReport } from './runner.js';
+export type Outcome = {
+    at: string;
+    result: CaseReport['result'];
+    retried: boolean;
+    ms?: number;
+    step?: string;
+    error?: string;
+};
+export type Ledger = Record<string, {
+    runs: number;
+    failed: number;
+    retried: number;
+    last: Outcome[];
+}>;
+export declare class Flakes {
+    private readonly file;
+    private readonly ledger;
+    constructor(file: string);
+    note(report: CaseReport & {
+        retried?: boolean;
+    }, at?: Date): void;
+    /** How long a case took when it last passed, if the ledger has seen it pass. */
+    lastMs(id: string): number | undefined;
+    /** The cases among `ids` whose recent runs disagree with each other, worded for a summary. */
+    flaky(ids: string[]): string[];
+    save(): void;
+}

package/lib/cases/flakes.js

@@ -0,0 +1,61 @@
+/**
+ * How each case has done across runs, kept in one JSON file: the last results,
+ * how often it needed its second chance, the step it last died on. A case that
+ * passes and fails without anything changing is the one to look at first.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+const KEPT = 20;
+export class Flakes {
+    file;
+    ledger;
+    constructor(file) {
+        this.file = file;
+        let read = {};
+        try {
+            read = JSON.parse(fs.readFileSync(file, 'utf8'));
+        }
+        catch {
+            // No ledger yet, or one this cannot read: start a new one.
+        }
+        this.ledger = read;
+    }
+    note(report, at = new Date()) {
+        const entry = this.ledger[report.id] ?? { runs: 0, failed: 0, retried: 0, last: [] };
+        this.ledger[report.id] = entry;
+        const broke = report.steps.find((s) => s.status === 'failed');
+        entry.runs += 1;
+        if (report.result === 'failed')
+            entry.failed += 1;
+        if (report.retried)
+            entry.retried += 1;
+        entry.last = [
+            ...entry.last,
+            {
+                at: at.toISOString(),
+                result: report.result,
+                retried: !!report.retried,
+                ms: report.ms,
+                ...(broke ? { step: broke.step, error: broke.error?.split('\n')[0] } : {}),
+            },
+        ].slice(-KEPT);
+    }
+    /** How long a case took when it last passed, if the ledger has seen it pass. */
+    lastMs(id) {
+        return [...(this.ledger[id]?.last ?? [])].reverse().find((o) => o.result === 'passed' && o.ms !== undefined)?.ms;
+    }
+    /** The cases among `ids` whose recent runs disagree with each other, worded for a summary. */
+    flaky(ids) {
+        return ids.flatMap((id) => {
+            const last = this.ledger[id]?.last ?? [];
+            const shaky = last.filter((o) => o.result === 'failed' || o.retried).length;
+            if (shaky === 0 || shaky === last.length)
+                return [];
+            return [`⚠ ${id} is flaky: ${shaky} of its last ${last.length} runs failed or needed a second chance`];
+        });
+    }
+    save() {
+        fs.mkdirSync(path.dirname(this.file), { recursive: true });
+        fs.writeFileSync(this.file, `${JSON.stringify(this.ledger, null, 2)}\n`);
+    }
+}

package/lib/cases/format.d.ts

@@ -20,6 +20,11 @@ export declare const readCases: (file: string) => Case[];
 /** `a.b.0.c` into a value; `a.*.b` collects b from every element of the list a. */
 export declare const dig: (json: unknown, dotted: string) => unknown;
 export type Scope = Record<string, unknown>;
+/**
+ * With LYNKEUS_SEED set, a case's `{{random.*}}` values are the same on every run: two runs of it then show
+ * the same names, which comparing their screens needs. Each case draws its own, so cases side by side differ.
+ */
+export declare const seeded: (scope: Scope, id: string) => void;
 /** `{{alias.path}}` from what earlier steps answered, `{{random.letters}}`, `{{env.NAME}}`. */
 export declare const interpolate: <T>(value: T, scope: Scope) => T;
 export type Matcher = {

package/lib/cases/format.js

@@ -94,12 +94,38 @@ export const dig = (json, dotted) => {
     }
     return current;
 };
+/** Where a seeded case keeps how far its random values have gone; no alias can be named this. */
+const SEED = '~seed';
+/**
+ * With LYNKEUS_SEED set, a case's `{{random.*}}` values are the same on every run: two runs of it then show
+ * the same names, which comparing their screens needs. Each case draws its own, so cases side by side differ.
+ */
+export const seeded = (scope, id) => {
+    const seed = process.env.LYNKEUS_SEED;
+    if (!seed || scope[SEED] !== undefined)
+        return;
+    let hash = 2166136261;
+    for (const char of `${seed}\n${id}`)
+        hash = Math.imul(hash ^ char.charCodeAt(0), 16777619);
+    scope[SEED] = hash >>> 0;
+};
 const LETTERS = 'abcdefghijklmnopqrstuvwxyz';
-const random = (kind) => {
+const random = (kind, scope) => {
+    const next = () => {
+        if (typeof scope[SEED] !== 'number')
+            return Math.random();
+        // mulberry32
+        const state = (scope[SEED] + 0x6d2b79f5) >>> 0;
+        scope[SEED] = state;
+        let t = state;
+        t = Math.imul(t ^ (t >>> 15), t | 1);
+        t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
+        return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+    };
     if (kind === 'letters')
-        return Array.from({ length: 6 }, () => LETTERS[Math.floor(Math.random() * 26)]).join('');
+        return Array.from({ length: 6 }, () => LETTERS[Math.floor(next() * 26)]).join('');
     if (kind === 'digits')
-        return Array.from({ length: 6 }, () => Math.floor(Math.random() * 10)).join('');
+        return Array.from({ length: 6 }, () => Math.floor(next() * 10)).join('');
     return undefined;
 };
 /** `{{alias.path}}` from what earlier steps answered, `{{random.letters}}`, `{{env.NAME}}`. */
@@ -108,7 +134,7 @@ export const interpolate = (value, scope) => {
         const whole = /^\{\{\s*([\w-]+)\.([\w.*-]+)\s*\}\}$/.exec(value);
         const lookup = (name, path) => {
             if (name === 'random')
-                return random(path);
+                return random(path, scope);
             if (name === 'env')
                 return process.env[path];
             if (!(name in scope))

package/lib/cases/portClaim.d.ts

@@ -0,0 +1,10 @@
+/**
+ * A run lets go of its driver port between cases, and the app dials whichever driver is listening: two runs
+ * on one port take turns at the app, and each reports steps the other drove. A run claims its ports for as
+ * long as it lasts; what it answers is the release, or why another run holds one.
+ */
+export declare const claimPorts: (ports: number[], dir?: string) => {
+    release: () => void;
+} | {
+    refused: string;
+};

package/lib/cases/portClaim.js

@@ -0,0 +1,55 @@
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+const alive = (pid) => {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (error) {
+        return error.code === 'EPERM';
+    }
+};
+/**
+ * A run lets go of its driver port between cases, and the app dials whichever driver is listening: two runs
+ * on one port take turns at the app, and each reports steps the other drove. A run claims its ports for as
+ * long as it lasts; what it answers is the release, or why another run holds one.
+ */
+export const claimPorts = (ports, dir = os.tmpdir()) => {
+    const held = [];
+    const release = () => {
+        for (const file of held) {
+            try {
+                if (fs.readFileSync(file, 'utf8').trim() === String(process.pid))
+                    fs.unlinkSync(file);
+            }
+            catch {
+                // Already gone.
+            }
+        }
+    };
+    for (const port of ports) {
+        const file = path.join(dir, `lynkeus-port-${port}.pid`);
+        for (;;) {
+            try {
+                fs.writeFileSync(file, String(process.pid), { flag: 'wx' });
+                held.push(file);
+                break;
+            }
+            catch (error) {
+                if (error.code !== 'EEXIST')
+                    throw error;
+            }
+            const owner = Number(fs.readFileSync(file, 'utf8').trim());
+            if (Number.isInteger(owner) && owner > 0 && owner !== process.pid && alive(owner)) {
+                release();
+                return {
+                    refused: `port ${port} is held by another lynkeus case run (pid ${owner}), which is probably still going: stop it, or give this run other --ports`,
+                };
+            }
+            // Left by a run that ended without letting go.
+            fs.rmSync(file, { force: true });
+        }
+    }
+    return { release };
+};

package/lib/cases/runfile.d.ts

@@ -0,0 +1,24 @@
+import { type RunCase } from 'lynkeus-protocol';
+import type { Case } from './format.js';
+import type { CaseReport } from './runner.js';
+export type Attempt = {
+    report: CaseReport;
+    attempt: number;
+    startedAt: Date;
+};
+export declare const toRunCase: (c: Case, { report, attempt, startedAt }: Attempt, root: string, dir?: string) => RunCase;
+export declare class RunFile {
+    private readonly dir;
+    private readonly root;
+    private readonly manifest;
+    private readonly last;
+    constructor(dir: string, root: string, startedAt?: Date);
+    /** Where a case's frames and film go: one folder per attempt. */
+    evidenceDir(id: string, attempt: number): string;
+    add(c: Case, attempt: Attempt): void;
+    finish(app?: {
+        platform?: string;
+        native?: boolean;
+    }, finishedAt?: Date): void;
+    private save;
+}

package/lib/cases/runfile.js

@@ -0,0 +1,108 @@
+/**
+ * Writes a run of cases as the directory `lynkeus-protocol` describes: run.json
+ * and cases.jsonl. The manifest is written at the start and again at the end,
+ * so a run that was cut short still says when it began and what it ran on.
+ */
+import { execFileSync } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+import { RUN_SCHEMA } from 'lynkeus-protocol';
+/** Paths in the run file are relative to its directory, so the directory can move. */
+const within = (dir, file) => (file ? (dir ? path.relative(dir, file) : file) : undefined);
+export const toRunCase = (c, { report, attempt, startedAt }, root, dir) => {
+    const steps = report.steps.map((s, index) => {
+        // The runner numbers an event's step from 1, in the order the steps ran.
+        const told = (report.events ?? [])
+            .filter((e) => e.step === index + 1)
+            .map((e) => ({ origin: 'app', name: e.name, ...(e.props ? { props: e.props } : {}), ...(e.route ? { route: e.route } : {}) }));
+        const sent = (s.sent ?? []).map((e) => ({ origin: 'app', provider: e.provider, name: e.name, ...(e.props ? { props: e.props } : {}) }));
+        const events = [...told, ...sent];
+        const screenshot = within(dir, s.screenshot);
+        return {
+            step: s.step,
+            status: s.status,
+            ms: s.ms,
+            ...(s.error ? { error: s.error } : {}),
+            section: index < c.setup.length ? 'setup' : 'steps',
+            ...(events.length > 0 ? { events } : {}),
+            ...(screenshot ? { screenshot } : {}),
+            ...(s.videoS !== undefined ? { videoS: s.videoS } : {}),
+            ...(s.backend?.length ? { backend: s.backend } : {}),
+        };
+    });
+    const description = typeof c.meta.description === 'string' ? c.meta.description : undefined;
+    const video = within(dir, report.video);
+    return {
+        id: report.id,
+        ...(report.title ? { title: report.title } : {}),
+        ...(description ? { description } : {}),
+        file: path.relative(root, path.resolve(root, report.file)),
+        attempt,
+        result: report.result,
+        startedAt: startedAt.toISOString(),
+        ms: report.ms,
+        steps,
+        ...(report.diagnosis ? { diagnosis: report.diagnosis } : {}),
+        ...(report.evidence ? { evidence: report.evidence } : {}),
+        ...(video ? { video } : {}),
+    };
+};
+/** The branch and commit the run was made from, so a report can say which code it tested. */
+const gitOf = (root) => {
+    const ask = (...args) => {
+        try {
+            return execFileSync('git', args, { cwd: root, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim() || undefined;
+        }
+        catch {
+            return undefined;
+        }
+    };
+    const commit = ask('rev-parse', '--short', 'HEAD');
+    if (!commit)
+        return undefined;
+    const branch = process.env.GITHUB_HEAD_REF || process.env.GITHUB_REF_NAME || ask('rev-parse', '--abbrev-ref', 'HEAD');
+    return { commit, ...(branch && branch !== 'HEAD' ? { branch } : {}) };
+};
+export class RunFile {
+    dir;
+    root;
+    manifest;
+    last = new Map();
+    constructor(dir, root, startedAt = new Date()) {
+        this.dir = dir;
+        this.root = root;
+        fs.mkdirSync(dir, { recursive: true });
+        fs.writeFileSync(path.join(dir, 'cases.jsonl'), '');
+        const git = gitOf(root);
+        this.manifest = { schema: RUN_SCHEMA, startedAt: startedAt.toISOString(), ...(git ? { git } : {}) };
+        this.save();
+    }
+    /** Where a case's frames and film go: one folder per attempt. */
+    evidenceDir(id, attempt) {
+        return path.join(this.dir, `${id.replace(/[^\w.=-]+/g, '_')}${attempt > 1 ? `.${attempt}` : ''}`);
+    }
+    add(c, attempt) {
+        const line = toRunCase(c, attempt, this.root, this.dir);
+        fs.appendFileSync(path.join(this.dir, 'cases.jsonl'), `${JSON.stringify(line)}\n`);
+        this.last.set(line.id, { result: line.result, attempt: line.attempt });
+    }
+    finish(app = {}, finishedAt = new Date()) {
+        const cases = [...this.last.values()];
+        this.manifest.finishedAt = finishedAt.toISOString();
+        if (app.platform)
+            this.manifest.platform = app.platform;
+        if (app.native !== undefined)
+            this.manifest.native = app.native;
+        this.manifest.totals = {
+            cases: cases.length,
+            passed: cases.filter((c) => c.result === 'passed').length,
+            failed: cases.filter((c) => c.result === 'failed').length,
+            unsupported: cases.filter((c) => c.result === 'unsupported').length,
+            retried: cases.filter((c) => c.attempt > 1).length,
+        };
+        this.save();
+    }
+    save() {
+        fs.writeFileSync(path.join(this.dir, 'run.json'), `${JSON.stringify(this.manifest, null, 2)}\n`);
+    }
+}