e2e 0.17.0 → 0.18.0

package/dist/run/in-process.js

@@ -5,6 +5,7 @@
  * `TargetWorker` core; only the transport and pair resolution differ, and the
  * scheduler is capped at one worker so nothing overlaps in this process.
  */
+import { isAbandonedRejection } from '../internal/abandoned.js';
 import { classifyError, ConfigurationError, serializeError } from '../internal/errors.js';
 import { pairKey } from './units.js';
 import { TargetWorker } from './worker/session.js';
@@ -18,6 +19,22 @@ class InProcessRunner {
     finish;
     exited = false;
     closing = false;
+    /**
+     * Set by a fatal error: the worker is dead to the scheduler from then on,
+     * as a crashed process would be, so nothing it reports about its unit
+     * while it winds down counts, and the unit's results are synthesized when
+     * it exits. Unlike a dead process it still disposes its engine, so the
+     * failures of that disposal (`shutdown-done`) are still reported.
+     */
+    crashed = false;
+    /** Takes what this process failed to catch while the runner lives; see `catchStrays`. */
+    onRejection = (cause) => {
+        if (isAbandonedRejection(cause))
+            return;
+        if (!this.worker.strayRejection(cause))
+            this.fail(cause);
+    };
+    onException = (cause) => this.fail(cause);
     constructor(targetName, workerSlot, events, options) {
         this.targetName = targetName;
         this.workerSlot = workerSlot;
@@ -30,20 +47,14 @@ class InProcessRunner {
         this.finish = finish;
         this.worker = new TargetWorker({
             emit: (message) => {
-                if (!this.exited)
-                    this.events.onMessage(message);
-            },
-            fatal: (cause) => {
-                if (this.exited)
+                if (this.exited || (this.crashed && message.type !== 'shutdown-done'))
                     return;
-                this.events.onMessage({
-                    type: 'fatal',
-                    error: serializeError(classifyError(cause)),
-                });
-                this.close();
+                this.events.onMessage(message);
             },
+            fatal: (cause) => this.fail(cause),
             finished: () => this.end('shut down'),
         }, () => this.bootstrap());
+        this.catchStrays();
         this.worker.start();
     }
     get alive() {
@@ -75,6 +86,34 @@ class InProcessRunner {
     kill() {
         this.close();
     }
+    /**
+     * Catches what a test leaves uncaught the way a worker process does
+     * (`worker/entry.ts`): a rejection nobody handled is charged to the attempt
+     * in flight, or recorded against the last test that finished, and is
+     * otherwise fatal to this worker, as an uncaught exception is (see
+     * `fail`). The
+     * listeners live exactly as long as the runner, so a host process keeps its
+     * own handling before and after the run. The scheduler keeps at most one
+     * in-process runner alive at a time, so nothing is charged twice.
+     */
+    catchStrays() {
+        process.on('unhandledRejection', this.onRejection);
+        process.on('uncaughtException', this.onException);
+    }
+    /**
+     * Reports an unrecoverable failure and ends the worker the way a process
+     * dies: its unit is interrupted and nothing more it reports is passed on,
+     * so on exit the scheduler fails the test in flight with `WORKER_CRASH`
+     * and skips the unit's others, as it does for a worker process.
+     */
+    fail(cause) {
+        if (this.exited || this.crashed)
+            return;
+        this.events.onMessage({ type: 'fatal', error: serializeError(classifyError(cause)) });
+        this.crashed = true;
+        this.worker.handle({ type: 'interrupt' });
+        this.close();
+    }
     /** Stops accepting work and asks the queue to wind down. */
     close() {
         if (this.exited || this.closing)
@@ -86,6 +125,8 @@ class InProcessRunner {
         if (this.exited)
             return;
         this.exited = true;
+        process.off('unhandledRejection', this.onRejection);
+        process.off('uncaughtException', this.onException);
         this.events.onExit(detail);
         this.finish();
     }

package/dist/run/process-output.d.ts

@@ -0,0 +1,73 @@
+/**
+ * What user code in an `e2e run` process prints outside a test body: the
+ * config's top-level code, a test file's top level while it is collected,
+ * and reporters in the runner; the config's top-level code again in each
+ * worker. It is redacted with every secret value the process knows, as a
+ * test's output is. Until a config has loaded its secrets are unknown, so
+ * what is printed meanwhile is held, and withheld if the load fails.
+ */
+import { type SecretLedger } from '../internal/redact.ts';
+import type { ListReporterOutput } from '../report/list.ts';
+export type OutputStream = 'stdout' | 'stderr';
+/** Writes text to one stream of the terminal; the callback fires once the stream took it. Answers false while the stream is backed up, like `Writable.write`. */
+export type TerminalWrite = (text: string, callback?: () => void) => boolean;
+/** Where runner output goes instead of the terminal while the list reporter shows the run. */
+export interface OutputView {
+    /** Prints redacted text from `stream`. */
+    show(stream: OutputStream, text: string): void;
+    /** Prints what `show` left unfinished; nothing more follows. */
+    end(): void;
+}
+/**
+ * Holds what is written while a config loads. Released once the load
+ * registered the config's secrets; dropped for a warning when it failed.
+ */
+export declare class ConfigLoadHold {
+    private held;
+    /** Holds `chunk` if a load is in flight; answers whether it did. */
+    hold(stream: OutputStream, chunk: string | Uint8Array): boolean;
+    /**
+     * Runs `load`, which loads a config and registers its secrets, holding
+     * what is written meanwhile. Then hands each held write to `release` in
+     * order, or, if `load` threw, `warn`s with how much was withheld.
+     */
+    during<T>(load: () => Promise<T>, release: (stream: OutputStream, chunk: string | Uint8Array) => void, warn: (line: string) => void): Promise<T>;
+}
+/**
+ * The runner process's user output, on its way to the terminal or, while
+ * the list reporter shows it, through the reporter, which prints it above
+ * its live window instead of letting it land inside.
+ */
+export declare class RunnerOutput {
+    private readonly terminal;
+    /** Each stream redacted on its own, across writes: a value split over two writes is still caught. */
+    private readonly redactors;
+    private readonly loadHold;
+    private view;
+    /** The terminal's stdout for the list reporter: its own lines and live window go straight there. */
+    readonly listOutput: ListReporterOutput;
+    constructor(terminal: Readonly<Record<OutputStream, TerminalWrite>>, ledger: SecretLedger);
+    /**
+     * One write of user code to `stream`, redacted: the tail a later write
+     * could complete into a value waits for it, or for `end`. Held while a
+     * config loads. Answers what the terminal did.
+     */
+    write(stream: OutputStream, chunk: string | Uint8Array, callback?: () => void): boolean;
+    /**
+     * Runs `load`, a config load that registers the config's secrets, holding
+     * what is printed until it ends: the config's top-level code may print a
+     * value before the process knows it is one. Released redacted once it
+     * loaded; withheld with a warning when it failed.
+     */
+    withholdDuring<T>(load: () => Promise<T>): Promise<T>;
+    /**
+     * Passes what is printed to `view` from now on, or to the terminal again
+     * when undefined. What the previous view was given is released to it
+     * first, unfinished lines included, and that view ended.
+     */
+    showThrough(view: OutputView | undefined): void;
+    /** Releases what is still held, redacted, to the terminal. Later writes stay redacted; only a tail a later write could complete into a value is held. */
+    end(): void;
+    private release;
+    private emit;
+}

package/dist/run/process-output.js

@@ -0,0 +1,130 @@
+/**
+ * What user code in an `e2e run` process prints outside a test body: the
+ * config's top-level code, a test file's top level while it is collected,
+ * and reporters in the runner; the config's top-level code again in each
+ * worker. It is redacted with every secret value the process knows, as a
+ * test's output is. Until a config has loaded its secrets are unknown, so
+ * what is printed meanwhile is held, and withheld if the load fails.
+ */
+import { StreamRedactor } from '../internal/redact.js';
+/** The warning that stands in for what a config printed before it failed to load. */
+function withheldWarning(bytes) {
+    return `withheld ${bytes} bytes of output printed while the config failed to load: its secrets are unknown, so the output cannot be redacted`;
+}
+/**
+ * Holds what is written while a config loads. Released once the load
+ * registered the config's secrets; dropped for a warning when it failed.
+ */
+export class ConfigLoadHold {
+    held;
+    /** Holds `chunk` if a load is in flight; answers whether it did. */
+    hold(stream, chunk) {
+        if (this.held === undefined)
+            return false;
+        // A copy: the writer may reuse its buffer once the write returns.
+        this.held.push({ stream, chunk: typeof chunk === 'string' ? chunk : Buffer.from(chunk) });
+        return true;
+    }
+    /**
+     * Runs `load`, which loads a config and registers its secrets, holding
+     * what is written meanwhile. Then hands each held write to `release` in
+     * order, or, if `load` threw, `warn`s with how much was withheld.
+     */
+    async during(load, release, warn) {
+        this.held = [];
+        let loaded = false;
+        try {
+            const value = await load();
+            loaded = true;
+            return value;
+        }
+        finally {
+            const held = this.held;
+            this.held = undefined;
+            if (loaded) {
+                for (const { stream, chunk } of held)
+                    release(stream, chunk);
+            }
+            else {
+                const bytes = held.reduce((total, { chunk }) => total + Buffer.byteLength(chunk), 0);
+                if (bytes > 0)
+                    warn(withheldWarning(bytes));
+            }
+        }
+    }
+}
+/**
+ * The runner process's user output, on its way to the terminal or, while
+ * the list reporter shows it, through the reporter, which prints it above
+ * its live window instead of letting it land inside.
+ */
+export class RunnerOutput {
+    terminal;
+    /** Each stream redacted on its own, across writes: a value split over two writes is still caught. */
+    redactors;
+    loadHold = new ConfigLoadHold();
+    view;
+    /** The terminal's stdout for the list reporter: its own lines and live window go straight there. */
+    listOutput;
+    constructor(terminal, ledger) {
+        this.terminal = terminal;
+        this.redactors = { stdout: new StreamRedactor(ledger), stderr: new StreamRedactor(ledger) };
+        this.listOutput = {
+            write: (line) => void terminal.stdout(`${line}\n`),
+            raw: (text) => void terminal.stdout(text),
+        };
+    }
+    /**
+     * One write of user code to `stream`, redacted: the tail a later write
+     * could complete into a value waits for it, or for `end`. Held while a
+     * config loads. Answers what the terminal did.
+     */
+    write(stream, chunk, callback) {
+        if (this.loadHold.hold(stream, chunk)) {
+            if (callback !== undefined)
+                process.nextTick(callback);
+            return true;
+        }
+        return this.emit(stream, this.redactors[stream].push(chunk), callback);
+    }
+    /**
+     * Runs `load`, a config load that registers the config's secrets, holding
+     * what is printed until it ends: the config's top-level code may print a
+     * value before the process knows it is one. Released redacted once it
+     * loaded; withheld with a warning when it failed.
+     */
+    withholdDuring(load) {
+        return this.loadHold.during(load, (stream, chunk) => this.write(stream, chunk), (line) => this.emit('stderr', `e2e: [warning] ${line}\n`));
+    }
+    /**
+     * Passes what is printed to `view` from now on, or to the terminal again
+     * when undefined. What the previous view was given is released to it
+     * first, unfinished lines included, and that view ended.
+     */
+    showThrough(view) {
+        if (this.view !== undefined) {
+            this.release();
+            this.view.end();
+        }
+        this.view = view;
+    }
+    /** Releases what is still held, redacted, to the terminal. Later writes stay redacted; only a tail a later write could complete into a value is held. */
+    end() {
+        this.showThrough(undefined);
+        this.release();
+    }
+    release() {
+        this.emit('stdout', this.redactors.stdout.flush());
+        this.emit('stderr', this.redactors.stderr.flush());
+    }
+    emit(stream, text, callback) {
+        if (text === '' || this.view !== undefined) {
+            if (text !== '')
+                this.view?.show(stream, text);
+            if (callback !== undefined)
+                process.nextTick(callback);
+            return true;
+        }
+        return this.terminal[stream](text, callback);
+    }
+}

package/dist/run/provision.d.ts

@@ -55,6 +55,8 @@ export interface PrepareScope {
     /** The same `env` the workers are started with: what prepare provisions must be where a launch will look for it. */
     readonly env: NodeJS.ProcessEnv;
     readonly signal: AbortSignal;
+    /** Whether the run asked for a visible surface, so a hook provisions for the browser it will launch. */
+    readonly headed: boolean;
     /** Where a hook's progress lines go, under the target they concern. */
     readonly notice: (targetName: string, line: string) => void;
 }

package/dist/run/provision.js

@@ -129,6 +129,7 @@ export class PreparedEngines {
                 runId: scope.runId,
                 targetName: target.name,
                 projectRoot: scope.projectRoot,
+                headed: scope.headed,
                 app: engineAppInfo(target.app),
                 slots,
                 env: scope.env,

package/dist/run/records.d.ts

@@ -30,13 +30,13 @@ export interface ArtifactRecord {
     /**
      * Mirrors report-1: how much of the file the runner masked. A screenshot is
      * `complete`; a trace is `complete` once every registered secret value was
-     * rewritten out of its text, and `not-required` when no secret reached its
-     * session (filled, or held by the engine); a video is `incomplete`, since a recording masks nothing
+     * rewritten out of its text, and `not-required` when the run has no
+     * secret value to rewrite; a video is `incomplete`, since a recording masks nothing
      * (a secure field renders its own dots, but anything else the screen
      * showed is in the frames), and is kept as it is; a download is
      * `incomplete` too, bytes the app served and the runner did not rewrite,
-     * unless a secret reached the session and the file is text the
-     * ledger was run over, which makes it `complete`. report-1 also admits an
+     * unless the file is text the session's ledger was run over, which makes
+     * it `complete`. report-1 also admits an
      * `incomplete` artifact without a `path`: a video a hosted service keeps,
      * recorded by `url`, or one its producer withheld, which this runner never
      * writes.

package/dist/run/retry.d.ts

@@ -17,3 +17,10 @@ export declare function isRetryEligible(attempt: RetryAttempt): boolean;
  * did fail, and only its second chance was cut short.
  */
 export declare function runWithRetries(maxAttempts: number, interruptSignal: AbortSignal, runOnce: (attemptIndex: number) => Promise<RetryAttempt | undefined>): Promise<ResultStatus>;
+/**
+ * The verdict a sequence of finished attempts reaches: a pass is `flaky`
+ * after a failure, an interrupt during a retry keeps the verdict before it,
+ * and a self-skip decides. Also the verdict of attempts a worker finished
+ * before the run's interrupt stopped it, so both read alike.
+ */
+export declare function retryVerdict(attempts: readonly RetryAttempt[]): ResultStatus;

package/dist/run/retry.js

@@ -13,25 +13,41 @@ export function isRetryEligible(attempt) {
  * did fail, and only its second chance was cut short.
  */
 export async function runWithRetries(maxAttempts, interruptSignal, runOnce) {
-    let finalStatus = 'failed';
-    let attemptCount = 0;
+    const attempts = [];
     for (let attemptIndex = 0; attemptIndex < maxAttempts; attemptIndex += 1) {
         if (interruptSignal.aborted)
             break;
         const attempt = await runOnce(attemptIndex);
         if (attempt === undefined)
             break;
-        attemptCount += 1;
-        if (attempt.status === 'passed')
-            return attemptCount > 1 ? 'flaky' : 'passed';
-        if (attempt.status === 'interrupted')
-            return attemptCount > 1 ? finalStatus : 'interrupted';
+        attempts.push(attempt);
         // A body that skipped itself has decided; a retry would only ask again.
-        if (attempt.status === 'skipped')
-            return 'skipped';
-        finalStatus = attempt.status;
+        if (attempt.status === 'passed' || attempt.status === 'interrupted' || attempt.status === 'skipped')
+            break;
         if (!isRetryEligible(attempt))
             break;
     }
-    return finalStatus;
+    return retryVerdict(attempts);
+}
+/**
+ * The verdict a sequence of finished attempts reaches: a pass is `flaky`
+ * after a failure, an interrupt during a retry keeps the verdict before it,
+ * and a self-skip decides. Also the verdict of attempts a worker finished
+ * before the run's interrupt stopped it, so both read alike.
+ */
+export function retryVerdict(attempts) {
+    let verdict = 'failed';
+    for (const [index, attempt] of attempts.entries()) {
+        switch (attempt.status) {
+            case 'passed':
+                return index > 0 ? 'flaky' : 'passed';
+            case 'interrupted':
+                return index > 0 ? verdict : 'interrupted';
+            case 'skipped':
+                return 'skipped';
+            default:
+                verdict = attempt.status;
+        }
+    }
+    return verdict;
 }

package/dist/run/runner.d.ts

@@ -6,6 +6,7 @@ import type { ExploreProgress } from '../explore/progress.ts';
 import { type Report1Document, type ReportExplore } from '../report/build.ts';
 import { type RunEventSink, type RunExitCode, type RunStatus } from './events.ts';
 import { type ResultRecord } from './records.ts';
+import type { RunnerOutput } from './process-output.ts';
 import type { BuiltinReporter, E2EConfig, RecordingMode } from '../types.ts';
 export interface RunOptions {
     cwd?: string | undefined;
@@ -109,6 +110,13 @@ export interface RunOptions {
     onEvent?: RunEventSink | undefined;
     /** Budget for each reporter's `onRunFinished`, in ms. Only the test harness sets it; there is no flag. */
     reporterTimeout?: number | undefined;
+    /**
+     * The process's stdout and stderr, when the caller claimed them (the CLI):
+     * what user code in this process prints is held while the config loads
+     * and shown through the list reporter while the run reports, and the list
+     * reporter writes to the terminal past it.
+     */
+    processOutput?: RunnerOutput | undefined;
 }
 /**
  * A registration supplied in memory, under the virtual file name the report

package/dist/run/runner.js

@@ -23,6 +23,7 @@ import { runUnits } from './scheduler.js';
 import { buildWorkPlans, plannedSlots } from './units.js';
 import { SessionStore } from './sessions.js';
 import { outputLayout } from './output.js';
+import { registerStaticSecrets } from './secrecy.js';
 import { claimRerunDir, pruneArtifacts } from './artifacts.js';
 import { carryForward, lastFailedIds, readLastRun, reportArtifactPaths } from './last-run.js';
 import { childProcessSpawner } from './worker/handle.js';
@@ -131,14 +132,27 @@ export async function run(options = {}) {
     //
     // Free ports for URLs declared with port 0 are chosen here, once: workers
     // re-resolve the config and get the assignments in their bootstrap.
+    //
+    // The config's secrets are known to the process before what its top-level
+    // code printed is released, so that output is redacted like any other.
+    const loadConfig = async () => {
+        const config = await loadRunConfig(options, cwd, env, cli);
+        registerStaticSecrets(config.allSecrets);
+        return allocateAppPorts(config);
+    };
+    const processOutput = options.processOutput;
     const loaded = await debug
-        .time('config.load', () => loadRunConfig(options, cwd, env, cli).then(allocateAppPorts))
+        .time('config.load', () => (processOutput === undefined ? loadConfig() : processOutput.withholdDuring(loadConfig)))
         .then((config) => ({ config, error: undefined }), (cause) => ({ config: undefined, error: classifyError(cause) }));
     // Every reporter, built-in or configured, is one `Reporter` on one spine:
     // the ids name the shipped ones, and a config failure still leaves the ids
     // the CLI asked for, so the failure renders through them.
     const reporterIds = loaded.config?.reporters ?? options.reporters ?? ['list'];
-    const listReporter = options.quiet === true || !reporterIds.includes('list') ? undefined : new ListReporter();
+    const listReporter = options.quiet === true || !reporterIds.includes('list') ? undefined : new ListReporter(processOutput?.listOutput);
+    // Until the run is over, what user code prints lands above the live window.
+    processOutput?.showThrough(listReporter === undefined
+        ? undefined
+        : { show: (stream, text) => listReporter.processOutput(stream, text), end: () => listReporter.endProcessOutput() });
     const activeReporters = [
         ...(listReporter === undefined ? [] : [listReporter]),
         ...reporterIds.filter((id) => id !== 'list').map((id) => STATELESS_REPORTERS[id]),
@@ -182,13 +196,13 @@ export async function run(options = {}) {
         emit({ type: 'run-stopped', failures, limit });
     };
     /**
-     * The exit code is a fold over run state — every result, every run error,
+     * The exit code is a fold over run state — every result and serial group, every run error,
      * the interrupt — never threaded through by hand. A run error recorded
      * anywhere, including during teardown or the report write, reaches the exit
      * code the same way.
      */
     const currentExitCode = () => combineExitCodes([
-        ...resultExitCodes(results),
+        ...verdictExitCodes(results, serialGroups),
         ...(loaded.config?.failOnSkippedFailure === true && someSkippedAfterFailure(results, serialGroups) ? [1] : []),
         ...runErrors.map((runError) => exitCodeForCategory(runError.error.category)),
         ...(interruptController.signal.aborted ? [130] : []),
@@ -293,6 +307,9 @@ export async function run(options = {}) {
             ...(reportPath === undefined ? {} : { reportPath }),
             ...(aiTracePath === undefined ? {} : { aiTracePath }),
         });
+        // The list reporter has printed its summary and stopped its window; a
+        // line another reporter left unfinished on `run-finished` prints too.
+        processOutput?.showThrough(undefined);
         if (debug.enabled) {
             process.stderr.write(debug.summary());
             process.stderr.write(agentStepTable(results, serialGroups));
@@ -425,7 +442,7 @@ export async function run(options = {}) {
         // stream as `notice` events under it, so the reporter prints them instead
         // of a worker's stderr fighting the live status block.
         try {
-            plans = await debug.time('engine.prepare', () => prepareEngines(plans, runWorkers, engines, { runId, projectRoot: config.projectRoot, env, signal: interrupted, notice }, emit));
+            plans = await debug.time('engine.prepare', () => prepareEngines(plans, runWorkers, engines, { runId, projectRoot: config.projectRoot, env, signal: interrupted, headed: options.headed ?? false, notice }, emit));
         }
         catch (cause) {
             if (!interrupted.aborted)
@@ -752,14 +769,20 @@ function plannedFiles(selection) {
 function statusOf(exitCode) {
     return exitCode === 0 ? 'passed' : exitCode === 1 ? 'failed' : exitCode === 130 ? 'interrupted' : 'error';
 }
-function resultExitCodes(results) {
+/**
+ * The exit codes the verdicts imply: a failed test or serial group exits 1,
+ * or its attempts' error category when that is worse. A serial group answers
+ * for its members, whose results carry no attempts, and for a group attempt
+ * that failed before any member ran (a launch), whose members are skipped.
+ */
+function verdictExitCodes(results, serialGroups) {
     const codes = [0];
-    for (const result of results) {
-        switch (result.status) {
+    for (const verdict of [...results, ...serialGroups]) {
+        switch (verdict.status) {
             case 'failed':
             case 'timed-out':
                 codes.push(1);
-                for (const attempt of result.attempts) {
+                for (const attempt of verdict.attempts) {
                     if (attempt.error !== undefined)
                         codes.push(exitCodeForCategory(attempt.error.category));
                 }