e2e 0.16.0 → 0.18.0

package/dist/run/retry.js

@@ -8,28 +8,46 @@ export function isRetryEligible(attempt) {
  * Runs up to `maxAttempts` attempts, stopping on pass, interrupt, or a
  * failure that does not consume retry budget. `runOnce` returns the
  * finished attempt, or undefined when the attempt could not start (the
- * loop stops and the last observed status stands).
+ * loop stops and the last observed status stands). An interrupt during a
+ * retry keeps the verdict the failed attempt before it reached: the test
+ * did fail, and only its second chance was cut short.
  */
 export async function runWithRetries(maxAttempts, interruptSignal, runOnce) {
-    let finalStatus = 'failed';
-    let attemptCount = 0;
+    const attempts = [];
     for (let attemptIndex = 0; attemptIndex < maxAttempts; attemptIndex += 1) {
         if (interruptSignal.aborted)
             break;
         const attempt = await runOnce(attemptIndex);
         if (attempt === undefined)
             break;
-        attemptCount += 1;
-        if (attempt.status === 'passed')
-            return attemptCount > 1 ? 'flaky' : 'passed';
-        if (attempt.status === 'interrupted')
-            return 'interrupted';
+        attempts.push(attempt);
         // A body that skipped itself has decided; a retry would only ask again.
-        if (attempt.status === 'skipped')
-            return 'skipped';
-        finalStatus = attempt.status;
+        if (attempt.status === 'passed' || attempt.status === 'interrupted' || attempt.status === 'skipped')
+            break;
         if (!isRetryEligible(attempt))
             break;
     }
-    return finalStatus;
+    return retryVerdict(attempts);
+}
+/**
+ * The verdict a sequence of finished attempts reaches: a pass is `flaky`
+ * after a failure, an interrupt during a retry keeps the verdict before it,
+ * and a self-skip decides. Also the verdict of attempts a worker finished
+ * before the run's interrupt stopped it, so both read alike.
+ */
+export function retryVerdict(attempts) {
+    let verdict = 'failed';
+    for (const [index, attempt] of attempts.entries()) {
+        switch (attempt.status) {
+            case 'passed':
+                return index > 0 ? 'flaky' : 'passed';
+            case 'interrupted':
+                return index > 0 ? verdict : 'interrupted';
+            case 'skipped':
+                return 'skipped';
+            default:
+                verdict = attempt.status;
+        }
+    }
+    return verdict;
 }

package/dist/run/runner.d.ts

@@ -5,7 +5,8 @@ import { type Shard, type TagMode } from '../collect/select.ts';
 import type { ExploreProgress } from '../explore/progress.ts';
 import { type Report1Document, type ReportExplore } from '../report/build.ts';
 import { type RunEventSink, type RunExitCode, type RunStatus } from './events.ts';
-import type { ResultRecord } from './records.ts';
+import { type ResultRecord } from './records.ts';
+import type { RunnerOutput } from './process-output.ts';
 import type { BuiltinReporter, E2EConfig, RecordingMode } from '../types.ts';
 export interface RunOptions {
     cwd?: string | undefined;
@@ -109,6 +110,13 @@ export interface RunOptions {
     onEvent?: RunEventSink | undefined;
     /** Budget for each reporter's `onRunFinished`, in ms. Only the test harness sets it; there is no flag. */
     reporterTimeout?: number | undefined;
+    /**
+     * The process's stdout and stderr, when the caller claimed them (the CLI):
+     * what user code in this process prints is held while the config loads
+     * and shown through the list reporter while the run reports, and the list
+     * reporter writes to the terminal past it.
+     */
+    processOutput?: RunnerOutput | undefined;
 }
 /**
  * A registration supplied in memory, under the virtual file name the report

package/dist/run/runner.js

@@ -18,11 +18,14 @@ import { writeJsonReport } from '../report/write.js';
 import { createRunEventEmitter, toEventResult } from './events.js';
 import { allocateAppPorts } from './app-ports.js';
 import { inProcessSpawner } from './in-process.js';
+import { someSkippedAfterFailure } from './records.js';
 import { runUnits } from './scheduler.js';
 import { buildWorkPlans, plannedSlots } from './units.js';
 import { SessionStore } from './sessions.js';
 import { outputLayout } from './output.js';
-import { lastFailedIds, readLastRun } from './last-run.js';
+import { registerStaticSecrets } from './secrecy.js';
+import { claimRerunDir, pruneArtifacts } from './artifacts.js';
+import { carryForward, lastFailedIds, readLastRun, reportArtifactPaths } from './last-run.js';
 import { childProcessSpawner } from './worker/handle.js';
 import { setSecretRegistry } from '../secrets.js';
 import { withAbort } from '../internal/time.js';
@@ -129,14 +132,27 @@ export async function run(options = {}) {
     //
     // Free ports for URLs declared with port 0 are chosen here, once: workers
     // re-resolve the config and get the assignments in their bootstrap.
+    //
+    // The config's secrets are known to the process before what its top-level
+    // code printed is released, so that output is redacted like any other.
+    const loadConfig = async () => {
+        const config = await loadRunConfig(options, cwd, env, cli);
+        registerStaticSecrets(config.allSecrets);
+        return allocateAppPorts(config);
+    };
+    const processOutput = options.processOutput;
     const loaded = await debug
-        .time('config.load', () => loadRunConfig(options, cwd, env, cli).then(allocateAppPorts))
+        .time('config.load', () => (processOutput === undefined ? loadConfig() : processOutput.withholdDuring(loadConfig)))
         .then((config) => ({ config, error: undefined }), (cause) => ({ config: undefined, error: classifyError(cause) }));
     // Every reporter, built-in or configured, is one `Reporter` on one spine:
     // the ids name the shipped ones, and a config failure still leaves the ids
     // the CLI asked for, so the failure renders through them.
     const reporterIds = loaded.config?.reporters ?? options.reporters ?? ['list'];
-    const listReporter = options.quiet === true || !reporterIds.includes('list') ? undefined : new ListReporter();
+    const listReporter = options.quiet === true || !reporterIds.includes('list') ? undefined : new ListReporter(processOutput?.listOutput);
+    // Until the run is over, what user code prints lands above the live window.
+    processOutput?.showThrough(listReporter === undefined
+        ? undefined
+        : { show: (stream, text) => listReporter.processOutput(stream, text), end: () => listReporter.endProcessOutput() });
     const activeReporters = [
         ...(listReporter === undefined ? [] : [listReporter]),
         ...reporterIds.filter((id) => id !== 'list').map((id) => STATELESS_REPORTERS[id]),
@@ -180,32 +196,45 @@ export async function run(options = {}) {
         emit({ type: 'run-stopped', failures, limit });
     };
     /**
-     * The exit code is a fold over run state — every result, every run error,
+     * The exit code is a fold over run state — every result and serial group, every run error,
      * the interrupt — never threaded through by hand. A run error recorded
      * anywhere, including during teardown or the report write, reaches the exit
      * code the same way.
      */
     const currentExitCode = () => combineExitCodes([
-        ...resultExitCodes(results),
+        ...verdictExitCodes(results, serialGroups),
+        ...(loaded.config?.failOnSkippedFailure === true && someSkippedAfterFailure(results, serialGroups) ? [1] : []),
         ...runErrors.map((runError) => exitCodeForCategory(runError.error.category)),
         ...(interruptController.signal.aborted ? [130] : []),
     ].filter((code) => code !== 130 || !(runAborted || stoppedEarly)));
     // Detected once the config names the project root; a report written before
     // that (a config failure) has no checkout to describe.
     let vcs;
-    const buildRunReport = (exitCode) => buildReport({
-        runId,
-        config: loaded.config,
-        vcs,
-        startedAt,
-        status: statusOf(exitCode),
-        exitCode,
-        results,
-        serialGroups,
-        runErrors,
-        targetProvenance,
-        explore: options.tests?.explore?.snapshot(),
-    });
+    // The report `--last-failed` selected from, once collection has read it;
+    // reporters get it beside this run's report to fold the rerun into it, and
+    // this run's report carries what it owed that this run left out.
+    let lastRun;
+    // The ids of every test collection found, a setup no selected test needed
+    // among them, and the files it could not collect: what a rerun can still
+    // carry when the report lists no row.
+    let rerunCollection = { testIds: new Set(), uncollectedFiles: new Set() };
+    const buildRunReport = (exitCode) => {
+        const document = buildReport({
+            runId,
+            config: loaded.config,
+            vcs,
+            startedAt,
+            status: statusOf(exitCode),
+            exitCode,
+            results,
+            serialGroups,
+            runErrors,
+            targetProvenance,
+            explore: options.tests?.explore?.snapshot(),
+        });
+        const carried = lastRun === undefined ? undefined : carryForward(lastRun, document, rerunCollection);
+        return carried === undefined ? document : { ...document, run: { ...document.run, carried } };
+    };
     /**
      * Writes the canonical report and returns its path only once the file
      * exists: a host must never be handed a path to a report that was not
@@ -249,9 +278,6 @@ export async function run(options = {}) {
             return undefined;
         }
     };
-    // The report `--last-failed` selected from, once collection has read it;
-    // reporters get it beside this run's report to fold the rerun into it.
-    let lastRun;
     /**
      * Whether the run got as far as its tests. Only such a run writes into the
      * output directory: one that stopped before leaves the previous run's
@@ -281,6 +307,9 @@ export async function run(options = {}) {
             ...(reportPath === undefined ? {} : { reportPath }),
             ...(aiTracePath === undefined ? {} : { aiTracePath }),
         });
+        // The list reporter has printed its summary and stopped its window; a
+        // line another reporter left unfinished on `run-finished` prints too.
+        processOutput?.showThrough(undefined);
         if (debug.enabled) {
             process.stderr.write(debug.summary());
             process.stderr.write(agentStepTable(results, serialGroups));
@@ -364,6 +393,10 @@ export async function run(options = {}) {
                     : collectInMemory(config.projectRoot, options.tests.file, options.tests.registration, consumesSession(options.tests.registration) ? await collectSetups(config) : undefined);
                 const inputs = await selectionInputs(options, config);
                 lastRun = inputs.lastRun;
+                rerunCollection = {
+                    testIds: new Set(collection.tests.map((test) => test.id)),
+                    uncollectedFiles: new Set(collection.uncollected.map((skipped) => skipped.file)),
+                };
                 const selection = repeatEach(select(collection, config, inputs.filters, options.passWithNoTests !== undefined ? { passWithNoTests: options.passWithNoTests } : {}), options.repeatEach ?? 1);
                 return { collection, selection };
             }));
@@ -409,7 +442,7 @@ export async function run(options = {}) {
         // stream as `notice` events under it, so the reporter prints them instead
         // of a worker's stderr fighting the live status block.
         try {
-            plans = await debug.time('engine.prepare', () => prepareEngines(plans, runWorkers, engines, { runId, projectRoot: config.projectRoot, env, signal: interrupted, notice }, emit));
+            plans = await debug.time('engine.prepare', () => prepareEngines(plans, runWorkers, engines, { runId, projectRoot: config.projectRoot, env, signal: interrupted, headed: options.headed ?? false, notice }, emit));
         }
         catch (cause) {
             if (!interrupted.aborted)
@@ -450,15 +483,26 @@ export async function run(options = {}) {
         // Tests are about to start, and only now is the previous run's evidence
         // given up: the artifact tree is emptied, so what is there once this run
         // ends is its own and nothing a report no longer names, and this run's
-        // report replaces the last. A run that stopped before here (no test
-        // selected, a collection error, a target that cannot record what it asks,
-        // an app that failed to start, an interrupt) leaves both, and
+        // report replaces the last. A `--last-failed` rerun keeps the evidence
+        // the report it reruns names instead, since that report's results fold
+        // into the rerun's and its carried tests do not run again, and files its
+        // own attempts in a fresh `rerun-<n>` directory beside it, so no attempt
+        // overwrites an earlier one's files. A run that stopped before here (no
+        // test selected, a collection error, a target that cannot record what it
+        // asks, an app that failed to start, an interrupt) leaves both, and
         // `--last-failed` still reads the run that executed. A wipe that fails
         // part way has already given up the old evidence, so this run's report
         // records the failure.
         testsStarted = true;
+        let rerunDir;
         try {
-            await rm(layout.artifacts, { recursive: true, force: true });
+            if (lastRun === undefined) {
+                await rm(layout.artifacts, { recursive: true, force: true });
+            }
+            else {
+                await pruneArtifacts(layout.artifacts, reportArtifactPaths(lastRun));
+                rerunDir = await claimRerunDir(layout.artifacts);
+            }
         }
         catch (cause) {
             recordFailure(cause, 'launch');
@@ -482,6 +526,7 @@ export async function run(options = {}) {
                 inMemory: options.tests,
                 runId,
                 artifactsRoot,
+                rerunDir,
                 sessionStore: store,
                 headed: options.headed ?? false,
                 debug,
@@ -495,6 +540,7 @@ export async function run(options = {}) {
                 ports: config.ports,
                 runId,
                 artifactsRoot,
+                rerunDir,
                 headed: options.headed ?? false,
                 sessionsRoot: layout.sessions,
                 sessionKeyBase64: store.exportKeyForWorker(),
@@ -723,14 +769,20 @@ function plannedFiles(selection) {
 function statusOf(exitCode) {
     return exitCode === 0 ? 'passed' : exitCode === 1 ? 'failed' : exitCode === 130 ? 'interrupted' : 'error';
 }
-function resultExitCodes(results) {
+/**
+ * The exit codes the verdicts imply: a failed test or serial group exits 1,
+ * or its attempts' error category when that is worse. A serial group answers
+ * for its members, whose results carry no attempts, and for a group attempt
+ * that failed before any member ran (a launch), whose members are skipped.
+ */
+function verdictExitCodes(results, serialGroups) {
     const codes = [0];
-    for (const result of results) {
-        switch (result.status) {
+    for (const verdict of [...results, ...serialGroups]) {
+        switch (verdict.status) {
             case 'failed':
             case 'timed-out':
                 codes.push(1);
-                for (const attempt of result.attempts) {
+                for (const attempt of verdict.attempts) {
                     if (attempt.error !== undefined)
                         codes.push(exitCodeForCategory(attempt.error.category));
                 }
@@ -747,13 +799,21 @@ function resultExitCodes(results) {
 /** Why a reporter's `onRunFinished` was given up on: the message is the stderr line. */
 class ReporterAbandoned extends Error {
 }
+/**
+ * The built-in reporters that write files beside `report.json`. A forced
+ * interrupt never abandons them: they are local writes, and giving one up
+ * would leave the previous run's `summary.md` or `junit.xml` beside this
+ * run's report, where CI reads it as this run's.
+ */
+const FILE_REPORTERS = new Set([STATELESS_REPORTERS.junit, STATELESS_REPORTERS.markdown]);
 /**
  * Awaits every reporter's `onRunFinished` at once, each within `timeoutMs`
- * and until a forced interrupt, and collects the summary rows they resolve
- * with. Each reporter gets a signal that aborts on either, so a well-behaved
- * one cancels its own work and leaves no handle holding the process. A
- * reporter that throws, runs out of time, or returns something other than
- * rows is one line on stderr; it can never change the run's outcome.
+ * and, except the built-in file reporters, until a forced interrupt, and
+ * collects the summary rows they resolve with. Each reporter gets a signal
+ * that aborts on either, so a well-behaved one cancels its own work and
+ * leaves no handle holding the process. A reporter that throws, runs out of
+ * time, or returns something other than rows is one line on stderr; it can
+ * never change the run's outcome.
  */
 async function runReporters(reporters, finished, timeoutMs, force) {
     const warn = (reporter, detail) => {
@@ -767,10 +827,12 @@ async function runReporters(reporters, finished, timeoutMs, force) {
         const abandon = new AbortController();
         const timer = setTimeout(() => abandon.abort(new ReporterAbandoned(`did not finish within ${budget}`)), timeoutMs);
         const onForce = () => abandon.abort(new ReporterAbandoned('abandoned: the run was forced to stop'));
-        if (force.aborted)
-            onForce();
-        else
-            force.addEventListener('abort', onForce, { once: true });
+        if (!FILE_REPORTERS.has(reporter)) {
+            if (force.aborted)
+                onForce();
+            else
+                force.addEventListener('abort', onForce, { once: true });
+        }
         try {
             const result = await withAbort(() => Promise.resolve(onRunFinished.call(reporter, finished, abandon.signal)), abandon.signal, () => abandon.signal.reason);
             if (result === undefined)