@shivaedev/vitest-rerun 0.0.0-stage → 0.1.0

package/src/internal/mergeAttempt.ts

@@ -0,0 +1,105 @@
+import { relative } from "node:path";
+import { type FlakyTest, failureMessage, testcases } from "#flakyTests.ts";
+import { junitClassname } from "#internal/junit.ts";
+import { type FailureHistory, type Files, type Snapshot, type Task, tests } from "#internal/snapshot.ts";
+
+type File = Files[number];
+
+interface Outcome {
+	readonly failures: Record<string, FailureHistory>;
+	readonly flaky: FlakyTest[];
+	readonly root: string;
+	readonly summary: (line: string) => void;
+}
+
+function mergeTest(first: Task, task: Task) {
+	const unresolved = first.result?.state === "fail" && !["pass", "fail"].includes(task.result?.state ?? "");
+	if (first.result !== undefined && (first.result.state === "pass" || unresolved)) {
+		task.mode = first.mode;
+		task.result = first.result;
+		task.logs = first.logs ?? [];
+	} else if (task.result !== undefined) {
+		task.result.duration = (task.result.duration ?? 0) + (first.result?.duration ?? 0);
+	}
+}
+
+function restorePassed(original: Task[], rerun: Task[]) {
+	for (const task of rerun) {
+		const first = original.find((item) => item.id === task.id);
+		if (first === undefined) {
+			throw new Error(`Rerun collected an unexpected test: ${task.name}`);
+		}
+		if (task.type === "suite" && first.type === "suite") {
+			restorePassed(first.tasks, task.tasks);
+		} else {
+			mergeTest(first, task);
+		}
+	}
+}
+
+function finishFile(file: File, next: File, ranTests: boolean) {
+	const duration = (file.result?.duration ?? 0) + (next.result?.duration ?? 0);
+	if (!ranTests && file.tasks.length > 0 && file.result !== undefined) {
+		next.result = { ...file.result };
+	}
+	if (next.result !== undefined) {
+		if (tests(next.tasks).some((test) => test.result?.state === "fail")) {
+			next.result.state = "fail";
+		}
+		next.result.startTime = file.result?.startTime ?? next.result.startTime ?? 0;
+		next.result.duration = duration;
+	}
+}
+
+function recordAttempts(file: File, retried: Task[], outcome: Outcome) {
+	const path = relative(outcome.root, file.filepath);
+	const classname = junitClassname(path, file.projectName ?? "");
+	for (const { name, task } of testcases(file.tasks).filter((testcase) => testcase.task.result?.state === "fail")) {
+		const again = retried.find((item) => item.id === task.id);
+		const prior = outcome.failures[task.id] ?? { attempts: 1, messages: [failureMessage(task)] };
+		if (again?.result?.state === "pass") {
+			delete outcome.failures[task.id];
+			outcome.flaky.push({ attempts: prior.attempts + 1, classname, file: path, messages: prior.messages, name });
+			outcome.summary(`- Flaky: ${classname} — ${name}`);
+		} else if (again?.result?.state === "fail") {
+			outcome.failures[task.id] = { attempts: prior.attempts + 1, messages: [...prior.messages, failureMessage(again)] };
+		}
+	}
+}
+
+function mergeFile(file: File, next: File, outcome: Outcome) {
+	const original = tests(file.tasks);
+	const retried = tests(next.tasks);
+	if (original.some((test) => !retried.some((item) => item.id === test.id))) {
+		throw new Error(`Rerun did not collect every original test in ${file.filepath}`);
+	}
+	const ranTests = retried.some(
+		(test) => ["pass", "fail"].includes(test.result?.state ?? "") && original.some((item) => item.id === test.id && item.result?.state !== "pass"),
+	);
+	recordAttempts(file, retried, outcome);
+	if (original.length === 0 && next.result?.state === "pass") {
+		outcome.summary(`- Flaky: ${junitClassname(relative(outcome.root, file.filepath), file.projectName ?? "")} — collection failure`);
+	} else if (original.length > 0) {
+		restorePassed(file.tasks, next.tasks);
+	}
+	finishFile(file, next, ranTests);
+}
+
+export interface Merged {
+	readonly failures: Readonly<Record<string, FailureHistory>>;
+	readonly files: Files;
+	readonly flaky: readonly FlakyTest[];
+}
+
+export function mergeAttempt(previous: Snapshot, rerun: Files, root: string, summary: (line: string) => void): Merged {
+	const outcome: Outcome = { failures: { ...previous.failures }, flaky: [...previous.flaky], root, summary };
+	const files = previous.files.map((file) => {
+		const next = rerun.find((item) => item.id === file.id);
+		if (next === undefined) {
+			return file;
+		}
+		mergeFile(file, next, outcome);
+		return next;
+	});
+	return { failures: outcome.failures, files, flaky: outcome.flaky };
+}

package/src/internal/reportPaths.ts

@@ -0,0 +1,20 @@
+import { join } from "node:path";
+
+export function reportPaths(outputDir: string) {
+	return {
+		flaky: join(outputDir, "flaky-tests.json"),
+		json: join(outputDir, "tests.json"),
+		junit: join(outputDir, "tests.junit.xml"),
+		rerunCache: join(outputDir, "vite-rerun"),
+	};
+}
+
+export function attemptPaths(outputDir: string, attempt: number) {
+	return {
+		json: join(outputDir, `tests.attempt-${attempt}.json`),
+		junit: join(outputDir, `tests.attempt-${attempt}.junit.xml`),
+		snapshot: join(outputDir, `tests.attempt-${attempt}.bin`),
+	};
+}
+
+export const ATTEMPT_REPORT = /^tests\.attempt-\d+\.(?:json|junit\.xml|bin)$/u;

package/src/internal/runAttempt.ts

@@ -0,0 +1,126 @@
+import { Config, Console, Effect, FileSystem, Schema } from "effect";
+import type { CliOptions, Vitest } from "vitest/node";
+import * as VitestNode from "vitest/node";
+import { failWith } from "#error.ts";
+import { writeFlakyTests } from "#flakyTests.ts";
+import { RerunJunitReporter } from "#internal/junit.ts";
+import { mergeAttempt } from "#internal/mergeAttempt.ts";
+import { attemptPaths, reportPaths } from "#internal/reportPaths.ts";
+import { failedFiles, readSnapshot, type Snapshot, saveSnapshot, tests } from "#internal/snapshot.ts";
+import { TimedJsonReporter } from "#internal/TimedJsonReporter.ts";
+
+export interface Attempt {
+	readonly attempt: number;
+	readonly outputDir: string;
+	readonly vitestArgs: readonly string[];
+}
+
+const CoverageReport = Schema.Struct({ coverageMap: Schema.optional(Schema.Unknown) });
+const decodeCoverage = Schema.decodeUnknownEffect(Schema.fromJsonString(CoverageReport));
+
+function promise<A>(message: string, run: () => Promise<A>) {
+	return Effect.tryPromise({ catch: failWith(message), try: run });
+}
+
+function start(options: CliOptions, reports: { json: string; junit: string }, cacheDir: string | undefined) {
+	return Effect.acquireRelease(
+		promise("Vitest could not start", () =>
+			VitestNode.createVitest(
+				{
+					...options,
+					reporters: ["default", new TimedJsonReporter({ outputFile: reports.json }), new RerunJunitReporter(reports.junit)],
+					retry: 0,
+					watch: false,
+				},
+				cacheDir === undefined ? {} : { cacheDir },
+			),
+		),
+		(vitest) => Effect.promise(() => vitest.close()),
+	);
+}
+
+const failedSpecifications = Effect.fn("VitestRerun.failedSpecifications")(function* (vitest: Vitest, previous: Snapshot, json: string) {
+	const failed = new Set(yield* failedFiles(json));
+	yield* promise("Vitest could not prepare the rerun", () => vitest.standalone());
+	const specifications = yield* promise("Vitest could not list test files", () => vitest.globTestSpecifications());
+	return specifications.flatMap((spec) => {
+		const file = previous.files.find((item) => item.id === spec.taskId && failed.has(spec.moduleId) && item.result?.state === "fail");
+		if (file === undefined) {
+			return [];
+		}
+		const runnable = tests(file.tasks).filter((test) => !["pass", "skip", "todo"].includes(test.result?.state ?? test.mode));
+		return [spec.project.createSpecification(spec.moduleId, file.tasks.length === 0 ? {} : { testIds: runnable.map((test) => test.id) }, spec.pool)];
+	});
+});
+
+const mergeCoverage = Effect.fn("VitestRerun.mergeCoverage")(function* (vitest: Vitest, outputDir: string, attempt: number) {
+	const fs = yield* FileSystem.FileSystem;
+	const maps: unknown[] = [];
+	for (let earlier = 1; earlier <= attempt; earlier += 1) {
+		const { coverageMap } = yield* decodeCoverage(yield* fs.readFileString(attemptPaths(outputDir, earlier).json));
+		if (coverageMap !== undefined) {
+			maps.push(coverageMap);
+		}
+	}
+	const provider = yield* promise("Vitest could not load its coverage provider", () => vitest.createCoverageProvider());
+	yield* promise("Vitest could not merge coverage", async () => provider?.mergeReports?.(maps));
+});
+
+const summaryLine = Effect.fn("VitestRerun.summaryLine")(function* (line: string) {
+	const fs = yield* FileSystem.FileSystem;
+	const summary = yield* Config.String("GITHUB_STEP_SUMMARY").pipe(Config.withDefault(""));
+	yield* Console.log(line);
+	if (summary !== "") {
+		yield* fs.writeFileString(summary, `${line}\n`, { flag: "a" });
+	}
+});
+
+const rerunFailed = Effect.fn("VitestRerun.rerunFailed")(function* (vitest: Vitest, { attempt, outputDir }: Attempt) {
+	const previous = yield* readSnapshot(attemptPaths(outputDir, attempt - 1).snapshot);
+	const final = reportPaths(outputDir);
+	const specifications = yield* failedSpecifications(vitest, previous, final.json);
+	if (specifications.length === 0) {
+		return yield* Effect.fail(failWith("No failed specifications found for the fresh rerun")(undefined));
+	}
+	yield* promise("Vitest could not rerun the failed tests", () => vitest.runTestSpecifications(specifications));
+	const lines: string[] = [];
+	const merged = mergeAttempt(previous, vitest.state.getFiles(), vitest.config.root, (line) => lines.push(line));
+	for (const file of merged.files) {
+		vitest.state.collectFiles(vitest.getProjectByName(file.projectName ?? ""), [file]);
+	}
+	yield* Effect.forEach(lines, summaryLine);
+	for (const reporter of [new TimedJsonReporter({ outputFile: final.json }), new RerunJunitReporter(final.junit)]) {
+		yield* promise("Vitest could not write the merged reports", async () => {
+			await reporter.onInit(vitest);
+			await reporter.onTestRunEnd(vitest.state.getTestModules());
+		});
+	}
+	if (merged.flaky.length > 0) {
+		yield* writeFlakyTests(final.flaky, merged.flaky);
+	}
+	if (vitest.config.coverage.enabled) {
+		yield* mergeCoverage(vitest, outputDir, attempt);
+	}
+	const snapshot: Snapshot = {
+		errors: vitest.state.getUnhandledErrors(),
+		failures: merged.failures,
+		files: vitest.state.getFiles(),
+		flaky: merged.flaky,
+	};
+	return snapshot;
+});
+
+export const runAttempt = Effect.fn("VitestRerun.runAttempt")(function* (input: Attempt) {
+	const { filter, options } = VitestNode.parseCLI(["vitest", "run", ...input.vitestArgs]);
+	const rerun = input.attempt > 1;
+	const { shard: _shard, ...unsharded } = options;
+	const reports = rerun ? attemptPaths(input.outputDir, input.attempt) : reportPaths(input.outputDir);
+	const vitest = yield* start(rerun ? unsharded : options, reports, rerun ? reportPaths(input.outputDir).rerunCache : undefined);
+	const snapshot: Snapshot = rerun
+		? yield* rerunFailed(vitest, input)
+		: yield* promise("Vitest could not run the tests", () => vitest.start(filter)).pipe(
+				Effect.map(() => ({ errors: vitest.state.getUnhandledErrors(), failures: {}, files: vitest.state.getFiles(), flaky: [] })),
+			);
+	yield* saveSnapshot(attemptPaths(input.outputDir, input.attempt).snapshot, snapshot);
+	return snapshot.files.some((file) => file.result?.state === "fail") || snapshot.errors.length > 0;
+}, Effect.scoped);

package/src/internal/snapshot.ts

@@ -0,0 +1,43 @@
+import { deserialize, serialize } from "node:v8";
+import { Effect, FileSystem, Schema } from "effect";
+import type { Vitest } from "vitest/node";
+import type { FlakyTest } from "#flakyTests.ts";
+
+export type Files = ReturnType<Vitest["state"]["getFiles"]>;
+export type Task = Files[number]["tasks"][number];
+
+export interface FailureHistory {
+	readonly attempts: number;
+	readonly messages: readonly string[];
+}
+
+export interface Snapshot {
+	readonly errors: readonly unknown[];
+	readonly failures: Readonly<Record<string, FailureHistory>>;
+	readonly files: Files;
+	readonly flaky: readonly FlakyTest[];
+}
+
+const Report = Schema.Struct({ testResults: Schema.Array(Schema.Struct({ name: Schema.String, status: Schema.Literals(["failed", "passed"]) })) });
+const decodeReport = Schema.decodeUnknownEffect(Schema.fromJsonString(Report));
+
+export const failedFiles = Effect.fn("VitestRerun.failedFiles")(function* (path: string) {
+	const fs = yield* FileSystem.FileSystem;
+	const report = yield* decodeReport(yield* fs.readFileString(path));
+	return [...new Set(report.testResults.filter((file) => file.status === "failed").map((file) => file.name))];
+});
+
+export function tests(tasks: readonly Task[]): Task[] {
+	return tasks.flatMap((task) => (task.type === "suite" ? tests(task.tasks) : [task]));
+}
+
+export const saveSnapshot = Effect.fn("VitestRerun.saveSnapshot")(function* (path: string, snapshot: Snapshot) {
+	const fs = yield* FileSystem.FileSystem;
+	yield* fs.writeFile(path, serialize(snapshot));
+});
+
+export const readSnapshot = Effect.fn("VitestRerun.readSnapshot")(function* (path: string) {
+	const fs = yield* FileSystem.FileSystem;
+	const snapshot: Snapshot = deserialize(yield* fs.readFile(path));
+	return snapshot;
+});

package/src/options.ts

@@ -0,0 +1,47 @@
+import { resolve } from "node:path";
+import { Effect } from "effect";
+import { VitestRerunError } from "#error.ts";
+
+export interface RerunOptions {
+	readonly cwd: string;
+	readonly outputDir: string;
+	readonly reruns: number;
+	readonly vitestArgs: readonly string[];
+}
+
+export const DEFAULT_OUTPUT_DIR = "test-results";
+export const DEFAULT_RERUNS = 1;
+
+function option(arg: string, name: string, next: string | undefined): { value: string; consumed: number } | undefined {
+	if (arg === name && next !== undefined) {
+		return { consumed: 2, value: next };
+	}
+	return arg.startsWith(`${name}=`) ? { consumed: 1, value: arg.slice(name.length + 1) } : undefined;
+}
+
+export function parseOptions(args: readonly string[], cwd: string): Effect.Effect<RerunOptions, VitestRerunError> {
+	let outputDir = DEFAULT_OUTPUT_DIR;
+	let reruns = String(DEFAULT_RERUNS);
+	const vitestArgs: string[] = [];
+	for (let index = 0; index < args.length; ) {
+		const arg = args[index] ?? "";
+		if (arg === "--") {
+			vitestArgs.push(...args.slice(index + 1));
+			break;
+		}
+		const output = option(arg, "--output-dir", args[index + 1]);
+		const count = option(arg, "--reruns", args[index + 1]);
+		if (output !== undefined) {
+			outputDir = output.value;
+		} else if (count === undefined) {
+			vitestArgs.push(arg);
+		} else {
+			reruns = count.value;
+		}
+		index += output?.consumed ?? count?.consumed ?? 1;
+	}
+	if (!/^\d+$/u.test(reruns)) {
+		return Effect.fail(new VitestRerunError({ message: `--reruns expects a whole number, got ${JSON.stringify(reruns)}` }));
+	}
+	return Effect.succeed({ cwd, outputDir: resolve(cwd, outputDir), reruns: Number(reruns), vitestArgs });
+}

package/src/rerun.ts

@@ -0,0 +1,64 @@
+import { extname, join } from "node:path";
+import process from "node:process";
+import { fileURLToPath } from "node:url";
+import { Console, Effect, FileSystem, Option } from "effect";
+import { ChildProcess, ChildProcessSpawner } from "effect/process";
+import { ATTEMPT_REPORT, attemptPaths, reportPaths } from "#internal/reportPaths.ts";
+import { failedFiles, readSnapshot } from "#internal/snapshot.ts";
+import type { RerunOptions } from "#options.ts";
+
+const ATTEMPT_MODULE = fileURLToPath(new URL(`./internal/attempt${extname(fileURLToPath(import.meta.url))}`, import.meta.url));
+
+const removeReports = Effect.fn("VitestRerun.removeReports")(function* (outputDir: string) {
+	const fs = yield* FileSystem.FileSystem;
+	yield* fs.makeDirectory(outputDir, { recursive: true });
+	const { flaky, json, junit } = reportPaths(outputDir);
+	const attempts = (yield* fs.readDirectory(outputDir)).filter((name) => ATTEMPT_REPORT.test(name));
+	yield* Effect.forEach([flaky, json, junit, ...attempts.map((name) => join(outputDir, name))], (path) => fs.remove(path, { force: true }));
+});
+
+const spawnAttempt = Effect.fn("VitestRerun.spawnAttempt")(function* (options: RerunOptions, attempt: number) {
+	const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
+	const command = ChildProcess.make(
+		process.execPath,
+		[...process.execArgv, ATTEMPT_MODULE, String(attempt), options.outputDir, ...options.vitestArgs],
+		{ cwd: options.cwd, extendEnv: true, stderr: "inherit", stdin: "inherit", stdout: "inherit" },
+	);
+	return yield* spawner.exitCode(command).pipe(
+		Effect.map((code) => Option.some(Number(code))),
+		Effect.catch((error) => Effect.as(Console.error(`vitest-rerun: attempt ${attempt} ended without an exit code: ${error.message}`), Option.none())),
+	);
+});
+
+const rerunnable = Effect.fn("VitestRerun.rerunnable")(function* (outputDir: string, attempt: number) {
+	const fs = yield* FileSystem.FileSystem;
+	const { snapshot } = attemptPaths(outputDir, attempt);
+	const { json } = reportPaths(outputDir);
+	if (!((yield* fs.exists(snapshot)) && (yield* fs.exists(json)))) {
+		return false;
+	}
+	return (yield* readSnapshot(snapshot)).errors.length === 0 && (yield* failedFiles(json)).length > 0;
+});
+
+const keepFirstAttempt = Effect.fn("VitestRerun.keepFirstAttempt")(function* (outputDir: string) {
+	const fs = yield* FileSystem.FileSystem;
+	const final = reportPaths(outputDir);
+	const first = attemptPaths(outputDir, 1);
+	yield* fs.copyFile(final.json, first.json);
+	yield* fs.copyFile(final.junit, first.junit);
+});
+
+export const rerun = Effect.fn("VitestRerun.rerun")(function* (options: RerunOptions) {
+	yield* removeReports(options.outputDir);
+	let code = yield* spawnAttempt(options, 1);
+	for (let attempt = 1; attempt <= options.reruns; attempt += 1) {
+		if (!(Option.exists(code, (status) => status !== 0) && (yield* rerunnable(options.outputDir, attempt)))) {
+			break;
+		}
+		if (attempt === 1) {
+			yield* keepFirstAttempt(options.outputDir);
+		}
+		code = yield* spawnAttempt(options, attempt + 1);
+	}
+	return Option.getOrElse(code, () => 1);
+});