@vitest/runner 5.0.0-beta.3 → 5.0.0-beta.4

package/dist/chunk-artifact.js

@@ -91,7 +91,8 @@ class TestFixtures {
 		"onTestFailed",
 		"onTestFinished",
 		"skip",
-		"annotate"
+		"annotate",
+		"bench"
 	];
 	static _fixtureOptionKeys = [
 		"auto",
@@ -937,6 +938,8 @@ function createTestHook(name, handler) {
 */
 function interpretTaskModes(file, namePattern, testLocations, testIds, testTagsFilter, onlyMode, parentIsOnly, allowOnly) {
 	const matchedLocations = [];
+	const testLocationsSet = testLocations !== undefined && testLocations.length !== 0 ? new Set(testLocations) : undefined;
+	const testIdsSet = testIds ? new Set(testIds) : undefined;
 	const traverseSuite = (suite, parentIsOnly, parentMatchedWithLocation) => {
 		const suiteIsOnly = parentIsOnly || suite.mode === "only";
 		// Check if any tasks in this suite have `.only` - if so, only those should run
@@ -963,8 +966,8 @@ function interpretTaskModes(file, namePattern, testLocations, testIds, testTagsF
 			// Match test location against provided locations, only run if present
 			// in `testLocations`. Note: if `includeTaskLocation` is not enabled,
 			// all test will be skipped.
-			if (testLocations !== undefined && testLocations.length !== 0) {
-				if (t.location && testLocations?.includes(t.location.line)) {
+			if (testLocationsSet !== undefined) {
+				if (t.location && testLocationsSet.has(t.location.line)) {
 					t.mode = "run";
 					matchedLocations.push(t.location.line);
 					hasLocationMatch = true;
@@ -978,7 +981,7 @@ function interpretTaskModes(file, namePattern, testLocations, testIds, testTagsF
 				if (namePattern && !getTaskFullName(t).match(namePattern)) {
 					t.mode = "skip";
 				}
-				if (testIds && !testIds.includes(t.id)) {
+				if (testIdsSet && !testIdsSet.has(t.id)) {
 					t.mode = "skip";
 				}
 				if (testTagsFilter && !testTagsFilter(t.tags || [])) {
@@ -1796,6 +1799,7 @@ function createSuiteCollector(name, factory = () => {}, mode, each, suiteOptions
 			meta: testMeta,
 			annotations: [],
 			artifacts: [],
+			benchmarks: [],
 			tags: testTags
 		};
 		const handler = options.handler;
@@ -2512,7 +2516,7 @@ async function collectTests(specs, runner) {
 				setHooks(file, fileHooks);
 				file.collectDuration = now$1() - collectStart;
 			} catch (e) {
-				const errors = e instanceof AggregateError ? e.errors.map((e) => processError(e, runner.config.diffOptions)) : [processError(e, runner.config.diffOptions)];
+				const errors = e instanceof AggregateError ? e.errors.map((e) => processError(e, runner.config._diffOptions)) : [processError(e, runner.config._diffOptions)];
 				file.result = {
 					state: "fail",
 					errors
@@ -2623,14 +2627,14 @@ async function callTestHooks(runner, test, hooks, sequence) {
 		try {
 			await Promise.all(hooks.map((fn) => limitMaxConcurrency(() => fn(test.context))));
 		} catch (e) {
-			failTask(test.result, e, runner.config.diffOptions);
+			failTask(test.result, e, runner.config._diffOptions);
 		}
 	} else {
 		for (const fn of hooks) {
 			try {
 				await limitMaxConcurrency(() => fn(test.context));
 			} catch (e) {
-				failTask(test.result, e, runner.config.diffOptions);
+				failTask(test.result, e, runner.config._diffOptions);
 			}
 		}
 	}
@@ -2995,12 +2999,12 @@ async function runTest(test, runner) {
 						test.result.state = "pass";
 					}
 				} catch (e) {
-					failTask(test.result, e, runner.config.diffOptions);
+					failTask(test.result, e, runner.config._diffOptions);
 				}
 				try {
 					await runner.onTaskFinished?.(test);
 				} catch (e) {
-					failTask(test.result, e, runner.config.diffOptions);
+					failTask(test.result, e, runner.config._diffOptions);
 				}
 				try {
 					await $("test.afterEach", () => callSuiteHook(suite, test, "afterEach", runner, [test.context, suite]));
@@ -3011,7 +3015,7 @@ async function runTest(test, runner) {
 					// Fixtures created for aroundEach will be cleaned up after aroundEach teardown
 					await callFixtureCleanupFrom(test.context, fixtureCheckpoint);
 				} catch (e) {
-					failTask(test.result, e, runner.config.diffOptions);
+					failTask(test.result, e, runner.config._diffOptions);
 				}
 				if (test.onFinished?.length) {
 					await $("test.onFinished", () => callTestHooks(runner, test, test.onFinished, "stack"));
@@ -3026,14 +3030,14 @@ async function runTest(test, runner) {
 					repeats: repeatCount
 				});
 			}).catch((error) => {
-				failTask(test.result, error, runner.config.diffOptions);
+				failTask(test.result, error, runner.config._diffOptions);
 			});
 			// Clean up fixtures that were created for aroundEach (before the checkpoint)
 			// This runs after aroundEach teardown has completed
 			try {
 				await callFixtureCleanup(test.context);
 			} catch (e) {
-				failTask(test.result, e, runner.config.diffOptions);
+				failTask(test.result, e, runner.config._diffOptions);
 			}
 			// skipped with new PendingError
 			if (test.result?.pending || test.result?.state === "skip") {
@@ -3167,7 +3171,7 @@ async function runSuite(suite, runner) {
 					try {
 						beforeAllCleanups = await $("suite.beforeAll", () => callSuiteHook(suite, suite, "beforeAll", runner, [suite]));
 					} catch (e) {
-						failTask(suite.result, e, runner.config.diffOptions);
+						failTask(suite.result, e, runner.config._diffOptions);
 						markTasksAsSkipped(suite, runner);
 						return;
 					}
@@ -3206,7 +3210,7 @@ async function runSuite(suite, runner) {
 							await Promise.all(contexts.map((context) => callFixtureCleanup(context)));
 						}
 					} catch (e) {
-						failTask(suite.result, e, runner.config.diffOptions);
+						failTask(suite.result, e, runner.config._diffOptions);
 					}
 				}
 			});
@@ -3215,7 +3219,7 @@ async function runSuite(suite, runner) {
 			if (!suiteRan) {
 				markTasksAsSkipped(suite, runner);
 			}
-			failTask(suite.result, e, runner.config.diffOptions);
+			failTask(suite.result, e, runner.config._diffOptions);
 		}
 		if (suite.mode === "run" || suite.mode === "queued") {
 			if (!runner.config.passWithNoTests && !hasTests(suite)) {

package/dist/index.d.ts

@@ -1,7 +1,7 @@
-import { d as TestArtifact, b as Test, S as Suite, e as SuiteHooks, f as FileSpecification, g as VitestRunner, F as File, h as TaskUpdateEvent, T as Task, i as TestAPI, j as SuiteAPI, k as SuiteCollector } from './tasks.d-B4a0-8OW.js';
-export { A as AfterAllListener, l as AfterEachListener, m as AroundAllListener, n as AroundEachListener, B as BeforeAllListener, o as BeforeEachListener, p as BrowserTraceArtifact, q as CancelReason, r as FailureScreenshotArtifact, s as Fixture, t as FixtureFn, u as FixtureOptions, v as Fixtures, I as ImportDuration, w as InferFixturesTypes, O as OnTestFailedHandler, x as OnTestFinishedHandler, R as Retry, y as RunMode, z as RuntimeContext, D as SequenceHooks, E as SequenceSetupFiles, G as SerializableRetry, H as SuiteFactory, J as SuiteOptions, K as TaskBase, L as TaskCustomOptions, M as TaskEventPack, N as TaskHook, P as TaskMeta, Q as TaskPopulated, U as TaskResult, W as TaskResultPack, X as TaskState, Y as TestAnnotation, Z as TestAnnotationArtifact, _ as TestAnnotationLocation, $ as TestArtifactBase, a0 as TestArtifactLocation, a1 as TestArtifactRegistry, a2 as TestAttachment, a3 as TestContext, a4 as TestFunction, a5 as TestOptions, a as TestTagDefinition, a6 as TestTags, a7 as Use, a8 as VisualRegressionArtifact, V as VitestRunnerConfig, a9 as VitestRunnerConstructor, aa as VitestRunnerImportSource, ab as afterAll, ac as afterEach, ad as aroundAll, ae as aroundEach, af as beforeAll, ag as beforeEach, ah as onTestFailed, ai as onTestFinished } from './tasks.d-B4a0-8OW.js';
+import { d as TestArtifact, b as Test, S as Suite, e as SuiteHooks, f as FileSpecification, g as VitestRunner, F as File, h as TaskUpdateEvent, T as Task, i as TestAPI, j as SuiteAPI, k as SuiteCollector } from './tasks.d-BR8WBtGa.js';
+export { A as AfterAllListener, l as AfterEachListener, m as AroundAllListener, n as AroundEachListener, B as BaselineData, o as BeforeAllListener, p as BeforeEachListener, q as BrowserTraceArtifact, r as CancelReason, s as FailureScreenshotArtifact, t as Fixture, u as FixtureFn, v as FixtureOptions, w as Fixtures, I as ImportDuration, x as InferFixturesTypes, O as OnTestFailedHandler, y as OnTestFinishedHandler, R as Retry, z as RunMode, D as RuntimeContext, E as SequenceHooks, G as SequenceSetupFiles, H as SerializableRetry, J as SuiteFactory, K as SuiteOptions, L as TaskBase, M as TaskCustomOptions, N as TaskEventData, P as TaskEventPack, Q as TaskHook, U as TaskMeta, W as TaskPopulated, X as TaskResult, Y as TaskResultPack, Z as TaskState, _ as TestAnnotation, $ as TestAnnotationArtifact, a0 as TestAnnotationLocation, a1 as TestArtifactBase, a2 as TestArtifactLocation, a3 as TestArtifactRegistry, a4 as TestAttachment, a5 as TestBenchmark, a6 as TestBenchmarkTask, a7 as TestContext, a8 as TestFunction, a9 as TestOptions, a as TestTagDefinition, aa as TestTags, ab as TestTryOptions, ac as Use, ad as VisualRegressionArtifact, V as VitestRunnerConfig, ae as VitestRunnerConstructor, af as VitestRunnerImportSource, ag as afterAll, ah as afterEach, ai as aroundAll, aj as aroundEach, ak as beforeAll, al as beforeEach, am as onTestFailed, an as onTestFinished } from './tasks.d-BR8WBtGa.js';
 import { Awaitable } from '@vitest/utils';
-import '@vitest/utils/diff';
+import 'tinybench';
 
 /**
 * @experimental

package/dist/{tasks.d-B4a0-8OW.d.ts → tasks.d-BR8WBtGa.d.ts}

@@ -1,5 +1,5 @@
 import { TestError, Awaitable } from '@vitest/utils';
-import { DiffOptions } from '@vitest/utils/diff';
+import { Statistics } from 'tinybench';
 
 /**
 * This is a subset of Vitest config that's required for the runner to work.
@@ -27,7 +27,6 @@ interface VitestRunnerConfig {
 	hookTimeout: number;
 	retry: SerializableRetry;
 	includeTaskLocation: boolean | undefined;
-	diffOptions?: DiffOptions;
 	tags: TestTagDefinition[];
 	tagsFilter: string[] | undefined;
 	strictTags: boolean;
@@ -65,6 +64,10 @@ interface VitestRunnerConstructor {
 	new (config: VitestRunnerConfig): VitestRunner;
 }
 type CancelReason = "keyboard-input" | "test-failure" | (string & Record<string, never>);
+interface TestTryOptions {
+	retry: number;
+	repeats: number;
+}
 interface VitestRunner {
 	/**
 	* First thing that's getting called before actually collecting and running tests.
@@ -91,10 +94,7 @@ interface VitestRunner {
 	/**
 	* Called before actually running the test function. Already has "result" with "state" and "startTime".
 	*/
-	onBeforeTryTask?: (test: Test, options: {
-		retry: number;
-		repeats: number;
-	}) => unknown;
+	onBeforeTryTask?: (test: Test, options: TestTryOptions) => unknown;
 	/**
 	* When the task has finished running, but before cleanup hooks are called
 	*/
@@ -106,18 +106,12 @@ interface VitestRunner {
 	/**
 	* Called right after running the test function. Doesn't have new state yet. Will not be called, if the test function throws.
 	*/
-	onAfterTryTask?: (test: Test, options: {
-		retry: number;
-		repeats: number;
-	}) => unknown;
+	onAfterTryTask?: (test: Test, options: TestTryOptions) => unknown;
 	/**
 	* Called after the retry resolution happened. Unlike `onAfterTryTask`, the test now has a new state.
 	* All `after` hooks were also called by this point.
 	*/
-	onAfterRetryTask?: (test: Test, options: {
-		retry: number;
-		repeats: number;
-	}) => unknown;
+	onAfterRetryTask?: (test: Test, options: TestTryOptions) => unknown;
 	/**
 	* Called before running a single suite. Doesn't have "result" yet.
 	*/
@@ -674,6 +668,39 @@ interface Test<ExtraContext = object> extends TaskPopulated {
 	*/
 	artifacts: TestArtifact[];
 	fullTestName: string;
+	/**
+	* An array of benchmark results generated by `context.bench` function.
+	* The benchmark is added only after `bench().run` or `bench.compare` is resolved.
+	*
+	* @experimental
+	*/
+	benchmarks: TestBenchmark[];
+}
+interface TestBenchmark {
+	name: string;
+	tasks: TestBenchmarkTask[];
+}
+type TestBenchmarkStatistics = Statistics;
+interface BaselineData {
+	latency: TestBenchmarkStatistics;
+	throughput: TestBenchmarkStatistics;
+	period: number;
+	totalTime: number;
+}
+interface TestBenchmarkTask {
+	name: string;
+	latency: TestBenchmarkStatistics;
+	throughput: TestBenchmarkStatistics;
+	period: number;
+	totalTime: number;
+	rank: number;
+	perProject?: boolean;
+	/**
+	* `true` when the task was produced by `bench.from()` rather than by
+	* executing a function. Reporters can use this to render the row as a
+	* static reference (no margin of error, no samples).
+	*/
+	fromStore?: boolean;
 }
 type Task = Test | Suite | File;
 type TestFunction<ExtraContext = object> = (context: TestContext & ExtraContext) => Awaitable<any> | void;
@@ -1325,13 +1352,13 @@ type SequenceSetupFiles = "list" | "parallel";
 /**
 * Represents a file or data attachment associated with a test artifact.
 *
-* Attachments can be either file-based (via `path`) or inline content (via `body`).
+* Attachments can be either path-based (via `path`) or inline content (via `body`).
 * The `contentType` helps consumers understand how to interpret the attachment data.
 */
 interface TestAttachment {
 	/** MIME type of the attachment (e.g., 'image/png', 'text/plain') */
 	contentType?: string;
-	/** File system path to the attachment */
+	/** Local file path or external HTTP(S) URL to the attachment. Relative paths are resolved from the project root. */
 	path?: string;
 	/** Inline attachment content as a string or raw binary data */
 	body?: string | Uint8Array | undefined;
@@ -1522,5 +1549,5 @@ interface TestArtifactRegistry {}
 */
 type TestArtifact = BrowserTraceArtifact | FailureScreenshotArtifact | TestAnnotationArtifact | VisualRegressionArtifact | TestArtifactRegistry[keyof TestArtifactRegistry];
 
-export { afterAll as ab, afterEach as ac, aroundAll as ad, aroundEach as ae, beforeAll as af, beforeEach as ag, onTestFailed as ah, onTestFinished as ai, createChainable as c };
-export type { TestArtifactBase as $, AfterAllListener as A, BeforeAllListener as B, ChainableFunction as C, SequenceHooks as D, SequenceSetupFiles as E, File as F, SerializableRetry as G, SuiteFactory as H, ImportDuration as I, SuiteOptions as J, TaskBase as K, TaskCustomOptions as L, TaskEventPack as M, TaskHook as N, OnTestFailedHandler as O, TaskMeta as P, TaskPopulated as Q, Retry as R, Suite as S, Task as T, TaskResult as U, VitestRunnerConfig as V, TaskResultPack as W, TaskState as X, TestAnnotation as Y, TestAnnotationArtifact as Z, TestAnnotationLocation as _, TestTagDefinition as a, TestArtifactLocation as a0, TestArtifactRegistry as a1, TestAttachment as a2, TestContext as a3, TestFunction as a4, TestOptions as a5, TestTags as a6, Use as a7, VisualRegressionArtifact as a8, VitestRunnerConstructor as a9, VitestRunnerImportSource as aa, Test as b, TestArtifact as d, SuiteHooks as e, FileSpecification as f, VitestRunner as g, TaskUpdateEvent as h, TestAPI as i, SuiteAPI as j, SuiteCollector as k, AfterEachListener as l, AroundAllListener as m, AroundEachListener as n, BeforeEachListener as o, BrowserTraceArtifact as p, CancelReason as q, FailureScreenshotArtifact as r, Fixture as s, FixtureFn as t, FixtureOptions as u, Fixtures as v, InferFixturesTypes as w, OnTestFinishedHandler as x, RunMode as y, RuntimeContext as z };
+export { afterAll as ag, afterEach as ah, aroundAll as ai, aroundEach as aj, beforeAll as ak, beforeEach as al, onTestFailed as am, onTestFinished as an, createChainable as c };
+export type { TestAnnotationArtifact as $, AfterAllListener as A, BaselineData as B, ChainableFunction as C, RuntimeContext as D, SequenceHooks as E, File as F, SequenceSetupFiles as G, SerializableRetry as H, ImportDuration as I, SuiteFactory as J, SuiteOptions as K, TaskBase as L, TaskCustomOptions as M, TaskEventData as N, OnTestFailedHandler as O, TaskEventPack as P, TaskHook as Q, Retry as R, Suite as S, Task as T, TaskMeta as U, VitestRunnerConfig as V, TaskPopulated as W, TaskResult as X, TaskResultPack as Y, TaskState as Z, TestAnnotation as _, TestTagDefinition as a, TestAnnotationLocation as a0, TestArtifactBase as a1, TestArtifactLocation as a2, TestArtifactRegistry as a3, TestAttachment as a4, TestBenchmark as a5, TestBenchmarkTask as a6, TestContext as a7, TestFunction as a8, TestOptions as a9, TestTags as aa, TestTryOptions as ab, Use as ac, VisualRegressionArtifact as ad, VitestRunnerConstructor as ae, VitestRunnerImportSource as af, Test as b, TestArtifact as d, SuiteHooks as e, FileSpecification as f, VitestRunner as g, TaskUpdateEvent as h, TestAPI as i, SuiteAPI as j, SuiteCollector as k, AfterEachListener as l, AroundAllListener as m, AroundEachListener as n, BeforeAllListener as o, BeforeEachListener as p, BrowserTraceArtifact as q, CancelReason as r, FailureScreenshotArtifact as s, Fixture as t, FixtureFn as u, FixtureOptions as v, Fixtures as w, InferFixturesTypes as x, OnTestFinishedHandler as y, RunMode as z };

package/dist/types.d.ts

@@ -1,3 +1,3 @@
-export { A as AfterAllListener, l as AfterEachListener, m as AroundAllListener, n as AroundEachListener, B as BeforeAllListener, o as BeforeEachListener, p as BrowserTraceArtifact, q as CancelReason, r as FailureScreenshotArtifact, F as File, f as FileSpecification, s as Fixture, t as FixtureFn, u as FixtureOptions, v as Fixtures, I as ImportDuration, w as InferFixturesTypes, O as OnTestFailedHandler, x as OnTestFinishedHandler, R as Retry, y as RunMode, z as RuntimeContext, D as SequenceHooks, E as SequenceSetupFiles, G as SerializableRetry, S as Suite, j as SuiteAPI, k as SuiteCollector, H as SuiteFactory, e as SuiteHooks, J as SuiteOptions, T as Task, K as TaskBase, L as TaskCustomOptions, M as TaskEventPack, N as TaskHook, P as TaskMeta, Q as TaskPopulated, U as TaskResult, W as TaskResultPack, X as TaskState, h as TaskUpdateEvent, b as Test, i as TestAPI, Y as TestAnnotation, Z as TestAnnotationArtifact, _ as TestAnnotationLocation, d as TestArtifact, $ as TestArtifactBase, a0 as TestArtifactLocation, a1 as TestArtifactRegistry, a2 as TestAttachment, a3 as TestContext, a4 as TestFunction, a5 as TestOptions, a as TestTagDefinition, a6 as TestTags, a7 as Use, a8 as VisualRegressionArtifact, g as VitestRunner, V as VitestRunnerConfig, a9 as VitestRunnerConstructor, aa as VitestRunnerImportSource } from './tasks.d-B4a0-8OW.js';
+export { A as AfterAllListener, l as AfterEachListener, m as AroundAllListener, n as AroundEachListener, B as BaselineData, o as BeforeAllListener, p as BeforeEachListener, q as BrowserTraceArtifact, r as CancelReason, s as FailureScreenshotArtifact, F as File, f as FileSpecification, t as Fixture, u as FixtureFn, v as FixtureOptions, w as Fixtures, I as ImportDuration, x as InferFixturesTypes, O as OnTestFailedHandler, y as OnTestFinishedHandler, R as Retry, z as RunMode, D as RuntimeContext, E as SequenceHooks, G as SequenceSetupFiles, H as SerializableRetry, S as Suite, j as SuiteAPI, k as SuiteCollector, J as SuiteFactory, e as SuiteHooks, K as SuiteOptions, T as Task, L as TaskBase, M as TaskCustomOptions, N as TaskEventData, P as TaskEventPack, Q as TaskHook, U as TaskMeta, W as TaskPopulated, X as TaskResult, Y as TaskResultPack, Z as TaskState, h as TaskUpdateEvent, b as Test, i as TestAPI, _ as TestAnnotation, $ as TestAnnotationArtifact, a0 as TestAnnotationLocation, d as TestArtifact, a1 as TestArtifactBase, a2 as TestArtifactLocation, a3 as TestArtifactRegistry, a4 as TestAttachment, a5 as TestBenchmark, a6 as TestBenchmarkTask, a7 as TestContext, a8 as TestFunction, a9 as TestOptions, a as TestTagDefinition, aa as TestTags, ab as TestTryOptions, ac as Use, ad as VisualRegressionArtifact, g as VitestRunner, V as VitestRunnerConfig, ae as VitestRunnerConstructor, af as VitestRunnerImportSource } from './tasks.d-BR8WBtGa.js';
 import '@vitest/utils';
-import '@vitest/utils/diff';
+import 'tinybench';

package/dist/utils.d.ts

@@ -1,7 +1,7 @@
-import { S as Suite, F as File, T as Task, a as TestTagDefinition, V as VitestRunnerConfig, b as Test } from './tasks.d-B4a0-8OW.js';
-export { C as ChainableFunction, c as createChainable } from './tasks.d-B4a0-8OW.js';
+import { S as Suite, F as File, T as Task, a as TestTagDefinition, V as VitestRunnerConfig, b as Test } from './tasks.d-BR8WBtGa.js';
+export { C as ChainableFunction, c as createChainable } from './tasks.d-BR8WBtGa.js';
 import { ParsedStack, Arrayable } from '@vitest/utils';
-import '@vitest/utils/diff';
+import 'tinybench';
 
 /**
 * If any tasks been marked as `only`, mark all other tasks as `skip`.

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@vitest/runner",
   "type": "module",
-  "version": "5.0.0-beta.3",
+  "version": "5.0.0-beta.4",
   "description": "Vitest test runner",
   "license": "MIT",
   "funding": "https://opencollective.com/vitest",
@@ -44,7 +44,8 @@
   ],
   "dependencies": {
     "pathe": "^2.0.3",
-    "@vitest/utils": "5.0.0-beta.3"
+    "tinybench": "^6.0.1",
+    "@vitest/utils": "5.0.0-beta.4"
   },
   "scripts": {
     "build": "premove dist && rollup -c",