@epicat/toon-reporter 0.0.5 → 0.0.7

package/README.md

@@ -1,6 +1,6 @@
 # toon-reporter
 
-A minimal Vitest reporter optimized for LLM consumption. Outputs test results in a compact, token-efficient format.
+A minimal Vitest and Playwright reporter optimized for LLM consumption. Outputs test results in a compact, token-efficient format.
 
 ## Installation
 
@@ -10,13 +10,15 @@ npm install @epicat/toon-reporter
 
 ## Usage
 
-### CLI
+### Vitest
+
+#### CLI
 
 ```bash
 npx vitest run --reporter=@epicat/toon-reporter
 ```
 
-### Config
+#### Config
 
 ```ts
 // vitest.config.ts
@@ -29,6 +31,34 @@ export default defineConfig({
 })
 ```
 
+### Playwright
+
+#### Config
+
+```ts
+// playwright.config.ts
+import { defineConfig } from '@playwright/test'
+
+export default defineConfig({
+  reporter: [['@epicat/toon-reporter/playwright']],
+})
+```
+
+#### With options
+
+```ts
+// playwright.config.ts
+import { defineConfig } from '@playwright/test'
+
+export default defineConfig({
+  reporter: [
+    ['@epicat/toon-reporter/playwright', {
+      outputFile: 'test-results.toon'
+    }]
+  ],
+})
+```
+
 ## Output Format
 
 ### All tests passing
@@ -37,7 +67,7 @@ export default defineConfig({
 passing: 42
 ```
 
-### With failures
+### With failures (Vitest)
 
 ```
 passing: 40
@@ -49,7 +79,15 @@ failing[2]:
     error: TypeError: Cannot read property 'id' of undefined
 ```
 
-### With parameterized test failures
+### With failures (Playwright)
+
+```
+passing: 1
+failing[1]{at,expected,got}:
+  "login.spec.ts:7:42",Welcome,Hello World
+```
+
+### With parameterized test failures (Vitest)
 
 Uses TOON tabular format for uniform parameter arrays:
 
@@ -69,12 +107,22 @@ Uses TOON tabular format for uniform arrays:
 ```
 passing: 38
 todo[1]{at,name}:
-  src/api.test.ts,implement error handling
+  "src/api.test.ts:15:3",implement error handling
 skipped[2]{at,name}:
-  src/utils.test.ts,handles edge case
+  "src/utils.test.ts:8:3",handles edge case
+```
+
+### With flaky tests (Playwright)
+
+Tests that fail initially but pass on retry are reported as flaky:
+
+```
+passing: 5
+flaky[1]{at,name,retries}:
+  "checkout.spec.ts:12:3",should complete payment,2
 ```
 
-### With coverage
+### With coverage (Vitest only)
 
 Coverage is automatically included when running with `--coverage`. No extra configuration needed:
 
@@ -112,7 +160,7 @@ With `verbose: true`, all files appear with per-file percentages:
 
 - **Green**: `passing` count
 - **Red**: `failing` header
-- **Yellow**: file paths
+- **Yellow**: `flaky` header, file paths
 - **Gray**: `skipped` tests
 - **Cyan**: `todo` tests
 
@@ -153,7 +201,7 @@ COLOR=1 npx vitest run --reporter=@epicat/toon-reporter
 Write report to a file instead of stdout.
 
 ```ts
-reporters: [['toon-reporter', { outputFile: 'test-results.txt' }]]
+reporters: [['@epicat/toon-reporter', { outputFile: 'test-results.txt' }]]
 ```
 
 ### `verbose`
@@ -166,6 +214,8 @@ reporters: [new ToonReporter({ verbose: true })]
 
 ## Skipped/Todo Line Numbers
 
+### Vitest
+
 To get line:column information for skipped and todo tests, enable `includeTaskLocation` in your vitest config:
 
 ```ts
@@ -173,7 +223,7 @@ To get line:column information for skipped and todo tests, enable `includeTaskLo
 export default defineConfig({
   test: {
     includeTaskLocation: true,
-    reporters: ['toon-reporter'],
+    reporters: ['@epicat/toon-reporter'],
   },
 })
 ```
@@ -181,19 +231,40 @@ export default defineConfig({
 Or via CLI:
 
 ```bash
-npx vitest run --reporter=toon-reporter --includeTaskLocation
+npx vitest run --reporter=@epicat/toon-reporter --includeTaskLocation
 ```
 
 Without this option, skipped/todo tests will only show the file path (not line:column). This is a Vitest limitation - test locations are only collected when this config is enabled before test collection.
 
+### Playwright
+
+Playwright always includes test locations. Tests marked with `test.fixme()` are reported as `todo`, while `test.skip()` tests are reported as `skipped`.
+
+## Playwright-Specific Features
+
+### Flaky Test Detection
+
+When `retries` is configured in your Playwright config, tests that fail initially but pass on retry are reported as flaky:
+
+```ts
+// playwright.config.ts
+export default defineConfig({
+  retries: 2,
+  reporter: [['@epicat/toon-reporter/playwright']],
+})
+```
+
+Output: `flaky[1]{at,name,retries}: "test.spec.ts:5:3",should work,1`
+
 ## Why?
 
 Traditional test reporters output verbose information optimized for human readability. When feeding test results to an LLM for automated fixing, this verbosity wastes tokens. This reporter outputs only what's needed:
 
 - Pass count
 - Failure locations with expected/got values
+- Flaky test detection with retry counts (Playwright)
 - Skipped/todo test names for context
-- Coverage totals and uncovered lines (when `--coverage` is enabled)
+- Coverage totals and uncovered lines (Vitest with `--coverage`)
 
 ## Token Efficiency
 

package/dist/index.d.mts

@@ -11,7 +11,6 @@ interface ToonReporterOptions {
   _captureOutput?: (output: string) => void;
 }
 declare class ToonReporter implements Reporter {
-  start: number;
   ctx: Vitest;
   options: ToonReporterOptions;
   private useColor;
@@ -25,6 +24,7 @@ declare class ToonReporter implements Reporter {
   private parseErrorLocation;
   private parseExpectedGot;
   private formatErrorMessage;
+  private buildReportForModules;
   onTestRunEnd(testModules: ReadonlyArray<any>, _unhandledErrors: ReadonlyArray<SerializedError>, _reason: TestRunEndReason): Promise<void>;
   private colorize;
   writeReport(report: string): Promise<void>;

package/dist/index.mjs

@@ -47,14 +47,14 @@ function shouldUseColor(option) {
 }
 var ToonReporter = class {
 	constructor(options = {}) {
-		this.start = 0;
 		this.options = options;
 		this.useColor = shouldUseColor(options.color);
 	}
 	onInit(ctx) {
 		this.ctx = ctx;
-		this.start = Date.now();
 		this.coverageMap = void 0;
+		const coverage = ctx.config.coverage;
+		if (coverage?.reporter) coverage.reporter = [];
 	}
 	onCoverage(coverage) {
 		this.coverageMap = coverage;
@@ -148,8 +148,8 @@ var ToonReporter = class {
 		if (name && name !== "Error" && !message.startsWith(name)) message = `${name}: ${message}`;
 		return message;
 	}
-	async onTestRunEnd(testModules, _unhandledErrors, _reason) {
-		const tests = getTests(testModules.map((m) => m.task));
+	buildReportForModules(modules) {
+		const tests = getTests(modules.map((m) => m.task));
 		const rootDir = this.ctx.config.root;
 		const failedTests = tests.filter((t) => t.result?.state === "fail");
 		const passedCount = tests.filter((t) => t.result?.state === "pass").length;
@@ -163,20 +163,17 @@ var ToonReporter = class {
 			const { expected, got } = this.parseExpectedGot(error);
 			const relPath = loc?.relPath || relative(rootDir, t.file.filepath);
 			const at = this.formatLocation(relPath, loc?.line || t.location?.line, loc?.column || t.location?.column);
+			const failureDetails = expected !== void 0 && got !== void 0 ? {
+				expected,
+				got
+			} : { error: this.formatErrorMessage(error) };
 			if (t.each) {
 				if (!grouped.has(at)) grouped.set(at, []);
-				grouped.get(at).push(expected !== void 0 && got !== void 0 ? {
-					expected,
-					got
-				} : { error: this.formatErrorMessage(error) });
-			} else {
-				const failure = { at };
-				if (expected !== void 0 && got !== void 0) {
-					failure.expected = expected;
-					failure.got = got;
-				} else if (error) failure.error = this.formatErrorMessage(error);
-				failures.push(failure);
-			}
+				grouped.get(at).push(failureDetails);
+			} else failures.push({
+				at,
+				...failureDetails
+			});
 		}
 		for (const [at, params] of grouped) failures.push({
 			at,
@@ -190,6 +187,29 @@ var ToonReporter = class {
 		if (failures.length > 0) report.failing = failures;
 		if (todoTests.length > 0) report.todo = todoTests.map(mapToSkipped);
 		if (skippedTests.length > 0) report.skipped = skippedTests.map(mapToSkipped);
+		return report;
+	}
+	async onTestRunEnd(testModules, _unhandledErrors, _reason) {
+		const projectNames = /* @__PURE__ */ new Set();
+		const modulesByProject = /* @__PURE__ */ new Map();
+		for (const m of testModules) {
+			const projectName = m.project?.name || "";
+			projectNames.add(projectName);
+			if (!modulesByProject.has(projectName)) modulesByProject.set(projectName, []);
+			modulesByProject.get(projectName).push(m);
+		}
+		if (projectNames.size > 1) {
+			const projectReports = {};
+			for (const [projectName, modules] of modulesByProject) projectReports[projectName || "default"] = this.buildReportForModules(modules);
+			const coverage$1 = this.getCoverageSummary();
+			if (coverage$1) await this.writeReport(encode({
+				...projectReports,
+				coverage: coverage$1
+			}));
+			else await this.writeReport(encode(projectReports));
+			return;
+		}
+		const report = this.buildReportForModules([...testModules]);
 		const coverage = this.getCoverageSummary();
 		if (coverage) report.coverage = coverage;
 		await this.writeReport(encode(report));

package/dist/playwright.d.mts

@@ -0,0 +1,24 @@
+import { FullConfig, FullResult, Reporter, Suite } from "@playwright/test/reporter";
+
+//#region src/toon-playwright-reporter.d.ts
+interface ToonPlaywrightReporterOptions {
+  outputFile?: string;
+  /** @internal Used for testing to capture output */
+  _captureOutput?: (output: string) => void;
+}
+declare class ToonPlaywrightReporter implements Reporter {
+  private config;
+  private suite;
+  private options;
+  constructor(options?: ToonPlaywrightReporterOptions);
+  onBegin(config: FullConfig, suite: Suite): void;
+  private formatLocation;
+  private stripAnsi;
+  private stripOuterQuotes;
+  private parseExpectedGot;
+  onEnd(result: FullResult): Promise<void>;
+  writeReport(report: string): Promise<void>;
+  printsToStdio(): boolean;
+}
+//#endregion
+export { ToonPlaywrightReporter, ToonPlaywrightReporter as default, type ToonPlaywrightReporterOptions };

package/dist/playwright.mjs

@@ -0,0 +1,108 @@
+import { existsSync, promises } from "node:fs";
+import { dirname, relative, resolve } from "pathe";
+import { encode } from "@toon-format/toon";
+
+//#region src/toon-playwright-reporter.ts
+var ToonPlaywrightReporter = class {
+	constructor(options = {}) {
+		this.options = options;
+	}
+	onBegin(config, suite) {
+		this.config = config;
+		this.suite = suite;
+	}
+	formatLocation(filePath, line, column) {
+		const relPath = relative(process.cwd(), filePath);
+		return line ? `${relPath}:${line}:${column || 0}` : relPath;
+	}
+	stripAnsi(str) {
+		return str.replace(/\x1b\[[0-9;]*m/g, "");
+	}
+	stripOuterQuotes(str) {
+		if (str.startsWith("\"") && str.endsWith("\"") || str.startsWith("'") && str.endsWith("'")) return str.slice(1, -1);
+		return str;
+	}
+	parseExpectedGot(error) {
+		const message = this.stripAnsi(error?.message || "");
+		const expectedMatch = message.match(/Expected:\s*(.+?)(?:\n|$)/i);
+		const receivedMatch = message.match(/Received:\s*(.+?)(?:\n|$)/i);
+		if (expectedMatch && receivedMatch) return {
+			expected: this.stripOuterQuotes(expectedMatch[1].trim()),
+			got: this.stripOuterQuotes(receivedMatch[1].trim())
+		};
+		const toBeMatch = message.match(/expect\((.+?)\)\.toBe\((.+?)\)/i);
+		if (toBeMatch) return {
+			expected: this.stripOuterQuotes(toBeMatch[2].trim()),
+			got: this.stripOuterQuotes(toBeMatch[1].trim())
+		};
+		return {};
+	}
+	async onEnd(result) {
+		const allTests = this.suite.allTests();
+		const passedTests = [];
+		const failedTests = [];
+		const skippedTests = [];
+		const todoTests = [];
+		const flakyTests = [];
+		for (const test of allTests) {
+			const lastResult = test.results[test.results.length - 1];
+			if (!lastResult) continue;
+			const status = lastResult.status;
+			const isFixme = test.annotations.some((a) => a.type === "fixme");
+			if (status === "passed" && test.results.length > 1 && test.results.some((r) => r.status === "failed")) flakyTests.push(test);
+			else if (status === "passed") passedTests.push(test);
+			else if (status === "failed" || status === "timedOut" || status === "interrupted") failedTests.push(test);
+			else if (status === "skipped") if (isFixme) todoTests.push(test);
+			else skippedTests.push(test);
+		}
+		const failures = [];
+		for (const test of failedTests) {
+			const error = test.results[test.results.length - 1]?.errors?.[0];
+			const loc = error?.location || test.location;
+			const failure = { at: this.formatLocation(loc.file, loc.line, loc.column) };
+			if (error) {
+				const { expected, got } = this.parseExpectedGot(error);
+				if (expected !== void 0 && got !== void 0) {
+					failure.expected = expected;
+					failure.got = got;
+				} else failure.error = error.message;
+			}
+			failures.push(failure);
+		}
+		const mapToSkipped = (test) => ({
+			at: this.formatLocation(test.location.file, test.location.line, test.location.column),
+			name: test.title
+		});
+		const mapToFlaky = (test) => ({
+			at: this.formatLocation(test.location.file, test.location.line, test.location.column),
+			name: test.title,
+			retries: test.results.length - 1
+		});
+		const report = { passing: passedTests.length };
+		if (flakyTests.length > 0) report.flaky = flakyTests.map(mapToFlaky);
+		if (failures.length > 0) report.failing = failures;
+		if (todoTests.length > 0) report.todo = todoTests.map(mapToSkipped);
+		if (skippedTests.length > 0) report.skipped = skippedTests.map(mapToSkipped);
+		await this.writeReport(encode(report));
+	}
+	async writeReport(report) {
+		if (this.options._captureOutput) {
+			this.options._captureOutput(report);
+			return;
+		}
+		const outputFile = this.options.outputFile;
+		if (outputFile) {
+			const reportFile = resolve(this.rootDir, outputFile);
+			const outputDirectory = dirname(reportFile);
+			if (!existsSync(outputDirectory)) await promises.mkdir(outputDirectory, { recursive: true });
+			await promises.writeFile(reportFile, report, "utf-8");
+			console.log(`TOON report written to ${reportFile}`);
+		} else console.log(report);
+	}
+	printsToStdio() {
+		return !this.options.outputFile;
+	}
+};
+
+//#endregion
+export { ToonPlaywrightReporter, ToonPlaywrightReporter as default };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@epicat/toon-reporter",
-  "version": "0.0.5",
+  "version": "0.0.7",
   "description": "A minimal Vitest reporter optimized for LLM consumption",
   "repository": {
     "type": "git",
@@ -16,23 +16,35 @@
   "types": "./dist/index.d.mts",
   "exports": {
     ".": "./dist/index.mjs",
+    "./playwright": "./dist/playwright.mjs",
     "./package.json": "./package.json"
   },
   "peerDependencies": {
+    "@playwright/test": ">=1.40.0",
     "vitest": ">=2.0.0"
   },
+  "peerDependenciesMeta": {
+    "@playwright/test": {
+      "optional": true
+    },
+    "vitest": {
+      "optional": true
+    }
+  },
   "dependencies": {
     "@toon-format/toon": "^2.0.1",
     "pathe": "^2.0.3"
   },
   "devDependencies": {
     "@changesets/cli": "^2.29.8",
+    "@playwright/test": "^1.57.0",
     "@types/node": "^24.10.1",
     "@vitest/coverage-v8": "^4.0.15",
     "@vitest/runner": "4.0.15",
     "@vitest/utils": "4.0.15",
     "gpt-tokenizer": "^3.4.0",
     "istanbul-lib-coverage": "^3.2.2",
+    "serve": "^14.2.5",
     "tsdown": "^0.16.4",
     "typescript": "^5.9.3",
     "vitest": "4.0.15"
@@ -41,6 +53,7 @@
     "build": "tsdown",
     "dev": "tsdown --watch",
     "test": "vitest run",
+    "test:coverage": "vitest run --coverage",
     "test:watch": "vitest"
   }
 }