@kinora/reporter 0.11.0 → 0.13.0

package/README.md

@@ -93,6 +93,20 @@ A scan you build yourself works too, as long as it is attached to the test with
 An assertion alone is invisible to a reporter. More in the
 [accessibility guide](https://docs.kinora.dev/guides/reporter/#accessibility).
 
+## Checking the app is up
+
+List `@kinora/reporter/check-base-url` in `globalSetup` and the run stops before any test when a
+project's `baseURL` does not answer (refused, unresolved, timed out, or a `502` / `503` / `504`):
+
+```ts
+export default defineConfig({
+  globalSetup: ['@kinora/reporter/check-base-url', './global-setup.ts'],
+})
+```
+
+`KINORA_CHECK_BASE_URL=false` skips it. More in the
+[reporter guide](https://docs.kinora.dev/guides/reporter/#checking-the-app-is-up).
+
 ## Documentation
 
 Full reporter options, PR/MR comments, CI examples, and self-hosting are in the docs:

package/dist/a11y.d.mts

@@ -1,6 +1,5 @@
 import { Page } from "@playwright/test";
 import AxeBuilder from "@axe-core/playwright";
-//#region src/a11y.d.ts
 export type AxeResults = Awaited<ReturnType<AxeBuilder['analyze']>>;
 export declare const A11Y_ATTACHMENT = "accessibility-scan-results";
 /**
@@ -11,5 +10,4 @@ export declare const A11Y_ATTACHMENT = "accessibility-scan-results";
  * record them as warnings. `configure` narrows or widens the scan, e.g.
  * `axe => axe.withTags(['wcag2a', 'wcag2aa'])`.
  */
-export declare function scanA11y(page: Page, configure?: (axe: AxeBuilder) => AxeBuilder): Promise<AxeResults>;
-//#endregion
+export declare function scanA11y(page: Page, configure?: (axe: AxeBuilder) => AxeBuilder): Promise<AxeResults>;

package/dist/a11y.mjs

@@ -1,5 +1,5 @@
 import { test } from "@playwright/test";
-//#region src/a11y.ts
+// src/a11y.ts
 const A11Y_ATTACHMENT = "accessibility-scan-results";
 /**
 * Scans the page with axe (`@axe-core/playwright`) and attaches the result to the running test,
@@ -21,5 +21,4 @@ async function scanA11y(page, configure) {
 	});
 	return results;
 }
-//#endregion
 export { A11Y_ATTACHMENT, scanA11y };

package/dist/check-base-url.d.mts

@@ -0,0 +1,36 @@
+import { FullConfig, FullProject } from "@playwright/test";
+export declare const CHECK_ATTEMPTS = 3;
+export declare const CHECK_TIMEOUT_MS = 10000;
+export declare const CHECK_DELAY_MS = 2000;
+type Project = Pick<FullProject, 'name' | 'dependencies' | 'use'>;
+export interface BaseUrlTarget {
+  url: string;
+  projects: string[];
+  ignoreHTTPSErrors: boolean;
+  proxy?: FullProject['use']['proxy'];
+  remoteBrowser: boolean;
+}
+export type ProbeOutcome = {
+  status: number;
+} | {
+  error: string;
+};
+export type Probe = (target: BaseUrlTarget, timeoutMs: number) => Promise<ProbeOutcome>;
+export interface CheckOptions {
+  attempts?: number;
+  timeoutMs?: number;
+  delayMs?: number;
+  probe?: Probe;
+  argv?: string[];
+  env?: Record<string, string | undefined>;
+}
+export declare function selectedProjects<P extends Project>(projects: P[], filtered: Project[] | undefined, argv: string[]): P[];
+export declare function baseUrlTargets(config: Pick<FullConfig, 'projects'> & {
+  filteredProjects?: Project[];
+}, argv?: string[], env?: Record<string, string | undefined>): BaseUrlTarget[];
+export declare function checkBaseUrl(target: BaseUrlTarget, options?: CheckOptions): Promise<string | null>;
+export declare function checkBaseUrls(config: Pick<FullConfig, 'projects'> & {
+  filteredProjects?: Project[];
+}, options?: CheckOptions): Promise<void>;
+export default function globalSetup(config: FullConfig): Promise<void>;
+export {};

package/dist/check-base-url.mjs

@@ -0,0 +1,119 @@
+import process from "node:process";
+// src/check-base-url.ts
+const CHECK_ATTEMPTS = 3;
+const CHECK_TIMEOUT_MS = 1e4;
+const CHECK_DELAY_MS = 2e3;
+const GATEWAY_DOWN = {
+	502: "bad gateway",
+	503: "service unavailable",
+	504: "gateway timeout"
+};
+function cliProjects(argv) {
+	let names = null;
+	for (let i = 0; i < argv.length; i++) if (argv[i].startsWith("--project=")) (names ??= []).push(argv[i].slice(10));
+	else if (argv[i] === "--project") while (argv[i + 1] && !argv[i + 1].startsWith("-")) (names ??= []).push(argv[++i]);
+	return names;
+}
+const wildcard = (pattern) => new RegExp(`^${pattern.split("*").map((p) => p.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join(".*")}$`, "i");
+function selectedProjects(projects, filtered, argv) {
+	let chosen;
+	if (filtered) {
+		const names = new Set(filtered.map((p) => p.name));
+		chosen = new Set(projects.filter((p) => filtered.includes(p) || names.has(p.name)));
+	} else {
+		const names = cliProjects(argv);
+		if (!names?.length) return projects;
+		const patterns = names.map(wildcard);
+		chosen = new Set(projects.filter((p) => patterns.some((re) => re.test(p.name))));
+	}
+	if (!argv.includes("--no-deps")) {
+		const byName = new Map(projects.map((p) => [p.name, p]));
+		const queue = [...chosen];
+		while (queue.length) for (const dep of queue.shift().dependencies) {
+			const p = byName.get(dep);
+			if (p && !chosen.has(p)) {
+				chosen.add(p);
+				queue.push(p);
+			}
+		}
+	}
+	return projects.filter((p) => chosen.has(p));
+}
+function baseUrlTargets(config, argv = process.argv, env = process.env) {
+	const targets = /* @__PURE__ */ new Map();
+	for (const project of selectedProjects(config.projects, config.filteredProjects, argv)) {
+		const url = project.use.baseURL;
+		if (!url || !/^https?:\/\//i.test(url)) continue;
+		const remoteBrowser = !!project.use.connectOptions || !!env.PW_TEST_CONNECT_WS_ENDPOINT;
+		const target = targets.get(url);
+		if (target) {
+			if (project.name) target.projects.push(project.name);
+			target.remoteBrowser ||= remoteBrowser;
+			continue;
+		}
+		targets.set(url, {
+			url,
+			projects: project.name ? [project.name] : [],
+			ignoreHTTPSErrors: !!project.use.ignoreHTTPSErrors,
+			proxy: project.use.proxy,
+			remoteBrowser
+		});
+	}
+	return [...targets.values()];
+}
+const playwrightProbe = async (target, timeoutMs) => {
+	const { request } = await import("@playwright/test");
+	const context = await request.newContext({
+		ignoreHTTPSErrors: target.ignoreHTTPSErrors,
+		proxy: target.proxy
+	});
+	try {
+		return { status: (await context.get(target.url, {
+			timeout: timeoutMs,
+			maxRedirects: 0,
+			failOnStatusCode: false
+		})).status() };
+	} catch (error) {
+		return { error: (error instanceof Error ? error.message : String(error)).split("\n")[0].trim().replace(/^apiRequestContext\.\w+:\s*/, "") };
+	} finally {
+		await context.dispose().catch(() => {});
+	}
+};
+async function checkBaseUrl(target, options = {}) {
+	const attempts = Math.max(1, options.attempts ?? 3);
+	const probe = options.probe ?? playwrightProbe;
+	let reason = "";
+	for (let attempt = 1; attempt <= attempts; attempt++) {
+		const outcome = await probe(target, options.timeoutMs ?? 1e4);
+		if ("status" in outcome) {
+			const down = GATEWAY_DOWN[outcome.status];
+			if (!down) return null;
+			reason = `${outcome.status} ${down}`;
+		} else reason = outcome.error;
+		if (attempt < attempts) await new Promise((resolve) => setTimeout(resolve, options.delayMs ?? 2e3));
+	}
+	return `${reason} (${attempts} ${attempts === 1 ? "attempt" : "attempts"})`;
+}
+async function checkBaseUrls(config, options = {}) {
+	const env = options.env ?? process.env;
+	if (env.KINORA_CHECK_BASE_URL === "false") return;
+	const targets = baseUrlTargets(config, options.argv, env);
+	const reasons = await Promise.all(targets.map((t) => checkBaseUrl(t, options)));
+	const down = targets.flatMap((t, i) => reasons[i] ? [{
+		t,
+		reason: reasons[i]
+	}] : []);
+	if (!down.length) return;
+	const message = [
+		"[kinora] The app under test did not answer, so no test ran:",
+		...down.map(({ t, reason }) => `  ${t.url}${t.projects.length ? ` (${t.projects.join(", ")})` : ""}: ${reason}${t.remoteBrowser ? `\n    The check ran from this machine, but the browser runs on another one and may reach ${new URL(t.url).host} from its own network.` : ""}`),
+		"Bring the environment back and run again, or set KINORA_CHECK_BASE_URL=false to run the tests anyway."
+	].join("\n");
+	const error = new Error(message);
+	error.stack = message;
+	throw error;
+}
+async function globalSetup(config) {
+	await checkBaseUrls(config);
+}
+export { CHECK_ATTEMPTS, CHECK_DELAY_MS, CHECK_TIMEOUT_MS, baseUrlTargets, checkBaseUrl, checkBaseUrls, globalSetup as default, selectedProjects };

package/dist/ct.d.mts

@@ -1,5 +1,4 @@
 import { TestInfo, expect } from "@playwright/test";
-//#region src/ct.d.ts
 type Annotations = TestInfo['annotations'];
 export declare function recordStory(annotations: Annotations, storyId: string): void;
 export declare function withStoryAnnotations<M extends (storyId: string, ...rest: any[]) => unknown>(mount: M, annotations: Annotations): M;
@@ -8,5 +7,4 @@ export declare function withStoryAnnotations<M extends (storyId: string, ...rest
  * the mounted story id on the test, which lets kinora group results by component and story.
  */
 export declare const test: import("@playwright/test").TestType<import("@playwright/test").PlaywrightTestArgs & import("@playwright/test").PlaywrightTestOptions, import("@playwright/test").PlaywrightWorkerArgs & import("@playwright/test").PlaywrightWorkerOptions>;
-//#endregion
 export { expect };

package/dist/ct.mjs

@@ -1,6 +1,6 @@
 import { STORY_ANNOTATION } from "@kinora/core";
 import { expect, test as test$1 } from "@playwright/test";
-//#region src/ct.ts
+// src/ct.ts
 function recordStory(annotations, storyId) {
 	if (!annotations.some((a) => a.type === STORY_ANNOTATION && a.description === storyId)) annotations.push({
 		type: STORY_ANNOTATION,
@@ -20,5 +20,4 @@ function withStoryAnnotations(mount, annotations) {
 const test = test$1.extend({ mount: async ({ mount }, use, testInfo) => {
 	await use(withStoryAnnotations(mount, testInfo.annotations));
 } });
-//#endregion
 export { expect, recordStory, test, withStoryAnnotations };

package/dist/index.d.mts

@@ -1,6 +1,5 @@
 import { AttachmentKind, CiMeta, GitMeta } from "@kinora/core";
 import { FullConfig, FullResult, Reporter, Suite } from "@playwright/test/reporter";
-//#region src/index.d.ts
 export interface KinoraReporterOptions {
   /** kinora server base URL. Defaults to env KINORA_URL, then the hosted cloud. Set for self-host. */
   url?: string;
@@ -45,5 +44,4 @@ export default class KinoraReporter implements Reporter {
   onEnd(result: FullResult): Promise<void>;
   private prCommentLabel;
   private maybePostPrComment;
-}
-//#endregion
+}

package/dist/index.mjs

@@ -1,8 +1,8 @@
 import { readFileSync, statSync } from "node:fs";
 import { readFile } from "node:fs/promises";
 import process from "node:process";
-import { DEFAULT_KINORA_URL, DEFAULT_UPLOAD_ATTACHMENTS, IngestError, a11yFromScans, createIngestClient, detectCiEnv, effectiveAttachments, flakyTestsOf, isUploadableAttachment, makeTestKey, parseAxeScan, postPrComment, resolvePrContext, resolveShardGroup } from "@kinora/core";
-//#region src/index.ts
+import { DEFAULT_KINORA_URL, DEFAULT_UPLOAD_ATTACHMENTS, IngestError, a11yFromScans, createIngestClient, describeRetry, detectCiEnv, effectiveAttachments, flakyTestsOf, isUploadableAttachment, makeTestKey, parseAxeScan, postPrComment, resolvePrContext, resolveShardGroup } from "@kinora/core";
+// src/index.ts
 function identity(test) {
 	let file = "";
 	let projectName = "";
@@ -109,6 +109,7 @@ var KinoraReporter = class {
 			groupKey: this.options.groupKey,
 			leg: this.prCommentLabel()
 		}) : void 0;
+		const baseUrls = [...new Set((this.config?.projects ?? []).flatMap((p) => p.use.baseURL ? [p.use.baseURL] : []))];
 		const payload = {
 			project: {
 				slug: this.options.project.slug,
@@ -121,6 +122,7 @@ var KinoraReporter = class {
 				playwrightVersion: this.config?.version,
 				git: this.options.git ?? detected.git,
 				ci: this.options.ci ?? detected.ci,
+				...baseUrls.length && { baseUrls },
 				...shard && group && {
 					...group,
 					shard: {
@@ -135,7 +137,8 @@ var KinoraReporter = class {
 			const client = createIngestClient({
 				baseUrl: url,
 				token,
-				regression: !!this.options.prComment
+				regression: !!this.options.prComment,
+				onRetry: (info) => console.warn(`[kinora] ${describeRetry(info)}`)
 			});
 			const res = await client.uploadRun(payload);
 			const kinds = this.options.uploadAttachments ?? DEFAULT_UPLOAD_ATTACHMENTS;
@@ -156,6 +159,7 @@ var KinoraReporter = class {
 				}
 			}
 			console.log(`[kinora] uploaded ${res.tests} tests + ${artifacts} artifacts (run ${res.runId}${res.shardProgress ? `, ${res.shardProgress.landed} of ${res.shardProgress.total} shards` : ""})`);
+			if (res.incident) console.warn(`[kinora] environment incident, not counted: ${res.incident.reason}`);
 			await this.maybePostPrComment(payload, res);
 		} catch (err) {
 			if (err instanceof IngestError && err.status === 402) console.warn(`[kinora] ${err.message}`);
@@ -198,7 +202,8 @@ var KinoraReporter = class {
 				...res.shardProgress && { shards: {
 					runId: res.runId,
 					...res.shardProgress
-				} }
+				} },
+				...res.incident && { incident: res.incident }
 			}, policy ?? "always");
 			if (outcome !== "skipped") console.warn(`[kinora] PR comment ${outcome}`);
 		} catch (err) {
@@ -206,5 +211,4 @@ var KinoraReporter = class {
 		}
 	}
 };
-//#endregion
 export { KinoraReporter as default };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@kinora/reporter",
   "type": "module",
-  "version": "0.11.0",
+  "version": "0.13.0",
   "description": "Playwright reporter that uploads test results to a kinora server",
   "author": "Joris Gallot",
   "license": "MIT",
@@ -22,7 +22,8 @@
   "exports": {
     ".": "./dist/index.mjs",
     "./ct": "./dist/ct.mjs",
-    "./a11y": "./dist/a11y.mjs"
+    "./a11y": "./dist/a11y.mjs",
+    "./check-base-url": "./dist/check-base-url.mjs"
   },
   "main": "./dist/index.mjs",
   "files": [
@@ -44,14 +45,14 @@
     }
   },
   "dependencies": {
-    "@kinora/core": "0.13.0"
+    "@kinora/core": "0.15.0"
   },
   "devDependencies": {
     "@axe-core/playwright": "^4.14.0",
     "@playwright/test": "^1.64.1",
     "@types/node": "^26.6.5",
     "@vitest/coverage-v8": "^5.0.5",
-    "tsdown": "^0.23.1",
+    "tsdown": "^0.23.3",
     "typescript": "~6.0.3",
     "vitest": "^5.0.5"
   },