@itmetsam/samurai-framework 0.1.0 → 0.2.0

package/src/runner/page-logs-report.ts

@@ -1,82 +0,0 @@
-import type { LogRecord, PageError } from "../browser/page-logs.js";
-import type { SamuraiTestConfig } from "../types/config.js";
-import type { TestError } from "../types/test.js";
-import { toTestError } from "./test-error.js";
-
-/** Config `logs` */
-export type LogsConfig = NonNullable<SamuraiTestConfig["logs"]>;
-
-/** The page threw exceptions nobody caught, and config `logs.failOnPageError` says that fails the test */
-export class PageErrorsError extends Error {
-  public readonly errors: readonly PageError[];
-
-  constructor(errors: readonly PageError[]) {
-    super(`${errors.length} uncaught page error(s): ${errors.map((error) => error.message).join("\n")}`);
-    this.name = "PageErrorsError";
-    this.errors = errors;
-  }
-}
-
-/** A route handler threw: the test's mocks did not do what the test meant */
-export class RouteErrorsError extends Error {
-  public readonly errors: readonly Error[];
-
-  constructor(errors: readonly Error[]) {
-    super(`${errors.length} route handler error(s): ${errors.map((error) => error.message).join("\n")}`);
-    this.name = "RouteErrorsError";
-    this.errors = errors;
-  }
-}
-
-export interface PageLogsInput {
-  config: LogsConfig | undefined;
-  entries: readonly LogRecord[];
-  errors: readonly PageError[];
-  /** How many entries the page's buffer dropped */
-  dropped: number;
-  /** The test body already failed (or timed out) */
-  testFailed: boolean;
-  /** The test called `page.allowPageErrors()` */
-  allowPageErrors: boolean;
-  /** Errors thrown by `page.route` handlers; they fail a passing test */
-  routeErrors?: readonly Error[];
-}
-
-export interface PageLogsVerdict {
-  /** Set when a test that passed has to fail because of page errors */
-  error?: TestError;
-  /** The entries to put in the report, per `logs.capture` */
-  logs?: LogRecord[];
-  logsDropped?: number;
-}
-
-function isIgnored(error: PageError, ignore: readonly (string | RegExp)[]): boolean {
-  return ignore.some((pattern) =>
-    typeof pattern === "string" ? error.message.includes(pattern) : pattern.test(error.message),
-  );
-}
-
-/**
- * Decides, once a test body is done, whether page errors fail it and which log entries go in the report:
- * `capture` "failures" (default) attaches them to failed tests, "all" always, "off" never.
- */
-export function judgePageLogs(input: PageLogsInput): PageLogsVerdict {
-  const { config, entries, errors, dropped, testFailed, allowPageErrors, routeErrors = [] } = input;
-  const capture = config?.capture ?? "failures";
-
-  let error: TestError | undefined;
-  if (routeErrors.length > 0 && !testFailed) {
-    error = toTestError(new RouteErrorsError(routeErrors));
-  } else if (config?.failOnPageError && !testFailed && !allowPageErrors) {
-    const counted = errors.filter((pageError) => !isIgnored(pageError, config.ignoreErrors ?? []));
-    if (counted.length > 0) error = toTestError(new PageErrorsError(counted));
-  }
-
-  const failed = testFailed || error !== undefined;
-  const attach = capture === "all" || (capture === "failures" && failed);
-  return {
-    ...(error && { error }),
-    ...(attach && { logs: [...entries] }),
-    ...(attach && dropped > 0 && { logsDropped: dropped }),
-  };
-}

package/src/runner/prepare-run.test.ts

@@ -1,33 +0,0 @@
-import { after, afterEach, test } from "node:test";
-import assert from "node:assert/strict";
-import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
-import { tmpdir } from "node:os";
-import path from "node:path";
-import { prepareRun } from "./prepare-run.js";
-import { peekRunSettings, setRunSettings } from "../config/run-settings.js";
-import { MASK, maskText, resetSecrets } from "../config/mask.js";
-
-const dir = mkdtempSync(path.join(tmpdir(), "samurai-run-"));
-writeFileSync(path.join(dir, ".env.staging"), "SAMURAI_SECRET_TEST_CARD=4242424242424242\n");
-after(() => rmSync(dir, { recursive: true, force: true }));
-afterEach(() => {
-  setRunSettings(undefined);
-  resetSecrets();
-});
-
-test("resolves settings, loads secrets for that environment and activates the run", () => {
-  const run = prepareRun(
-    { environments: { dev: {}, staging: { baseURL: "https://staging.harbor.shop" } } },
-    { environment: "staging" },
-    { cwd: dir, env: { SAMURAI_SECRET_PIN: "12" } as unknown as NodeJS.ProcessEnv },
-  );
-  assert.equal(run.settings.environment, "staging");
-  assert.equal(peekRunSettings(), run.settings);
-  assert.deepEqual(Object.fromEntries(run.secrets), { TEST_CARD: "4242424242424242", PIN: "12" });
-  assert.equal(maskText("card 4242424242424242 pin 12"), `card ${MASK} pin ${MASK}`);
-});
-
-test("an unknown environment fails before anything is activated", () => {
-  assert.throws(() => prepareRun({ environments: { dev: {} } }, { environment: "stg" }, { cwd: dir, env: {} as unknown as NodeJS.ProcessEnv }));
-  assert.equal(peekRunSettings(), undefined);
-});

package/src/runner/prepare-run.ts

@@ -1,36 +0,0 @@
-import { isShortSecret, registerSecret } from "../config/mask.js";
-import {
-  resolveRunSettings,
-  setRunSettings,
-  type RunOverrides,
-  type RunSettings,
-} from "../config/run-settings.js";
-import { loadSecrets } from "../config/secrets.js";
-import logger from "../logger/index.js";
-import type { SamuraiTestConfig } from "../types/config.js";
-
-export interface PreparedRun {
-  settings: RunSettings;
-  secrets: ReadonlyMap<string, string>;
-}
-
-/**
- * Starts a run: picks the environment, loads its secrets and registers every value for masking
- * (all of them, not only the ones a test reads), then makes the settings active.
- */
-export function prepareRun(
-  config: SamuraiTestConfig,
-  overrides: RunOverrides,
-  options: { cwd?: string; env?: NodeJS.ProcessEnv } = {},
-): PreparedRun {
-  const settings = resolveRunSettings(config, overrides);
-  const secrets = loadSecrets(settings.environment, options);
-  for (const [name, value] of secrets) {
-    registerSecret(name, value);
-    if (isShortSecret(value)) {
-      logger.warn(`Secret "${name}" is short; masking it may hide unrelated text in logs`);
-    }
-  }
-  setRunSettings(settings);
-  return { settings, secrets };
-}

package/src/runner/registry.test.ts

@@ -1,59 +0,0 @@
-import { afterEach, test } from "node:test";
-import assert from "node:assert/strict";
-import { fileURLToPath } from "node:url";
-import {
-  clearRegistry,
-  describe as samuraiDescribe,
-  registeredTests,
-  test as samuraiTest,
-} from "./registry.js";
-
-afterEach(() => clearRegistry());
-
-const noop = async () => {};
-
-test("a test outside describe keeps its title", () => {
-  samuraiTest("logs in", noop);
-  const [registered] = registeredTests();
-  assert.equal(registered?.name, "logs in");
-  assert.deepEqual(registered?.titlePath, ["logs in"]);
-  assert.equal(registered?.function, noop);
-});
-
-test("nested describe titles are joined with >", () => {
-  samuraiDescribe("Checkout", () => {
-    samuraiDescribe("Guest", () => {
-      samuraiTest("applies coupon", noop);
-    });
-    samuraiTest("pays with saved card", noop);
-  });
-  samuraiTest("after", noop);
-  assert.deepEqual(
-    registeredTests().map((t) => t.name),
-    ["Checkout > Guest > applies coupon", "Checkout > pays with saved card", "after"],
-  );
-});
-
-test("the title stack is restored when a describe callback throws", () => {
-  assert.throws(() =>
-    samuraiDescribe("Broken", () => {
-      throw new Error("boom");
-    }),
-  );
-  samuraiTest("after", noop);
-  assert.equal(registeredTests()[0]?.name, "after");
-});
-
-test("an async describe callback is rejected", () => {
-  assert.throws(
-    () => samuraiDescribe("Async", async () => {}),
-    { message: "describe() callback must be synchronous; it only registers tests" },
-  );
-  samuraiTest("after", noop);
-  assert.equal(registeredTests()[0]?.name, "after");
-});
-
-test("records the calling spec file", () => {
-  samuraiTest("where", noop);
-  assert.equal(registeredTests()[0]?.file, fileURLToPath(import.meta.url));
-});

package/src/runner/registry.ts

@@ -1,46 +0,0 @@
-import { fileURLToPath } from "node:url";
-import logger from "../logger/index.js";
-import type { RegisteredTestCase, TestCaseBase } from "../types/test.js";
-
-const registered: RegisteredTestCase[] = [];
-/** Titles of the describe blocks being registered, outermost first */
-const titleStack: string[] = [];
-
-/** Groups the tests registered inside `fn` under `title`. `fn` runs immediately and must be synchronous. */
-export function describe(title: string, fn: () => void): void {
-  titleStack.push(title);
-  try {
-    const result: unknown = fn();
-    if (result instanceof Promise) {
-      result.catch(() => {});
-      throw new Error("describe() callback must be synchronous; it only registers tests");
-    }
-  } finally {
-    titleStack.pop();
-  }
-}
-
-export function test(title: string, fn: TestCaseBase["function"]): void {
-  const titlePath = [...titleStack, title];
-  const name = titlePath.join(" > ");
-  registered.push({ name, titlePath, function: fn, file: callerFile() });
-  logger.debug("Registered test %s", name);
-}
-
-export function registeredTests(): readonly RegisteredTestCase[] {
-  return registered;
-}
-
-/** Forgets every registered test (tests only) */
-export function clearRegistry(): void {
-  registered.length = 0;
-  titleStack.length = 0;
-}
-
-/** File of the code that called test(): stack lines are Error, callerFile, test, caller */
-function callerFile(): string {
-  const caller = new Error().stack?.split("\n")[3] ?? "";
-  const match = caller.match(/\((.+?):\d+:\d+\)$/) ?? caller.match(/at (.+?):\d+:\d+$/);
-  const location = match?.[1] ?? "";
-  return location.startsWith("file://") ? fileURLToPath(location) : location;
-}

package/src/runner/reporter.test.ts

@@ -1,73 +0,0 @@
-import { after, afterEach, test } from "node:test";
-import assert from "node:assert/strict";
-import { mkdtempSync, readFileSync, rmSync } from "node:fs";
-import { tmpdir } from "node:os";
-import path from "node:path";
-import TestReporter from "./reporter.js";
-import { MASK, registerSecret, resetSecrets } from "../config/mask.js";
-import type { RegisteredTestCase } from "../types/test.js";
-
-const dir = mkdtempSync(path.join(tmpdir(), "samurai-reporter-"));
-after(() => rmSync(dir, { recursive: true, force: true }));
-afterEach(() => resetSecrets());
-
-function testCase(file: string, name: string): RegisteredTestCase {
-  return { file, name, titlePath: name.split(" > "), function: async () => {} };
-}
-
-async function report(run: (reporter: TestReporter) => void) {
-  const output = path.join(dir, `${Math.random()}.json`);
-  const reporter = new TestReporter({ environment: "staging", output });
-  reporter.onStart();
-  run(reporter);
-  await reporter.onEnd();
-  return JSON.parse(readFileSync(output, "utf8"));
-}
-
-test("records the environment", async () => {
-  const summary = await report(() => {});
-  assert.equal(summary.environment, "staging");
-});
-
-test("tests with the same title in different files are kept apart", async () => {
-  const a = testCase("a.spec.ts", "Checkout > pays");
-  const b = testCase("b.spec.ts", "Checkout > pays");
-  const summary = await report((reporter) => {
-    reporter.onTestStart(a);
-    reporter.onTestStart(b);
-    reporter.onTestEnd(a);
-    reporter.onTestEnd(b, { message: "boom", type: "error" });
-  });
-  assert.deepEqual(
-    summary.tests.map((t: { file: string; name: string; status: string }) => [t.file, t.name, t.status]),
-    [
-      ["a.spec.ts", "Checkout > pays", "success"],
-      ["b.spec.ts", "Checkout > pays", "failed"],
-    ],
-  );
-});
-
-test("masks secrets in errors and logs", async () => {
-  registerSecret("TEST_CARD", "4242424242424242");
-  const t = testCase("a.spec.ts", "pays");
-  const summary = await report((reporter) => {
-    reporter.onTestStart(t);
-    reporter.onTestEnd(
-      t,
-      {
-        message: 'expected "4242424242424242"',
-        type: "assertion",
-        stack: "AssertionError: expected 4242424242424242\n    at x",
-        expected: "4242424242424242",
-        actual: "4242",
-      },
-      { logs: [{ level: "info", type: "console", text: "card 4242424242424242", timestamp: 1 }] },
-    );
-  });
-  const [result] = summary.tests;
-  assert.equal(result.message, `expected "${MASK}"`);
-  assert.equal(result.expected, MASK);
-  assert.equal(result.actual, "4242");
-  assert.doesNotMatch(result.stack, /4242424242424242/);
-  assert.equal(result.logs[0].text, `card ${MASK}`);
-});

package/src/runner/reporter.ts

@@ -1,109 +0,0 @@
-import { mkdir, writeFile } from "fs/promises";
-import path from "path";
-import { maskDeep } from "../config/mask.js";
-import type {
-  PartialTestResult,
-  RegisteredTestCase,
-  TestError,
-  TestLogEntry,
-  TestResult,
-  TestSummary,
-} from "../types/test.js";
-
-export interface ReporterOptions {
-  /** The environment the run uses, recorded in the report */
-  environment: string;
-  /** Where the report is written. @default <cwd>/result/report.json */
-  output?: string;
-}
-
-/** Results are keyed by file and full title, so equal titles in different files or describe blocks stay apart */
-function keyOf(test: RegisteredTestCase): string {
-  return `${test.file}::${test.name}`;
-}
-
-export default class TestReporter {
-  private testResults: Map<string, TestResult> = new Map();
-  private summary: TestSummary | undefined;
-  private environment: string;
-  private output: string;
-
-  constructor(options: ReporterOptions) {
-    this.environment = options.environment;
-    this.output = options.output ?? path.join(process.cwd(), "result/report.json");
-  }
-
-  public onStart() {
-    const t = performance.mark(`group-start`);
-    this.summary = {
-      duration: 0,
-      status: "failed",
-      startTime: t.startTime,
-      environment: this.environment,
-      tests: [],
-    };
-  }
-
-  public async onEnd() {
-    performance.mark(`group-finish`);
-    const duration = performance.measure(
-      "test-duration",
-      `group-start`,
-      `group-finish`,
-    ).duration;
-
-    this.summary = {
-      ...this.summary,
-      duration,
-      environment: this.environment,
-      tests: Array.from(this.testResults.values()),
-      status: Array.from(this.testResults.values()).every(
-        (r: TestResult) => r.status === "success",
-      )
-        ? "success"
-        : "failed",
-    } as TestSummary;
-
-    await mkdir(path.dirname(this.output), { recursive: true });
-    await writeFile(this.output, JSON.stringify(this.summary, null, 2), {
-      encoding: "utf8",
-    });
-  }
-
-  public onTestStart(test: RegisteredTestCase) {
-    const key = keyOf(test);
-    const t = performance.mark(`${key}-start`);
-    this.testResults.set(key, {
-      name: test.name,
-      file: test.file,
-      status: "started",
-      startTime: t.startTime,
-    });
-  }
-
-  public onTestEnd(
-    test: RegisteredTestCase,
-    error?: TestError,
-    logs?: { logs?: TestLogEntry[]; logsDropped?: number },
-  ) {
-    const key = keyOf(test);
-    performance.mark(`${key}-finish`);
-    const duration = performance.measure(
-      "test-duration",
-      `${key}-start`,
-      `${key}-finish`,
-    ).duration;
-
-    this.testResults.set(
-      key,
-      maskDeep({
-        ...(this.testResults.get(key) as PartialTestResult),
-        duration,
-        status: error ? "failed" : "success",
-        ...(error ? error : undefined),
-        ...(logs?.logs && { logs: logs.logs }),
-        ...(logs?.logsDropped && { logsDropped: logs.logsDropped }),
-      } as TestResult),
-    );
-  }
-}

package/src/runner/test-error.test.ts

@@ -1,39 +0,0 @@
-import { test } from "node:test";
-import assert from "node:assert/strict";
-import { toTestError } from "./test-error.js";
-import { AssertionError } from "../assert/assertion-error.js";
-
-test("AssertionError becomes an assertion error with expected/actual", () => {
-  const err = new AssertionError({
-    matcher: "toHaveText",
-    expected: /contact/i,
-    actual: "Home",
-    locator: "//h1",
-  });
-  const result = toTestError(err);
-  assert.equal(result.type, "assertion");
-  assert.equal(result.message, err.message);
-  assert.equal(result.stack, err.stack);
-  assert.equal(result.expected, "/contact/i");
-  assert.equal(result.actual, "Home");
-});
-
-test("a BigInt AssertionError can be written to the report", () => {
-  const result = toTestError(new AssertionError({ matcher: "toBe", expected: 4n, actual: 3n }));
-  assert.doesNotThrow(() => JSON.stringify(result));
-  assert.equal(result.expected, "4n");
-  assert.equal(result.actual, "3n");
-});
-
-test("other errors stay type error", () => {
-  const err = new Error("boom");
-  assert.deepEqual(toTestError(err), { message: "boom", stack: err.stack, type: "error" });
-});
-
-test("thrown strings stay type error", () => {
-  assert.deepEqual(toTestError("PORT 9223 is already in use!"), {
-    message: "PORT 9223 is already in use!",
-    stack: undefined,
-    type: "error",
-  });
-});

package/src/runner/test-error.ts

@@ -1,20 +0,0 @@
-import { AssertionError, toReportValue } from "../assert/assertion-error.js";
-import type { TestError } from "../types/test.js";
-
-/** Maps anything a test threw to the error stored in the report */
-export function toTestError(err: unknown): TestError {
-  if (err instanceof AssertionError) {
-    return {
-      message: err.message,
-      stack: err.stack,
-      type: "assertion",
-      expected: toReportValue(err.expected),
-      actual: toReportValue(err.actual),
-    };
-  }
-  return {
-    message: err instanceof Error ? err.message : String(err),
-    stack: err instanceof Error ? err.stack : undefined,
-    type: "error",
-  };
-}

package/src/runner/test-runner.ts

@@ -1,184 +0,0 @@
-import assert from "assert";
-import { glob } from "node:fs/promises";
-import path from "path";
-import { pathToFileURL } from "url";
-import { Browser } from "../browser/browser.js";
-import { readConfig } from "../config/config.js";
-import logger from "../logger/index.js";
-import type { SupportedBrowser } from "../types/browser.js";
-import type { RegisteredTestCase, TestFixtures } from "../types/test.js";
-import { registeredTests } from "./registry.js";
-import type { PreparedRun } from "./prepare-run.js";
-import { createEnvFixture } from "../config/variables.js";
-import { createSecretsFixture } from "../config/secrets.js";
-import TestReporter from "./reporter.js";
-import type { SamuraiGroup } from "../types/config.js";
-import Page from "../browser/page.js";
-import { takePendingAssertions } from "../assert/expect.js";
-import { toTestError } from "./test-error.js";
-import { judgePageLogs } from "./page-logs-report.js";
-import type { LogsConfig } from "./page-logs-report.js";
-
-export default class TestRunner {
-  private testFiles: string[];
-  private run: PreparedRun;
-  private group?: SamuraiGroup;
-
-  constructor(testFiles: string[], run: PreparedRun, group?: SamuraiGroup) {
-    this.testFiles = testFiles;
-    this.run = run;
-    this.group = group;
-  }
-
-  static async init(run: PreparedRun, group?: SamuraiGroup) {
-    logger.verbose("Initializing test runner");
-    let srcDir = "";
-
-    if (group && group.src) {
-      srcDir = group.src;
-    } else {
-      srcDir = (await readConfig("srcDir")) ?? "./src";
-    }
-
-    assert(srcDir, "No srcDir declared!");
-
-    const cwd = path.resolve(srcDir);
-    logger.verbose("Reading test files from: %s", cwd);
-    const testFiles: string[] = [];
-    for await (const entry of glob("**/*.spec.ts", {
-      cwd,
-    })) {
-      testFiles.push(entry);
-    }
-    logger.debug("Found %d tests in srcDir", testFiles.length);
-    return new TestRunner(testFiles, run, group);
-  }
-
-  public async start() {
-    const reporter = new TestReporter({ environment: this.run.settings.environment });
-    for (let i = 0; i < this.testFiles.length; i++) {
-      const file = this.testFiles[i] as string;
-      logger.verbose("Trying to register file: %s", file);
-      const targetPath = path.resolve((await readConfig("srcDir")) ?? "./src", file);
-      await import(pathToFileURL(targetPath).href);
-    }
-
-    logger.verbose("Running test files");
-
-    // TODO: Implement test grouping
-    reporter.onStart();
-    for (const test of registeredTests()) {
-      await this.executeTestCase(test, reporter);
-    }
-    await reporter.onEnd();
-  }
-
-  private async executeTestCase(
-    test: RegisteredTestCase,
-    reporter: TestReporter,
-  ) {
-    logger.verbose("Starting test: %s", test.name);
-    reporter.onTestStart(test);
-    const selectedBrowser: SupportedBrowser = (await readConfig("browser")) ?? "firefox";
-    const { settings, secrets } = this.run;
-    const timeout = settings.timeout;
-
-    let browser: Browser | undefined;
-    let page: Page | undefined;
-    let timeoutTimer: NodeJS.Timeout | undefined;
-    const timeoutController = new AbortController();
-
-    await Promise.race([
-      new Promise<void>((resolve) => {
-        timeoutTimer = setTimeout(
-          () => {
-            reporter.onTestEnd(test, {
-              message: `Test timed out after ${timeout / 1000} seconds`,
-              type: "timeout",
-            });
-            timeoutController.abort(new Error("Test timed out"));
-            resolve();
-          },
-          timeout,
-        );
-      }),
-      new Promise<void>(async (resolve, reject) => {
-        try {
-          const launched = await Browser.launch(
-            selectedBrowser,
-            {
-              port: 9223,
-              headless: false,
-            },
-            timeoutController.signal,
-          );
-          browser = launched.browser;
-          page = launched.page;
-          const fixtures: TestFixtures = {
-            page: launched.page,
-            browser: launched.browser,
-            env: createEnvFixture(settings.environment, settings.variables),
-            secrets: createSecretsFixture(settings.environment, secrets),
-          };
-          await test.function(fixtures);
-          // Already reported as timed out
-          if (timeoutController.signal.aborted) return resolve();
-          const unawaited = takePendingAssertions();
-          if (unawaited.length > 0) {
-            reporter.onTestEnd(
-              test,
-              {
-                message: unawaited
-                  .map((matcher) => `expect(locator).${matcher}() was not awaited`)
-                  .join("\n"),
-                type: "assertion",
-              },
-              await judgeLogs(page, true),
-            );
-            return resolve();
-          }
-          const verdict = await judgeLogs(page, false);
-          reporter.onTestEnd(test, verdict.error, verdict);
-          resolve();
-        } catch (err) {
-          if (timeoutController.signal.aborted) return resolve();
-          reporter.onTestEnd(test, toTestError(err), await judgeLogs(page, true));
-          resolve();
-        }
-      }),
-    ]);
-
-    clearTimeout(timeoutTimer);
-    takePendingAssertions();
-    logger.verbose("Closing browser for test: %s", test.name);
-    await browser?.close();
-  }
-}
-
-/** Reads the page's log once the test body is done and applies config `logs` (see judgePageLogs) */
-async function judgeLogs(page: Page | undefined, testFailed: boolean) {
-  if (!page) return {};
-  try {
-    // Events and command replies share one ordered connection: after this, everything logged so far is in
-    await page.syncLogs();
-  } catch (err) {
-    logger.debug("Could not sync the page's logs", { err });
-  }
-  let config: LogsConfig | undefined;
-  try {
-    config = await readConfig("logs");
-  } catch (err) {
-    logger.debug("Could not read logs config, using defaults", { err });
-  }
-  return judgePageLogs({
-    config,
-    entries: page.getLogs(),
-    errors: page.pageErrors(),
-    dropped: page.logsDropped,
-    testFailed,
-    allowPageErrors: page.pageErrorsAllowed,
-    routeErrors: page.routeErrors(),
-  });
-}
-
-export { describe, test } from "./registry.js";