@itmetsam/samurai-framework 0.16.0 → 0.18.0

package/dist/capabilities.d.ts

@@ -25,8 +25,14 @@ export declare const CAPABILITIES: {
     readonly testIds: true;
     /** `loadConfig` from `./project`: the config with every default applied, the way a run uses it. */
     readonly resolvedConfig: true;
+    /** `loadConfig({ written: true })`: the config as written next to the resolved one, and `writtenConfig`. */
+    readonly writtenConfig: true;
     /** The recorder's first `onStatus` call, `{ state: "recording", toolbar }`, right after it starts, and a `RecordResult` with every field always present. */
     readonly recorderStartStatus: true;
+    /** `outputDir` on `runTests` and `runFlowInProject`: a folder per run for the report and the screenshots. */
+    readonly outputDir: true;
+    /** `runFlowInProject`'s `outputDir` gives each node its own folder, `nodes/<node id>/`. */
+    readonly flowNodeFolders: true;
 };
 /** The shape of {@link CAPABILITIES}. */
 export type Capabilities = typeof CAPABILITIES;

package/dist/capabilities.js

@@ -25,6 +25,12 @@ export const CAPABILITIES = {
     testIds: true,
     /** `loadConfig` from `./project`: the config with every default applied, the way a run uses it. */
     resolvedConfig: true,
+    /** `loadConfig({ written: true })`: the config as written next to the resolved one, and `writtenConfig`. */
+    writtenConfig: true,
     /** The recorder's first `onStatus` call, `{ state: "recording", toolbar }`, right after it starts, and a `RecordResult` with every field always present. */
     recorderStartStatus: true,
+    /** `outputDir` on `runTests` and `runFlowInProject`: a folder per run for the report and the screenshots. */
+    outputDir: true,
+    /** `runFlowInProject`'s `outputDir` gives each node its own folder, `nodes/<node id>/`. */
+    flowNodeFolders: true,
 };

package/dist/config/config.d.ts

@@ -21,6 +21,8 @@ export declare function browsersDir(): string;
 export declare function validateConfig(config: SamuraiTestConfig): void;
 /** The whole `samurai.config.ts` of the project (the working directory unless a project is active) */
 export declare function loadRawConfig(): Promise<SamuraiTestConfig>;
+/** The default export of `samurai.config.ts` in `dir`, not validated */
+export declare function importConfigFile(dir: string): Promise<SamuraiTestConfig>;
 /** A project's config without keeping the project active (`withProject` is one at a time): the given one, else its samurai.config.ts */
 export declare function projectConfig(options: {
     projectDir?: string;

package/dist/config/config.js

@@ -37,13 +37,18 @@ export async function loadRawConfig() {
         validateConfig(activeProject.config);
         return activeProject.config;
     }
-    const configUrl = path.join(projectDir(), "samurai.config.ts");
+    const config = await importConfigFile(projectDir());
+    validateConfig(config);
+    return config;
+}
+/** The default export of `samurai.config.ts` in `dir`, not validated */
+export async function importConfigFile(dir) {
+    const configUrl = path.join(dir, "samurai.config.ts");
     logger.debug("Reading config from: %s", configUrl);
     const config = (await import(pathToFileURL(configUrl).href));
     if (!config.default) {
         throw new Error("samurai.config.ts does not export defineConfig as default!");
     }
-    validateConfig(config.default);
     return config.default;
 }
 /** A project's config without keeping the project active (`withProject` is one at a time): the given one, else its samurai.config.ts */

package/dist/config/resolved-config.d.ts

@@ -91,6 +91,26 @@ export declare function resolveConfig(config: SamuraiTestConfig, options: {
     projectDir: string;
     env: Env;
 }): ResolvedConfig;
+/**
+ * `samurai.config.ts` as the file states it, as JSON: nothing filled in (an absent `workers` stays absent, a config
+ * without `environments` has none), `logs.ignoreErrors` patterns as {@link ResolvedIgnoreError}, and `steps` as
+ * their kinds. For showing and editing the config; {@link ResolvedConfig} is what a run uses.
+ */
+export type WrittenConfig = Omit<SamuraiTestConfig, "logs" | "steps"> & {
+    logs?: Omit<NonNullable<SamuraiTestConfig["logs"]>, "ignoreErrors"> & {
+        ignoreErrors?: ResolvedIgnoreError[];
+    };
+    steps?: string[];
+};
+/** Both views of a config: as written, and resolved; `resolved` is absent, and `error` says why, when the config is invalid */
+export interface ConfigViews {
+    written: WrittenConfig;
+    resolved?: ResolvedConfig;
+    /** The error a run would throw for this config */
+    error?: string;
+}
+/** The config as written, as plain JSON (see {@link WrittenConfig}). Pure: reads no files. */
+export declare function writtenConfig(config: SamuraiTestConfig): WrittenConfig;
 /**
  * A project's `samurai.config.ts` with every default applied, the way a run uses it (see {@link ResolvedConfig}).
  * Leaves no project active. The caller registers `tsx` first (the framework does not), so the config file can be
@@ -98,9 +118,17 @@ export declare function resolveConfig(config: SamuraiTestConfig, options: {
  * (Node caches it): a later edit to `samurai.config.ts` is seen only in a new process.
  * @param options.projectDir The project folder. @default the working directory
  * @param options.env Where `CI` is read from. @default `process.env`
+ * @param options.written Also return the config as written (see {@link ConfigViews})
  */
 export declare function loadConfig(options?: {
     projectDir?: string;
     env?: Env;
+    written?: false;
 }): Promise<ResolvedConfig>;
+/** With `written: true`: the config as written next to the resolved one (see {@link ConfigViews}); an invalid config is reported in `error`, not thrown */
+export declare function loadConfig(options: {
+    projectDir?: string;
+    env?: Env;
+    written: true;
+}): Promise<ConfigViews>;
 export {};

package/dist/config/resolved-config.js

@@ -1,6 +1,6 @@
 import path from "node:path";
 import { browsersFor } from "./browsers-for.js";
-import { projectConfig, validateConfig } from "./config.js";
+import { importConfigFile, projectConfig, validateConfig } from "./config.js";
 import { DEFAULT_COLLECT_BODIES, DEFAULT_COMMAND_TIMEOUT, DEFAULT_DOWNLOADS_DIR, DEFAULT_FAIL_ON_PAGE_ERROR, DEFAULT_FILES_DIR, DEFAULT_IDLE_TIME, DEFAULT_LOGS_CAPTURE, DEFAULT_MAX_BODY_SIZE, DEFAULT_NAVIGATION_TIMEOUT, DEFAULT_RETRIES, DEFAULT_ROUTE_TIMEOUT, DEFAULT_SRC_DIR, DEFAULT_TRACK_NETWORK, DEFAULT_VIEWPORT, DEFAULT_WAIT_UNTIL, } from "./defaults.js";
 import { forbidsOnly } from "./forbid-only.js";
 import { IMPLICIT_ENVIRONMENT, resolveRunSettings, UnknownEnvironmentError, } from "./run-settings.js";
@@ -97,16 +97,39 @@ export function resolveConfig(config, options) {
         steps: (config.steps ?? []).map((step) => step.kind),
     };
 }
-/**
- * A project's `samurai.config.ts` with every default applied, the way a run uses it (see {@link ResolvedConfig}).
- * Leaves no project active. The caller registers `tsx` first (the framework does not), so the config file can be
- * imported; an app does that in the child process that calls this. The config module is imported once per process
- * (Node caches it): a later edit to `samurai.config.ts` is seen only in a new process.
- * @param options.projectDir The project folder. @default the working directory
- * @param options.env Where `CI` is read from. @default `process.env`
- */
+/** The config as written, as plain JSON (see {@link WrittenConfig}). Pure: reads no files. */
+export function writtenConfig(config) {
+    const { logs, steps, ...rest } = config;
+    const { ignoreErrors, ...logSettings } = logs ?? {};
+    const written = {
+        ...rest,
+        ...(logs && {
+            logs: {
+                ...logSettings,
+                ...(ignoreErrors && { ignoreErrors: ignoreErrorsOf(ignoreErrors) }),
+            },
+        }),
+        ...(steps && { steps: steps.map((step) => step.kind) }),
+    };
+    // A JSON copy: shares nothing with the config, and leaves out what JSON can't carry
+    return JSON.parse(JSON.stringify(written));
+}
 export async function loadConfig(options = {}) {
     const projectDir = path.resolve(options.projectDir ?? process.cwd());
-    const config = await projectConfig({ projectDir });
-    return resolveConfig(config, { projectDir, env: options.env ?? process.env });
+    const env = options.env ?? process.env;
+    if (!options.written) {
+        const config = await projectConfig({ projectDir });
+        return resolveConfig(config, { projectDir, env });
+    }
+    const config = await importConfigFile(projectDir);
+    const written = writtenConfig(config);
+    try {
+        return { written, resolved: resolveConfig(config, { projectDir, env }) };
+    }
+    catch (error) {
+        return {
+            written,
+            error: error instanceof Error ? error.message : String(error),
+        };
+    }
 }

package/dist/flows/run/handlers/support.js

@@ -7,7 +7,7 @@ export const fail = (e) => ({
 export async function runNodeTests(node, run, list) {
     let result;
     try {
-        result = await run.runTests(list, (e) => run.onTestEvent(node.id, e), run.nodeBrowsers?.[node.id]);
+        result = await run.runTests(node.id, list, (e) => run.onTestEvent(node.id, e), run.nodeBrowsers?.[node.id]);
     }
     catch (e) {
         return fail(e);

package/dist/flows/run/handlers/types.d.ts

@@ -14,7 +14,7 @@ export interface HandlerRun {
     /** The browsers of the nodes that have their own (a Group node's group); the others use the project's */
     nodeBrowsers?: Record<string, SupportedBrowser[]>;
     signal: AbortSignal;
-    runTests: (tests: NodeTest[], onEvent: (e: RunEvent) => void, browsers?: SupportedBrowser[]) => Promise<"passed" | "failed" | "cancelled">;
+    runTests: (nodeId: string, tests: NodeTest[], onEvent: (e: RunEvent) => void, browsers?: SupportedBrowser[]) => Promise<"passed" | "failed" | "cancelled">;
     sleep: (ms: number, signal: AbortSignal) => Promise<void>;
     onTestEvent: (nodeId: string, e: RunEvent) => void;
     /** Definitions of kinds core doesn't have (steps), for expression scopes */

package/dist/flows/run/node-pool.d.ts

@@ -4,6 +4,6 @@ import type { WorkItem } from "../../runner/worker-protocol.js";
 import type { SupportedBrowser } from "../../types/browser.js";
 import type { NodeTest } from "./events.js";
 /** The work items of a node: each test in each browser, the browsers one after the other */
-export declare function itemsOf(tests: NodeTest[], browsers: readonly SupportedBrowser[]): WorkItem[];
+export declare function itemsOf(tests: NodeTest[], browsers: readonly SupportedBrowser[], outputDir?: string): WorkItem[];
 /** Runs the tests of a node on the pool and reports them as a run would */
 export declare function runOnPool(pool: Pick<WorkerPool, "run">, items: WorkItem[], onEvent: (event: RunEvent) => void, signal: AbortSignal): Promise<"passed" | "failed" | "cancelled">;

package/dist/flows/run/node-pool.js

@@ -1,7 +1,12 @@
 /** The work items of a node: each test in each browser, the browsers one after the other */
-export function itemsOf(tests, browsers) {
+export function itemsOf(tests, browsers, outputDir) {
     return browsers
-        .flatMap((browser) => tests.map((t) => ({ file: t.file, testName: t.name, browser })))
+        .flatMap((browser) => tests.map((t) => ({
+        file: t.file,
+        testName: t.name,
+        browser,
+        ...(outputDir !== undefined && { outputDir }),
+    })))
         .map((item, index) => ({ ...item, index }));
 }
 /** Runs the tests of a node on the pool and reports them as a run would */

package/dist/flows/run/project-options.d.ts

@@ -20,6 +20,12 @@ export interface RunFlowInProjectOptions extends ProjectOptions {
     /** `"fail"` (default): a flow with a node that can't run is refused. `"skip"`: those nodes are skipped. */
     unsupported?: "fail" | "skip";
     port?: number;
+    /**
+     * The folder for the tests' named screenshots, instead of `<project>/result` (see `runTests`). Each node writes to
+     * `nodes/<node id>/` inside it, so a test that runs in two nodes keeps both sets; screenshot paths in a node's
+     * results are relative to that folder. Relative to the project folder
+     */
+    outputDir?: string;
     /**
      * How many worker processes the tests of the whole run share: a number, or a percentage like `"50%"`. Unset
      * takes `SAMURAI_WORKERS` (`workersEnv`), then the config's `workers`, then half the CPUs. 1 runs each Test

package/dist/flows/run/project.js

@@ -1,4 +1,5 @@
 import path from "node:path";
+import { cleanName } from "../../browser/evidence.js";
 import { DEFAULT_SRC_DIR } from "../../config/defaults.js";
 import { projectConfig } from "../../config/config.js";
 import { checkFlow, withKeys, } from "../core/index.js";
@@ -194,9 +195,13 @@ async function walkFlowInProject(options) {
             unsupported,
             handlers: { ...HANDLERS, ...steps.handlers },
             nodes: steps.nodes,
-            runTests: async ({ tests, onEvent, signal, browsers }) => {
+            runTests: async ({ nodeId, tests, onEvent, signal, browsers }) => {
+                // Each node gets a folder of its own: the same test in two nodes would overwrite its screenshots
+                const outputDir = options.outputDir !== undefined
+                    ? path.join(options.outputDir, "nodes", cleanName(nodeId))
+                    : undefined;
                 if (pool)
-                    return runOnPool(pool, itemsOf(tests, browsers ?? browsersFor({ project: config.browsers })), onEvent, signal);
+                    return runOnPool(pool, itemsOf(tests, browsers ?? browsersFor({ project: config.browsers }), outputDir), onEvent, signal);
                 const summary = await run({
                     projectDir,
                     config,
@@ -214,6 +219,7 @@ async function walkFlowInProject(options) {
                     files: [...new Set(tests.map((t) => t.file))],
                     testNames: tests.map((t) => t.name),
                     reportPath: false,
+                    ...(outputDir !== undefined && { outputDir }),
                     signal,
                     onEvent,
                 });

package/dist/flows/run/run.d.ts

@@ -20,6 +20,8 @@ export interface RunFlowOptions {
     signal?: AbortSignal;
     onEvent?: (event: FlowEvent) => void;
     runTests: (input: {
+        /** The node the tests are for: a node runs at most once in a run */
+        nodeId: string;
         tests: NodeTest[];
         onEvent: (e: RunEvent) => void;
         signal: AbortSignal;

package/dist/flows/run/run.js

@@ -34,7 +34,8 @@ export async function runFlow(options) {
         ...(options.nodeBrowsers && { nodeBrowsers: options.nodeBrowsers }),
         signal,
         sleep: options.sleep ?? defaultSleep,
-        runTests: (tests, onTestEvent, browsers) => options.runTests({
+        runTests: (nodeId, tests, onTestEvent, browsers) => options.runTests({
+            nodeId,
             tests,
             onEvent: onTestEvent,
             signal,

package/dist/project/index.d.ts

@@ -2,4 +2,4 @@ export { ConfigEditError, editConfig, hasConfigPath, propertyText, type ConfigVa
 export { addVariable, ExistsError, VARIABLE_NAME } from "./variables.js";
 export { isGitIgnored, SECRET_NAME, setSecret } from "./secrets-file.js";
 export { cleanFileName, filesFolder, isFolderIgnored, RECORD_LIMIT, storeFile, } from "./files-store.js";
-export { loadConfig, type ResolvedConfig, type ResolvedEnvironment, type ResolvedIgnoreError, } from "../config/resolved-config.js";
+export { loadConfig, writtenConfig, type ConfigViews, type ResolvedConfig, type ResolvedEnvironment, type ResolvedIgnoreError, type WrittenConfig, } from "../config/resolved-config.js";

package/dist/project/index.js

@@ -2,4 +2,4 @@ export { ConfigEditError, editConfig, hasConfigPath, propertyText, } from "./con
 export { addVariable, ExistsError, VARIABLE_NAME } from "./variables.js";
 export { isGitIgnored, SECRET_NAME, setSecret } from "./secrets-file.js";
 export { cleanFileName, filesFolder, isFolderIgnored, RECORD_LIMIT, storeFile, } from "./files-store.js";
-export { loadConfig, } from "../config/resolved-config.js";
+export { loadConfig, writtenConfig, } from "../config/resolved-config.js";

package/dist/runner/options.d.ts

@@ -28,7 +28,13 @@ export interface RunnerOptions {
     headless?: boolean;
     /** Remote debugging port of the browser. @default DEFAULT_PORT (9223) */
     port?: number;
-    /** Where the report is written; `false` writes none. @default `<project>/result/report.json` */
+    /**
+     * The folder for what the run writes, instead of `<project>/result`: `report.json` and the named screenshots
+     * (their paths in the results are relative to it). Relative to the project folder.
+     * @default `<project>/result`
+     */
+    outputDir?: string;
+    /** Where the report is written; `false` writes none. Beats `outputDir` for the report. @default `report.json` in `outputDir` */
     reportPath?: string | false;
     /** Receives start and end of the run and of every test, as they happen */
     onEvent?: (event: RunEvent) => void;

package/dist/runner/output.d.ts

@@ -0,0 +1,6 @@
+import type { RunnerOptions } from "./options.js";
+/** Where a run writes: the folder its screenshots go in, and its report (`false`: none) */
+export declare function runOutput(options: Pick<RunnerOptions, "outputDir" | "reportPath">, projectDir: string): {
+    resultDir: string;
+    report: string | false;
+};

package/dist/runner/output.js

@@ -0,0 +1,14 @@
+import path from "node:path";
+/** Where a run writes: the folder its screenshots go in, and its report (`false`: none) */
+export function runOutput(options, projectDir) {
+    const { outputDir, reportPath } = options;
+    const resultDir = outputDir !== undefined
+        ? path.resolve(projectDir, outputDir)
+        : typeof reportPath === "string"
+            ? path.dirname(reportPath)
+            : path.join(projectDir, "result");
+    return {
+        resultDir,
+        report: reportPath ?? path.join(resultDir, "report.json"),
+    };
+}

package/dist/runner/run-test.js

@@ -1,4 +1,3 @@
-import path from "path";
 import { takePendingAssertions } from "../assert/expect.js";
 import { Browser } from "../browser/browser.js";
 import { Evidence } from "../browser/evidence.js";
@@ -11,6 +10,7 @@ import { judgePageLogs } from "./page-logs-report.js";
 import { DEFAULT_PORT } from "./ports.js";
 import { runWithEachHooks } from "./hooks.js";
 import { isSkipSignal, staticSkip } from "./skip.js";
+import { runOutput } from "./output.js";
 import { testDirOf } from "./test-dir.js";
 import { toTestError } from "./test-error.js";
 /**
@@ -75,10 +75,7 @@ export async function runOneTest(test, browserName, run, options, srcDir, report
     const outcome = new TestOutcome(reporter, test, browserName);
     const { settings, secrets } = run;
     const timeout = settings.timeout;
-    const { reportPath } = options;
-    const resultDir = typeof reportPath === "string"
-        ? path.dirname(reportPath)
-        : path.join(projectDir(), "result");
+    const { resultDir } = runOutput(options, projectDir());
     let browser;
     let page;
     let timeoutTimer;

package/dist/runner/test-runner.js

@@ -17,6 +17,7 @@ import { browsersOfRun, loadSpecs, selectTests } from "./select.js";
 import { resolveRetries } from "./retries-option.js";
 import { runWithRetries } from "./retry.js";
 import { resolveWorkers } from "../config/workers-option.js";
+import { runOutput } from "./output.js";
 import { WorkerPool } from "./worker-pool.js";
 import { processWorkerFactory } from "./worker-process.js";
 /** Finds the spec files, picks the tests and browsers of a run and runs each test in each browser */
@@ -61,7 +62,7 @@ export default class TestRunner {
      * was also written to the report. With more than one worker the tests run at the same time in worker processes.
      */
     async start() {
-        const { reportPath, onEvent, signal } = this.options;
+        const { onEvent, signal } = this.options;
         // The specs are imported once: their top-level code runs once per run
         const loaded = await loadSpecs(this.testFiles);
         const config = await loadRawConfig();
@@ -72,7 +73,7 @@ export default class TestRunner {
         const reporter = new TestReporter({
             environment: this.run.settings.environment,
             browsers,
-            ...(reportPath !== undefined && { output: reportPath }),
+            output: runOutput(this.options, projectDir()).report,
             ...(onEvent && { onEvent }),
         });
         // How the run goes is settled first: a bad `workers` is an error before any browser starts
@@ -159,7 +160,7 @@ export default class TestRunner {
     /** The pool of a run: up to `workers` worker processes that read the same project and settings as this run */
     createPool(workers, retries) {
         const { settings } = this.run;
-        const { reportPath, headless, port } = this.options;
+        const { reportPath, outputDir, headless, port } = this.options;
         const options = {
             workers,
             factory: processWorkerFactory(),
@@ -173,6 +174,7 @@ export default class TestRunner {
                 expectTimeout: settings.expectTimeout,
                 ...(headless !== undefined && { headless }),
                 ...(typeof reportPath === "string" && { reportPath }),
+                ...(outputDir !== undefined && { outputDir }),
                 files: this.testFiles,
                 basePort: port ?? DEFAULT_PORT,
                 ...(retries > 0 && { retries }),

package/dist/runner/worker-protocol.d.ts

@@ -11,6 +11,8 @@ export interface WorkItem {
     browser: SupportedBrowser;
     /** Items with the same key never run at the same time */
     serialKey?: string;
+    /** Where this test's screenshots go, instead of the run's `outputDir` (a flow gives each node its own) */
+    outputDir?: string;
 }
 /** What a worker needs to start: the project and the settings that are the same for every test */
 export interface WorkerInit {
@@ -24,6 +26,8 @@ export interface WorkerInit {
     retries?: number;
     /** Where the run's report goes when it is not the default; screenshots are kept beside it */
     reportPath?: string;
+    /** The run's `outputDir`: screenshots go there */
+    outputDir?: string;
     /** Spec files to import, absolute */
     files: string[];
     /** 0 for the first worker; picks the worker's browser port and profile folder */

package/dist/runner/worker-session.js

@@ -36,7 +36,7 @@ export function openSession(init) {
             });
             resolve({
                 tests: loaded.all,
-                async runTest(test, browser, signal, onEvent) {
+                async runTest(test, browser, signal, onEvent, perTest = {}) {
                     const reporter = new TestReporter({
                         environment: run.settings.environment,
                         browsers: [browser],
@@ -48,6 +48,9 @@ export function openSession(init) {
                         ...(init.reportPath !== undefined && {
                             reportPath: init.reportPath,
                         }),
+                        ...((perTest.outputDir ?? init.outputDir) !== undefined && {
+                            outputDir: perTest.outputDir ?? init.outputDir,
+                        }),
                         port,
                         signal,
                     };

package/dist/runner/worker.d.ts

@@ -6,7 +6,9 @@ import type { FromWorker, ToWorker, WorkerInit } from "./worker-protocol.js";
 export interface WorkerSession {
     tests: readonly RegisteredTestCase[];
     /** Runs one test; `onEvent` gets the reporter's events for it. Aborting `signal` ends the test as aborted */
-    runTest(test: RegisteredTestCase, browser: SupportedBrowser, signal: AbortSignal, onEvent: (event: RunEvent) => void): Promise<void>;
+    runTest(test: RegisteredTestCase, browser: SupportedBrowser, signal: AbortSignal, onEvent: (event: RunEvent) => void, options?: {
+        outputDir?: string;
+    }): Promise<void>;
     /** Ends the project's session */
     close(): Promise<void>;
 }

package/dist/runner/worker.js

@@ -54,7 +54,7 @@ export function handleMessages(channel, { open, exit }) {
                     id,
                     result: { ...event.result, worker: workerIndex },
                 });
-            });
+            }, item.outputDir !== undefined ? { outputDir: item.outputDir } : {});
         }
         finally {
             current = undefined;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@itmetsam/samurai-framework",
-  "version": "0.16.0",
+  "version": "0.18.0",
   "description": "Browser test framework over WebDriver BiDi: auto-waiting locators, environments and secrets, a recorder, and a command line.",
   "main": "dist/api.js",
   "exports": {