@itmetsam/samurai-framework 0.20.0 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/dist/api.d.ts +1 -1
  2. package/dist/assert/assertion-error.d.ts +3 -1
  3. package/dist/assert/assertion-error.js +2 -2
  4. package/dist/assert/locator-assertions.d.ts +35 -4
  5. package/dist/assert/locator-assertions.js +56 -28
  6. package/dist/assert/page-assertions.js +1 -1
  7. package/dist/assert/pending-assertions.d.ts +3 -2
  8. package/dist/assert/pending-assertions.js +5 -4
  9. package/dist/assert/text-match.d.ts +6 -0
  10. package/dist/assert/text-match.js +12 -0
  11. package/dist/capabilities.d.ts +16 -5
  12. package/dist/capabilities.js +16 -5
  13. package/dist/cli/flow-reporter.d.ts +1 -1
  14. package/dist/cli/flow-reporter.js +2 -2
  15. package/dist/cli/flow.js +4 -4
  16. package/dist/flows/core/check-io.d.ts +23 -0
  17. package/dist/flows/core/check-io.js +84 -0
  18. package/dist/flows/core/check.d.ts +16 -2
  19. package/dist/flows/core/check.js +20 -4
  20. package/dist/flows/core/expression/scope.d.ts +9 -2
  21. package/dist/flows/core/expression/scope.js +12 -4
  22. package/dist/flows/core/index.d.ts +2 -1
  23. package/dist/flows/core/index.js +2 -1
  24. package/dist/flows/core/nodes/condition.js +0 -1
  25. package/dist/flows/core/nodes/define.d.ts +0 -2
  26. package/dist/flows/core/nodes/group.js +0 -1
  27. package/dist/flows/core/nodes/test.js +0 -1
  28. package/dist/flows/core/schema.d.ts +10 -5
  29. package/dist/flows/core/schema.js +29 -1
  30. package/dist/flows/core/steps.d.ts +7 -0
  31. package/dist/flows/core/steps.js +15 -0
  32. package/dist/flows/core/wiring.d.ts +72 -0
  33. package/dist/flows/core/wiring.js +151 -0
  34. package/dist/flows/run/branch-context.d.ts +10 -4
  35. package/dist/flows/run/branch-context.js +6 -4
  36. package/dist/flows/run/events.d.ts +30 -2
  37. package/dist/flows/run/handlers/group.js +6 -2
  38. package/dist/flows/run/handlers/support.d.ts +7 -2
  39. package/dist/flows/run/handlers/support.js +36 -7
  40. package/dist/flows/run/handlers/test.js +6 -2
  41. package/dist/flows/run/handlers/types.d.ts +11 -3
  42. package/dist/flows/run/node-folder.d.ts +8 -3
  43. package/dist/flows/run/node-folder.js +28 -4
  44. package/dist/flows/run/node-inputs.d.ts +5 -0
  45. package/dist/flows/run/node-inputs.js +62 -0
  46. package/dist/flows/run/node-pool.d.ts +2 -2
  47. package/dist/flows/run/node-pool.js +6 -6
  48. package/dist/flows/run/node-runner.d.ts +24 -0
  49. package/dist/flows/run/node-runner.js +48 -0
  50. package/dist/flows/run/per-browser.d.ts +21 -0
  51. package/dist/flows/run/per-browser.js +46 -0
  52. package/dist/flows/run/project-facts.d.ts +13 -0
  53. package/dist/flows/run/project-facts.js +12 -0
  54. package/dist/flows/run/project-options.d.ts +9 -2
  55. package/dist/flows/run/project.d.ts +13 -8
  56. package/dist/flows/run/project.js +48 -51
  57. package/dist/flows/run/report.d.ts +7 -7
  58. package/dist/flows/run/report.js +3 -5
  59. package/dist/flows/run/run.d.ts +21 -7
  60. package/dist/flows/run/run.js +21 -11
  61. package/dist/flows/run/walker.d.ts +23 -7
  62. package/dist/flows/run/walker.js +18 -7
  63. package/dist/locator/locator.d.ts +3 -1
  64. package/dist/locator/locator.js +6 -0
  65. package/dist/recorder/apply.js +17 -7
  66. package/dist/recorder/normalise.d.ts +18 -3
  67. package/dist/recorder/normalise.js +18 -3
  68. package/dist/recorder/session-secrets.js +4 -0
  69. package/dist/recorder/toolbar/action-guard.d.ts +2 -5
  70. package/dist/recorder/toolbar/action-guard.js +9 -5
  71. package/dist/recorder/toolbar/actions/insert.d.ts +5 -2
  72. package/dist/recorder/toolbar/actions/insert.js +43 -7
  73. package/dist/recorder/toolbar/actions/pick.js +2 -0
  74. package/dist/recorder/toolbar/actions/reply.d.ts +2 -0
  75. package/dist/recorder/toolbar/actions/reply.js +9 -0
  76. package/dist/recorder/toolbar/actions/save-output.d.ts +9 -0
  77. package/dist/recorder/toolbar/actions/save-output.js +46 -0
  78. package/dist/recorder/toolbar/controller-fixture.d.ts +2 -2
  79. package/dist/recorder/toolbar/controller-fixture.js +2 -0
  80. package/dist/recorder/toolbar/controller.d.ts +2 -1
  81. package/dist/recorder/toolbar/controller.js +14 -7
  82. package/dist/recorder/toolbar/insertables.d.ts +7 -0
  83. package/dist/recorder/toolbar/insertables.js +12 -0
  84. package/dist/recorder/toolbar/inspect.js +3 -1
  85. package/dist/recorder/toolbar/protocol.d.ts +20 -3
  86. package/dist/recorder/toolbar/source/events-page.js +2 -1
  87. package/dist/recorder/toolbar/source/fields.js +8 -0
  88. package/dist/recorder/toolbar/source/handlers.js +19 -12
  89. package/dist/recorder/toolbar/source/picking.js +22 -7
  90. package/dist/recorder/toolbar/source/recorder-side.js +6 -0
  91. package/dist/recorder/toolbar/source/setup.js +3 -3
  92. package/dist/recorder/toolbar/source/views-panels.d.ts +1 -1
  93. package/dist/recorder/toolbar/source/views-panels.js +50 -10
  94. package/dist/recorder/toolbar/source/views-rail.js +5 -2
  95. package/dist/recorder/toolbar/source.d.ts +5 -5
  96. package/dist/recorder/toolbar/source.js +5 -5
  97. package/dist/recorder/toolbar/types.d.ts +10 -3
  98. package/dist/runner/options.d.ts +5 -0
  99. package/dist/runner/pooled-run.d.ts +3 -2
  100. package/dist/runner/pooled-run.js +4 -2
  101. package/dist/runner/registry.d.ts +7 -3
  102. package/dist/runner/registry.js +29 -11
  103. package/dist/runner/reporter.d.ts +4 -0
  104. package/dist/runner/reporter.js +2 -0
  105. package/dist/runner/run-test.js +23 -2
  106. package/dist/runner/run.d.ts +7 -0
  107. package/dist/runner/run.js +8 -1
  108. package/dist/runner/test-io.d.ts +30 -0
  109. package/dist/runner/test-io.js +111 -0
  110. package/dist/runner/test-runner.js +1 -1
  111. package/dist/runner/worker-protocol.d.ts +2 -0
  112. package/dist/runner/worker-session.js +1 -0
  113. package/dist/runner/worker.d.ts +1 -0
  114. package/dist/runner/worker.js +4 -1
  115. package/dist/steps/edit.d.ts +14 -3
  116. package/dist/steps/edit.js +65 -23
  117. package/dist/steps/emit.d.ts +2 -0
  118. package/dist/steps/emit.js +32 -8
  119. package/dist/steps/index.d.ts +1 -1
  120. package/dist/steps/model.d.ts +42 -2
  121. package/dist/steps/parse/expectation.d.ts +8 -1
  122. package/dist/steps/parse/expectation.js +17 -11
  123. package/dist/steps/parse/step.js +64 -8
  124. package/dist/steps/parse.js +25 -16
  125. package/dist/steps/test-options.d.ts +25 -0
  126. package/dist/steps/test-options.js +158 -0
  127. package/dist/steps/text.d.ts +7 -0
  128. package/dist/steps/text.js +12 -0
  129. package/dist/types/test.d.ts +34 -0
  130. package/dist/wait/output-sink.d.ts +24 -0
  131. package/dist/wait/output-sink.js +59 -0
  132. package/package.json +1 -1
@@ -9,22 +9,22 @@
9
9
  * `globalThis[Symbol.for("samurai.toolbar")]` in the sandbox holds the hook:
10
10
  * - `update(state)`: the recorder's state, after every change and every page load.
11
11
  * - `reply(id, result)`: the answer to `pick` (a `PickInfo`), to `page-info` (a `PageInfo`) or to `assert`,
12
- * `assert-page`, `insert` and `add-insertable` (an `ActionReply`).
12
+ * `assert-page`, `insert`, `add-insertable` and `save-output` (an `ActionReply`).
13
13
  * - `root`: the shadow root, for tests.
14
14
  * - `press(action, detail?)`: runs the handler a click on that control runs (for tests). Actions:
15
15
  * `pause`, `resume`, `stop`, `collapse`, `expand`, `resume-yes`, `resume-no` (rail);
16
- * `assert`, `page`, `insert`, `wait`, `screenshot` (open or close the tool);
16
+ * `assert`, `output`, `page`, `insert`, `wait`, `screenshot` (open or close the tool);
17
17
  * `pick { target }` (the element a picking click hit);
18
- * `matcher { matcher }`, `not`, `assert-add` (Assert panel);
18
+ * `matcher { matcher }`, `not`, `assert-add` (Assert panel); `output-save` (Save as output panel);
19
19
  * `page-matcher { matcher }`, `page-not`, `page-add` (Page panel);
20
20
  * `insert-item { kind, name }`, `insert-first`, `insert-new { kind }` (Insert list);
21
21
  * `new-kind { kind }`, `new-save`, `new-cancel` (New variable or secret);
22
22
  * `wait-add`, `screenshot-add`; `escape` (the Esc key); `dismiss { id }` (a warning's Dismiss button).
23
- * Form inputs carry `data-field` (`expected`, `attribute`, `cssName`, `countIndex`, `pageExpected`, `filter`,
23
+ * Form inputs carry `data-field` (`expected`, `attribute`, `cssName`, `countIndex`, `outputName`, `outputPattern`, `pageExpected`, `filter`,
24
24
  * `name`, `value`, `waitKind`, `seconds`, `shotName`, `fullPage`) and update on `input`/`change`.
25
25
  * - `rebind(send)`: used by a second run in the same document (a new recording), which keeps the host and
26
26
  * swaps in that recording's toolbar channel.
27
- * - `state()`, `picking()`: the last state from `update` and whether the Assert picker is on (for tests).
27
+ * - `state()`, `picking()`: the last state from `update` and whether the Assert or Save as output picker is on (for tests).
28
28
  * - `dispose()`: removes the host and every listener the toolbar put on the window and document (the
29
29
  * recording ended while the document stays open).
30
30
  *
@@ -24,22 +24,22 @@ import { ATTACH_JS } from "./source/attach.js";
24
24
  * `globalThis[Symbol.for("samurai.toolbar")]` in the sandbox holds the hook:
25
25
  * - `update(state)`: the recorder's state, after every change and every page load.
26
26
  * - `reply(id, result)`: the answer to `pick` (a `PickInfo`), to `page-info` (a `PageInfo`) or to `assert`,
27
- * `assert-page`, `insert` and `add-insertable` (an `ActionReply`).
27
+ * `assert-page`, `insert`, `add-insertable` and `save-output` (an `ActionReply`).
28
28
  * - `root`: the shadow root, for tests.
29
29
  * - `press(action, detail?)`: runs the handler a click on that control runs (for tests). Actions:
30
30
  * `pause`, `resume`, `stop`, `collapse`, `expand`, `resume-yes`, `resume-no` (rail);
31
- * `assert`, `page`, `insert`, `wait`, `screenshot` (open or close the tool);
31
+ * `assert`, `output`, `page`, `insert`, `wait`, `screenshot` (open or close the tool);
32
32
  * `pick { target }` (the element a picking click hit);
33
- * `matcher { matcher }`, `not`, `assert-add` (Assert panel);
33
+ * `matcher { matcher }`, `not`, `assert-add` (Assert panel); `output-save` (Save as output panel);
34
34
  * `page-matcher { matcher }`, `page-not`, `page-add` (Page panel);
35
35
  * `insert-item { kind, name }`, `insert-first`, `insert-new { kind }` (Insert list);
36
36
  * `new-kind { kind }`, `new-save`, `new-cancel` (New variable or secret);
37
37
  * `wait-add`, `screenshot-add`; `escape` (the Esc key); `dismiss { id }` (a warning's Dismiss button).
38
- * Form inputs carry `data-field` (`expected`, `attribute`, `cssName`, `countIndex`, `pageExpected`, `filter`,
38
+ * Form inputs carry `data-field` (`expected`, `attribute`, `cssName`, `countIndex`, `outputName`, `outputPattern`, `pageExpected`, `filter`,
39
39
  * `name`, `value`, `waitKind`, `seconds`, `shotName`, `fullPage`) and update on `input`/`change`.
40
40
  * - `rebind(send)`: used by a second run in the same document (a new recording), which keeps the host and
41
41
  * swaps in that recording's toolbar channel.
42
- * - `state()`, `picking()`: the last state from `update` and whether the Assert picker is on (for tests).
42
+ * - `state()`, `picking()`: the last state from `update` and whether the Assert or Save as output picker is on (for tests).
43
43
  * - `dispose()`: removes the host and every listener the toolbar put on the window and document (the
44
44
  * recording ended while the document stays open).
45
45
  *
@@ -47,6 +47,8 @@ export interface ControllerOptions {
47
47
  /** Name → value; undefined when not set */
48
48
  secrets: Record<string, string | undefined>;
49
49
  secretsIgnored: boolean;
50
+ /** The test's declared inputs: name → default */
51
+ inputs?: Record<string, string>;
50
52
  /** Screenshot steps already in the test, for the default name */
51
53
  screenshots: number;
52
54
  }
@@ -56,20 +58,25 @@ export type Message = ToolbarMessage & {
56
58
  export type Of<A extends Message["action"]> = Extract<Message, {
57
59
  action: A;
58
60
  }>;
59
- /** The element picked for an Assert, and the locator behind each of its Count choices */
61
+ /** The element picked for an Assert or Save as output, the locator behind each of its Count choices, and its text */
60
62
  export interface PickState {
61
63
  id: number;
62
64
  locator: LocatorSpec;
63
65
  counts: LocatorCall[];
66
+ /** Its text as `outputs.save` reads it, when it was picked, secrets included: it never goes back to the page */
67
+ text: string;
64
68
  }
65
- /** What the action handlers (assert, insert, add a variable or secret) use of the controller */
69
+ /** The messages that expect a reply, and that a double click sends twice */
70
+ export type Replying = Of<"assert" | "assert-page" | "insert" | "add-insertable" | "save-output">;
71
+ /** What the action handlers (assert, insert, add a variable, secret or input, save as output) use of the controller */
66
72
  export interface ActionContext {
67
73
  readonly deps: ControllerDeps;
68
74
  readonly options: ControllerOptions;
69
75
  readonly variables: Map<string, string>;
70
76
  readonly secrets: Map<string, string | undefined>;
77
+ readonly inputs: Map<string, string>;
71
78
  isPaused(): boolean;
72
- /** The picked element an `assert` with the same id refers to */
79
+ /** The picked element an `assert` or `save-output` with the same id refers to */
73
80
  pick(): PickState | undefined;
74
81
  setPick(pick: PickState | undefined): void;
75
82
  showToast(text: string): void;
@@ -24,6 +24,11 @@ export interface RunnerOptions {
24
24
  * @default the config's `retries`, else 0
25
25
  */
26
26
  retries?: number | string;
27
+ /**
28
+ * Values for the inputs the tests declare, by name: every test of the run reads the names it declares from
29
+ * here, and its own default for a name missing here
30
+ */
31
+ inputs?: Record<string, string>;
27
32
  /** Run the browser without a window. @default false */
28
33
  headless?: boolean;
29
34
  /** Remote debugging port of the browser. @default DEFAULT_PORT (9223) */
@@ -16,9 +16,10 @@ export interface PlannedTest {
16
16
  }
17
17
  /**
18
18
  * Lists the run: every selected test in the first browser, then every selected test in the next one. The
19
- * position in the list is the item's `index`. `serialKey` says which group makes a test run alone.
19
+ * position in the list is the item's `index`. `serialKey` says which group makes a test run alone. `inputs`, the
20
+ * run's input values, go with every item.
20
21
  */
21
- export declare function planItems(selected: RegisteredTestCase[], browsers: SupportedBrowser[], serialKey: (test: RegisteredTestCase) => string | undefined): {
22
+ export declare function planItems(selected: RegisteredTestCase[], browsers: SupportedBrowser[], serialKey: (test: RegisteredTestCase) => string | undefined, inputs?: Record<string, string>): {
22
23
  items: WorkItem[];
23
24
  planned: PlannedTest[];
24
25
  };
@@ -2,9 +2,10 @@ import { idOf } from "./select.js";
2
2
  import { staticSkip } from "./skip.js";
3
3
  /**
4
4
  * Lists the run: every selected test in the first browser, then every selected test in the next one. The
5
- * position in the list is the item's `index`. `serialKey` says which group makes a test run alone.
5
+ * position in the list is the item's `index`. `serialKey` says which group makes a test run alone. `inputs`, the
6
+ * run's input values, go with every item.
6
7
  */
7
- export function planItems(selected, browsers, serialKey) {
8
+ export function planItems(selected, browsers, serialKey, inputs) {
8
9
  const planned = browsers.flatMap((browser) => selected.map((test) => ({ test, browser })));
9
10
  // A test that does not run (skip, fixme) is in the plan but not in the queue; `index` is its place in the plan
10
11
  const items = planned.flatMap(({ test, browser }, index) => {
@@ -18,6 +19,7 @@ export function planItems(selected, browsers, serialKey) {
18
19
  testName: test.name,
19
20
  browser,
20
21
  ...(key !== undefined && { serialKey: key }),
22
+ ...(inputs !== undefined && { inputs }),
21
23
  },
22
24
  ];
23
25
  });
@@ -1,14 +1,18 @@
1
- import type { AllHook, EachHook, RegisteredTestCase, TestCaseBase, TestDetails } from "../types/test.js";
1
+ import type { AllHook, EachHook, RegisteredTestCase, TestCaseBase, TestDetails, TestOptions } from "../types/test.js";
2
2
  type Body = TestCaseBase["function"];
3
3
  /** `test(title, fn)`, `test.skip(title, fn)`, `test.skip(condition, reason)` and the other forms */
4
4
  export interface TestFunction {
5
5
  (title: string, fn: Body): void;
6
+ (title: string, options: TestOptions, fn: Body): void;
6
7
  /** Registers a test that is not run */
7
8
  skip: SkipFunction;
8
9
  /** Registers a test that is not run because it is known to be broken */
9
10
  fixme: SkipFunction;
10
11
  /** Runs only this test (and the other `only` ones) */
11
- only(title: string, fn: Body): void;
12
+ only: {
13
+ (title: string, fn: Body): void;
14
+ (title: string, options: TestOptions, fn: Body): void;
15
+ };
12
16
  }
13
17
  /**
14
18
  * With a title and a body it registers a skipped test, `{ reason }` saying why; with a condition it skips the
@@ -16,7 +20,7 @@ export interface TestFunction {
16
20
  */
17
21
  interface SkipFunction {
18
22
  (title: string, fn: Body): void;
19
- (title: string, details: TestDetails, fn: Body): void;
23
+ (title: string, options: TestOptions, fn: Body): void;
20
24
  (condition: boolean, reason?: string): void;
21
25
  }
22
26
  /** `describe.skip` and `describe.fixme`: every test inside gets the modifier, and `{ reason }` says why */
@@ -1,6 +1,7 @@
1
1
  import { fileURLToPath } from "node:url";
2
2
  import logger from "../logger/index.js";
3
3
  import { SkipSignal } from "./skip.js";
4
+ import { validateTestIO } from "./test-io.js";
4
5
  import { addHook, currentSuite, enterSuite, leaveSuite, modifierOf, resetSuites, } from "./suite.js";
5
6
  const registered = [];
6
7
  /** Runs `fn` as the `describe` block `title`, under `modifier` when there is one. `fn` runs immediately and must be synchronous. */
@@ -17,7 +18,8 @@ function openDescribe(file, title, fn, modifier, reason) {
17
18
  leaveSuite();
18
19
  }
19
20
  }
20
- function add(file, title, fn, modifier, reason) {
21
+ function add(file, title, fn, modifier, extras = {}) {
22
+ const { reason, inputs, outputs } = extras;
21
23
  const suite = currentSuite(file);
22
24
  const titlePath = [...titlesOf(suite), title];
23
25
  const name = titlePath.join(" > ");
@@ -28,6 +30,8 @@ function add(file, title, fn, modifier, reason) {
28
30
  file,
29
31
  suite,
30
32
  ...modifierOf({ modifier, ...(reason !== undefined && { reason }) }, suite),
33
+ ...(inputs && { inputs }),
34
+ ...(outputs && { outputs }),
31
35
  });
32
36
  logger.debug("Registered test %s", name);
33
37
  }
@@ -39,23 +43,37 @@ function titlesOf(suite) {
39
43
  titles.unshift(s.title);
40
44
  return titles;
41
45
  }
42
- /** The body and `{ reason }` of `(title, fn)` and `(title, details, fn)` */
43
- function bodyAndReason(call, title, second, third) {
44
- const [details, fn] = typeof second === "function"
46
+ /** The body and options of `(title, fn)` and `(title, options, fn)`; `describe` only reads `reason` from them */
47
+ function bodyAndOptions(call, title, second, third) {
48
+ const [options, fn] = typeof second === "function"
45
49
  ? [undefined, second]
46
50
  : [second, third];
47
51
  if (typeof fn !== "function")
48
52
  throw new TypeError(`${call}(${JSON.stringify(title)}) needs a function as its last argument`);
49
- const reason = details?.reason;
50
- return { fn: fn, ...(reason !== undefined && { reason }) };
53
+ const reason = options?.reason;
54
+ return {
55
+ fn: fn,
56
+ ...(reason !== undefined && { reason }),
57
+ ...(options && call.startsWith("test") && validateTestIO(call, options)),
58
+ };
59
+ }
60
+ /** `test()` and `test.only()`: a reason only makes sense with skip and fixme */
61
+ function registering(call, modifier) {
62
+ return ((title, second, third) => {
63
+ const { fn, reason, ...io } = bodyAndOptions(call, title, second, third);
64
+ if (reason !== undefined)
65
+ throw new Error(`${call}(): reason only applies to skip and fixme`);
66
+ // Frames: Error, callerFile, this function, the spec
67
+ add(callerFile(), title, fn, modifier, io);
68
+ });
51
69
  }
52
70
  /** `test.skip` and `test.fixme`: a title and a body register a test with the modifier; a condition skips the running test */
53
71
  function skipping(annotation) {
54
72
  return ((first, second, third) => {
55
73
  if (typeof first === "string") {
56
- const { fn, reason } = bodyAndReason(`test.${annotation}`, first, second, third);
74
+ const { fn, ...extras } = bodyAndOptions(`test.${annotation}`, first, second, third);
57
75
  // Frames: Error, callerFile, this function, the spec
58
- add(callerFile(), first, fn, annotation, reason);
76
+ add(callerFile(), first, fn, annotation, extras);
59
77
  return;
60
78
  }
61
79
  if (first)
@@ -65,14 +83,14 @@ function skipping(annotation) {
65
83
  /** `describe.skip` and `describe.fixme`, with or without `{ reason }` */
66
84
  function describeSkipping(annotation) {
67
85
  return ((title, second, third) => {
68
- const { fn, reason } = bodyAndReason(`describe.${annotation}`, title, second, third);
86
+ const { fn, reason } = bodyAndOptions(`describe.${annotation}`, title, second, third);
69
87
  openDescribe(callerFile(), title, fn, annotation, reason);
70
88
  });
71
89
  }
72
- export const test = Object.assign((title, fn) => add(callerFile(), title, fn), {
90
+ export const test = Object.assign(registering("test"), {
73
91
  skip: skipping("skip"),
74
92
  fixme: skipping("fixme"),
75
- only: (title, fn) => add(callerFile(), title, fn, "only"),
93
+ only: registering("test.only", "only"),
76
94
  });
77
95
  export const describe = Object.assign((title, fn) => openDescribe(callerFile(), title, fn), {
78
96
  skip: describeSkipping("skip"),
@@ -56,6 +56,10 @@ export interface TestEndExtras extends TestRetries {
56
56
  name: string;
57
57
  path: string;
58
58
  }[];
59
+ /** The test's inputs, when it declares any */
60
+ inputs?: Record<string, string>;
61
+ /** The test's saved outputs, when it declares any */
62
+ outputs?: Record<string, string>;
59
63
  }
60
64
  export default class TestReporter implements TestSink {
61
65
  private testResults;
@@ -146,6 +146,8 @@ export default class TestReporter {
146
146
  ...(extras?.logs && { logs: extras.logs }),
147
147
  ...(extras?.logsDropped && { logsDropped: extras.logsDropped }),
148
148
  ...(extras?.screenshots?.length && { screenshots: extras.screenshots }),
149
+ ...(extras?.inputs && { inputs: extras.inputs }),
150
+ ...(extras?.outputs && { outputs: extras.outputs }),
149
151
  ...(extras?.retry && { retry: extras.retry }),
150
152
  ...(extras?.flaky && { flaky: true }),
151
153
  ...(extras?.previousAttempts?.length && {
@@ -13,6 +13,8 @@ import { isSkipSignal, staticSkip } from "./skip.js";
13
13
  import { runOutput } from "./output.js";
14
14
  import { testDirOf } from "./test-dir.js";
15
15
  import { toTestError } from "./test-error.js";
16
+ import { createInputsFixture, createOutputsFixture, ioOfAttempt, missingOutputs, } from "./test-io.js";
17
+ import { beginOutputs, endOutputs } from "../wait/output-sink.js";
16
18
  /**
17
19
  * The state of one test run. A test ends once, whichever of finishing, failing, timing out, an abort or a
18
20
  * closed browser comes first: that one is reported and the rest are ignored.
@@ -28,6 +30,8 @@ class TestOutcome {
28
30
  this.closing = false;
29
31
  /** A person closed the browser, or it crashed, while the test ran */
30
32
  this.browserClosed = false;
33
+ /** The test's inputs and outputs, as its report entry lists them when it ends */
34
+ this.io = () => ({});
31
35
  }
32
36
  get isEnded() {
33
37
  return this.ended;
@@ -49,6 +53,7 @@ class TestOutcome {
49
53
  this.reporter.onTestEnd(this.test, this.browser, error, {
50
54
  ...logs,
51
55
  screenshots: this.evidence?.screenshots,
56
+ ...this.io(),
52
57
  });
53
58
  }
54
59
  }
@@ -137,6 +142,8 @@ export async function runOneTest(test, browserName, run, options, srcDir, report
137
142
  workerIndex: resources.workerIndex ?? 0,
138
143
  env: createEnvFixture(settings.environment, settings.variables),
139
144
  secrets: createSecretsFixture(settings.environment, secrets),
145
+ inputs: createInputsFixture(test.inputs, options.inputs),
146
+ outputs: createOutputsFixture(sink),
140
147
  };
141
148
  await runWithEachHooks(test.suite, fixtures, timeoutController.signal, () => test.function(fixtures));
142
149
  // Already reported as timed out
@@ -146,12 +153,18 @@ export async function runOneTest(test, browserName, run, options, srcDir, report
146
153
  if (unawaited.length > 0) {
147
154
  outcome.finish({
148
155
  message: unawaited
149
- .map((matcher) => `expect(locator).${matcher}() was not awaited`)
156
+ .map((label) => `${label} was not awaited`)
150
157
  .join("\n"),
151
158
  type: "assertion",
152
159
  }, await judgeLogs(page, true));
153
160
  return;
154
161
  }
162
+ // Checked only after a pass: a test that failed first reports its own failure
163
+ const missing = missingOutputs(test.outputs, sink);
164
+ if (missing) {
165
+ outcome.finish(missing, await judgeLogs(page, true));
166
+ return;
167
+ }
155
168
  const verdict = await judgeLogs(page, false);
156
169
  outcome.finish(verdict.error, verdict);
157
170
  }
@@ -168,7 +181,15 @@ export async function runOneTest(test, browserName, run, options, srcDir, report
168
181
  : toTestError(err), await judgeLogs(page, true));
169
182
  }
170
183
  };
171
- await Promise.race([limit, gone, runBody()]);
184
+ // Open for this attempt only: whatever a capture still saves after the attempt ended is ignored
185
+ const sink = beginOutputs(test.outputs ?? []);
186
+ outcome.io = () => ioOfAttempt(test, options.inputs, sink);
187
+ try {
188
+ await Promise.race([limit, gone, runBody()]);
189
+ }
190
+ finally {
191
+ endOutputs();
192
+ }
172
193
  clearTimeout(timeoutTimer);
173
194
  if (onRunAborted)
174
195
  options.signal?.removeEventListener("abort", onRunAborted);
@@ -30,6 +30,13 @@ export interface FoundTest {
30
30
  modifier?: Modifier;
31
31
  /** The `{ reason }` given with the `skip` or `fixme` that applies */
32
32
  reason?: string;
33
+ /** The inputs the test declares, in declaration order, with their defaults; absent when it declares none */
34
+ inputs?: {
35
+ name: string;
36
+ default: string;
37
+ }[];
38
+ /** The names of the outputs the test declares; absent when it declares none */
39
+ outputs?: string[];
33
40
  }
34
41
  /**
35
42
  * Imports the project's specs without running anything and lists the tests the options select, each with the id
@@ -43,7 +43,7 @@ export async function listTests(options = {}) {
43
43
  return inProject(options, async (runner) => {
44
44
  const { tests, srcDir } = await runner.list();
45
45
  return tests.map((test) => {
46
- const { name, file, modifier, reason } = test;
46
+ const { name, file, modifier, reason, inputs, outputs } = test;
47
47
  const relativeFile = toPosix(path.relative(srcDir, file));
48
48
  return {
49
49
  id: testId(relativeFile, name),
@@ -52,6 +52,13 @@ export async function listTests(options = {}) {
52
52
  relativeFile,
53
53
  ...(modifier && { modifier }),
54
54
  ...(reason !== undefined && { reason }),
55
+ ...(inputs && {
56
+ inputs: Object.entries(inputs).map(([name, value]) => ({
57
+ name,
58
+ default: value,
59
+ })),
60
+ }),
61
+ ...(outputs && { outputs }),
55
62
  };
56
63
  });
57
64
  });
@@ -0,0 +1,30 @@
1
+ import type { TestError, TestOptions, TestOutputs } from "../types/test.js";
2
+ import type { OutputSink } from "../wait/output-sink.js";
3
+ /**
4
+ * Checks the `inputs` and `outputs` of a `test()` options object and returns them, leaving out an empty or absent
5
+ * list. `call` is the form used (`test`, `test.only`, ...), named in the error.
6
+ */
7
+ export declare function validateTestIO(call: string, options: TestOptions): {
8
+ inputs?: Record<string, string>;
9
+ outputs?: string[];
10
+ };
11
+ /**
12
+ * The `inputs` fixture: each declared input with the value the run supplies, else its default. Reading a
13
+ * name the test did not declare throws.
14
+ */
15
+ export declare function createInputsFixture(declared: Record<string, string> | undefined, supplied: Record<string, string> | undefined): Readonly<Record<string, string>>;
16
+ /**
17
+ * The `outputs` fixture of one attempt: `save` waits for the element's text (non-empty, or matching the pattern), as
18
+ * `toHaveText` reads it, and keeps it in `sink`. It keeps to its own attempt's sink, so a late value never lands in a later test.
19
+ */
20
+ export declare function createOutputsFixture(sink: OutputSink): TestOutputs;
21
+ /** The failure of a passing attempt that did not save every output it declared */
22
+ export declare function missingOutputs(declared: readonly string[] | undefined, sink: OutputSink): TestError | undefined;
23
+ /** What an attempt's result lists: its inputs when it declares any, its outputs so far when it declares any */
24
+ export declare function ioOfAttempt(test: {
25
+ inputs?: Record<string, string>;
26
+ outputs?: string[];
27
+ }, supplied: Record<string, string> | undefined, sink: OutputSink): {
28
+ inputs?: Record<string, string>;
29
+ outputs?: Record<string, string>;
30
+ };
@@ -0,0 +1,111 @@
1
+ import { captured, textCheck } from "../assert/text-match.js";
2
+ import { readEventually } from "../assert/locator-assertions.js";
3
+ import { track } from "../assert/pending-assertions.js";
4
+ import { isPassthroughKey } from "../config/variables.js";
5
+ import { isIoName } from "../flows/core/steps.js";
6
+ function checkName(call, kind, name) {
7
+ if (typeof name !== "string" || !isIoName(name))
8
+ throw new Error(`${call}(): ${kind} ${quoted(name)} is not a valid name`);
9
+ }
10
+ const has = (object, key) => Object.prototype.hasOwnProperty.call(object, key);
11
+ /** A name in a message: strings quoted, anything else as written */
12
+ function quoted(name) {
13
+ return typeof name === "string" ? JSON.stringify(name) : String(name);
14
+ }
15
+ /**
16
+ * Checks the `inputs` and `outputs` of a `test()` options object and returns them, leaving out an empty or absent
17
+ * list. `call` is the form used (`test`, `test.only`, ...), named in the error.
18
+ */
19
+ export function validateTestIO(call, options) {
20
+ const { inputs, outputs } = options;
21
+ const result = {};
22
+ if (inputs !== undefined) {
23
+ if (typeof inputs !== "object" || inputs === null || Array.isArray(inputs))
24
+ throw new Error(`${call}(): inputs must be an object of names and defaults`);
25
+ for (const [name, value] of Object.entries(inputs)) {
26
+ checkName(call, "input", name);
27
+ if (typeof value !== "string")
28
+ throw new Error(`${call}(): input "${name}" needs a string default`);
29
+ }
30
+ if (Object.keys(inputs).length > 0)
31
+ result.inputs = { ...inputs };
32
+ }
33
+ if (outputs !== undefined) {
34
+ if (!Array.isArray(outputs))
35
+ throw new Error(`${call}(): outputs must be an array of names`);
36
+ const seen = new Set();
37
+ for (const name of outputs) {
38
+ checkName(call, "output", name);
39
+ if (seen.has(name))
40
+ throw new Error(`${call}(): output "${name}" is listed twice`);
41
+ seen.add(name);
42
+ }
43
+ if (outputs.length > 0)
44
+ result.outputs = [...outputs];
45
+ }
46
+ return result;
47
+ }
48
+ /**
49
+ * The `inputs` fixture: each declared input with the value the run supplies, else its default. Reading a
50
+ * name the test did not declare throws.
51
+ */
52
+ export function createInputsFixture(declared, supplied) {
53
+ return new Proxy(Object.freeze(resolveInputs(declared, supplied)), {
54
+ get(target, key, receiver) {
55
+ if (isPassthroughKey(target, key) || has(target, key))
56
+ return Reflect.get(target, key, receiver);
57
+ throw new Error(`inputs.${String(key)}: not declared, add it to the test's inputs`);
58
+ },
59
+ });
60
+ }
61
+ /** Each declared input with the supplied value, else its default; names the test does not declare are left out */
62
+ function resolveInputs(declared, supplied) {
63
+ return Object.fromEntries(Object.entries(declared ?? {}).map(([name, value]) => [
64
+ name,
65
+ supplied && has(supplied, name) ? supplied[name] : value,
66
+ ]));
67
+ }
68
+ /**
69
+ * The `outputs` fixture of one attempt: `save` waits for the element's text (non-empty, or matching the pattern), as
70
+ * `toHaveText` reads it, and keeps it in `sink`. It keeps to its own attempt's sink, so a late value never lands in a later test.
71
+ */
72
+ export function createOutputsFixture(sink) {
73
+ return {
74
+ save: (name, locator, pattern) => track("outputs.save()", saveText(sink, name, locator, pattern)),
75
+ };
76
+ }
77
+ async function saveText(sink, name, locator, pattern) {
78
+ if (!sink.declares(name))
79
+ throw new Error(`outputs.save(): output ${JSON.stringify(name)} is not declared`);
80
+ const matches = pattern ? textCheck(pattern, "exact") : undefined;
81
+ const text = await readEventually({
82
+ name: "toHaveText",
83
+ locator,
84
+ expected: pattern ?? "an element with text",
85
+ read: () => locator.textContent(),
86
+ // Without a pattern, an element that has not rendered its value yet (no text) is waited for
87
+ check: (actual) => actual !== null && (matches ? matches(actual) : actual !== ""),
88
+ call: `outputs.save(${JSON.stringify(name)})`,
89
+ });
90
+ sink.set(name, pattern ? captured(pattern, text) : text);
91
+ }
92
+ /** The failure of a passing attempt that did not save every output it declared */
93
+ export function missingOutputs(declared, sink) {
94
+ const saved = sink.values();
95
+ const missing = (declared ?? []).filter((name) => !has(saved, name));
96
+ if (missing.length === 0)
97
+ return undefined;
98
+ return {
99
+ message: missing
100
+ .map((name) => `output ${JSON.stringify(name)} was never saved`)
101
+ .join("\n"),
102
+ type: "assertion",
103
+ };
104
+ }
105
+ /** What an attempt's result lists: its inputs when it declares any, its outputs so far when it declares any */
106
+ export function ioOfAttempt(test, supplied, sink) {
107
+ return {
108
+ ...(test.inputs && { inputs: resolveInputs(test.inputs, supplied) }),
109
+ ...(test.outputs && { outputs: sink.values() }),
110
+ };
111
+ }
@@ -88,7 +88,7 @@ export default class TestRunner {
88
88
  env: process.env.SAMURAI_RETRIES,
89
89
  config: config.retries,
90
90
  });
91
- const plan = planItems(selected, browsers, serialKeyOf(loaded.groups, config.groups, loaded.srcDir));
91
+ const plan = planItems(selected, browsers, serialKeyOf(loaded.groups, config.groups, loaded.srcDir), this.options.inputs);
92
92
  // A pool larger than what can overlap would only hold idle workers: a group that runs one at a time needs one
93
93
  const lanes = concurrency(plan.items);
94
94
  const mode = poolMode({
@@ -13,6 +13,8 @@ export interface WorkItem {
13
13
  serialKey?: string;
14
14
  /** Where this test's screenshots go, instead of the run's `outputDir` (a flow gives each node its own) */
15
15
  outputDir?: string;
16
+ /** The values the run supplies for the inputs tests declare (`runTests({ inputs })`) */
17
+ inputs?: Record<string, string>;
16
18
  }
17
19
  /** What a worker needs to start: the project and the settings that are the same for every test */
18
20
  export interface WorkerInit {
@@ -51,6 +51,7 @@ export function openSession(init) {
51
51
  ...((perTest.outputDir ?? init.outputDir) !== undefined && {
52
52
  outputDir: perTest.outputDir ?? init.outputDir,
53
53
  }),
54
+ ...(perTest.inputs !== undefined && { inputs: perTest.inputs }),
54
55
  port,
55
56
  signal,
56
57
  };
@@ -8,6 +8,7 @@ export interface WorkerSession {
8
8
  /** Runs one test; `onEvent` gets the reporter's events for it. Aborting `signal` ends the test as aborted */
9
9
  runTest(test: RegisteredTestCase, browser: SupportedBrowser, signal: AbortSignal, onEvent: (event: RunEvent) => void, options?: {
10
10
  outputDir?: string;
11
+ inputs?: Record<string, string>;
11
12
  }): Promise<void>;
12
13
  /** Ends the project's session */
13
14
  close(): Promise<void>;
@@ -54,7 +54,10 @@ export function handleMessages(channel, { open, exit }) {
54
54
  id,
55
55
  result: { ...event.result, worker: workerIndex },
56
56
  });
57
- }, item.outputDir !== undefined ? { outputDir: item.outputDir } : {});
57
+ }, {
58
+ ...(item.outputDir !== undefined && { outputDir: item.outputDir }),
59
+ ...(item.inputs !== undefined && { inputs: item.inputs }),
60
+ });
58
61
  }
59
62
  finally {
60
63
  current = undefined;
@@ -1,4 +1,4 @@
1
- import type { Step } from "./model.js";
1
+ import type { Step, TestInput } from "./model.js";
2
2
  export type StepEdit = {
3
3
  op: "replace";
4
4
  test: number;
@@ -32,10 +32,21 @@ export type StepEdit = {
32
32
  | {
33
33
  op: "remove-test";
34
34
  test: number;
35
+ }
36
+ /**
37
+ * Sets the test's `inputs` and `outputs` in its options object: a given list replaces that list, one left out
38
+ * stays. When both end up empty the options object goes, unless it holds a `reason`
39
+ */
40
+ | {
41
+ op: "declare";
42
+ test: number;
43
+ inputs?: TestInput[];
44
+ outputs?: string[];
35
45
  };
36
46
  /**
37
47
  * Applies one edit and returns the new source. Only the touched statement changes, plus the test's parameters
38
- * when the step reads a fixture (`env`, `secrets`) they don't have yet: everything else, formatting and
39
- * comments included, stays byte for byte.
48
+ * when the step reads a fixture (`env`, `secrets`, `inputs`, `outputs`) they don't have yet, and its `outputs`
49
+ * when the step saves one not declared yet: everything else, formatting and comments included, stays byte for
50
+ * byte. A step that reads an undeclared input is refused: `declare` it, with its default, first.
40
51
  */
41
52
  export declare function applyEdit(source: string, edit: StepEdit): string;