lawspec 0.20.1 → 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/CHANGELOG.md +361 -2
  2. package/README.md +3 -3
  3. package/beam-coverage.mjs +160 -0
  4. package/beam-doctor.mjs +306 -0
  5. package/bin/lawspec.mjs +306 -38
  6. package/build.json +492 -106
  7. package/compatibility.json +17 -0
  8. package/core.wasm +0 -0
  9. package/core_jsffi.js +2 -2
  10. package/doctor.mjs +7 -1
  11. package/examples/native-payments/elixir/lawspec.json +149 -0
  12. package/examples/native-payments/elixir/lib/payments_domain.ex +54 -0
  13. package/examples/native-payments/elixir/test/support/payment_generators.ex +9 -0
  14. package/examples/native-payments/erlang/include/payments_domain.hrl +2 -0
  15. package/examples/native-payments/erlang/lawspec.json +148 -0
  16. package/examples/native-payments/erlang/src/payments_domain.erl +21 -0
  17. package/examples/native-payments/erlang/test/payment_generators.erl +10 -0
  18. package/examples/native-payments/gleam/lawspec.json +149 -0
  19. package/examples/native-payments/gleam/prepare.mjs +48 -0
  20. package/examples/native-payments/gleam/src/payments_domain.gleam +32 -0
  21. package/examples/native-payments/gleam/test/payment_generators.gleam +13 -0
  22. package/examples/specs/abilities.lawspec +103 -0
  23. package/examples/specs/ability_imports.lawspec +107 -0
  24. package/examples/specs/builtins.lawspec +121 -0
  25. package/examples/specs/crypto.lawspec +51 -0
  26. package/examples/specs/distribution.lawspec +29 -0
  27. package/examples/specs/failures.lawspec +96 -0
  28. package/examples/specs/handler_bindings.lawspec +65 -0
  29. package/examples/specs/harness.lawspec +75 -0
  30. package/examples/specs/harness_testing.lawspec +45 -0
  31. package/examples/specs/matchers.lawspec +112 -0
  32. package/examples/specs/resources.lawspec +116 -0
  33. package/examples/specs/scheduling_order.lawspec +18 -0
  34. package/examples/specs/scheduling_overlap.lawspec +14 -0
  35. package/examples/specs/tables.lawspec +64 -0
  36. package/failure-database.mjs +46 -0
  37. package/index.d.ts +12 -1
  38. package/native-examples.mjs +7 -2
  39. package/package.json +1 -1
  40. package/templates.mjs +105 -16
  41. package/test-command.mjs +316 -35
package/test-command.mjs CHANGED
@@ -9,19 +9,23 @@
9
9
  // a law that calls adapters, every file in the target project that LawSpec
10
10
  // did not generate. A passing run records each law's key and seed; a law runs
11
11
  // again when its key changes or with --fresh.
12
- import { createHash } from "node:crypto";
12
+ import { createHash, randomUUID } from "node:crypto";
13
13
  import { readdir, readFile, stat } from "node:fs/promises";
14
14
  import path from "node:path";
15
15
 
16
16
  // Folders that hold dependencies, build output or caches, not code under test.
17
17
  const skipped = new Set([".git", ".lawspec", "node_modules", "dist", "build", "target", "out", ".gradle",
18
- ".stack-work", "__pycache__", ".hypothesis", ".pytest_cache", ".venv", "venv", ".idea", ".vscode"]);
18
+ ".stack-work", "_build", "deps", ".elixir_ls", "__pycache__", ".hypothesis", ".pytest_cache", ".venv", "venv", ".idea", ".vscode"]);
19
19
  // Build and lock files, which choose the toolchain and dependencies.
20
20
  const environmentFiles = ["package.json", "package-lock.json", "tsconfig.json", "pyproject.toml", "go.mod",
21
21
  "go.sum", "pom.xml", "build.gradle", "build.gradle.kts", "settings.gradle", "settings.gradle.kts",
22
- "Cargo.toml", "Cargo.lock", "stack.yaml", "stack.yaml.lock", "package.yaml"];
22
+ "Cargo.toml", "Cargo.lock", "stack.yaml", "stack.yaml.lock", "package.yaml",
23
+ "rebar.config", "rebar.config.script", "rebar.lock", "mix.exs", "mix.lock", "gleam.toml", "manifest.toml"];
23
24
 
24
25
  const digest = (value) => createHash("sha256").update(value).digest("hex");
26
+ // The crypto builder owns these outputs. Inputs and other priv/ assets still
27
+ // contribute to the adapter digest; rebuilding the NIF cannot invalidate a pass.
28
+ const cryptoBuildOutput = (file) => /^priv\/lawspec_crypto_native\.(?:so|dll)(?:\.build|\.\d+\.tmp)?$/.test(file);
25
29
 
26
30
  // Every file in a project that LawSpec did not generate, by content.
27
31
  export async function projectDigest(root, generated) {
@@ -33,7 +37,7 @@ export async function projectDigest(root, generated) {
33
37
  const relative = folder ? `${folder}/${entry.name}` : entry.name;
34
38
  if (entry.isDirectory()) {
35
39
  if (!skipped.has(entry.name) && !relative.endsWith("testdata/rapid")) await walk(relative);
36
- } else if (entry.isFile() && !generated.has(relative)) {
40
+ } else if (entry.isFile() && !generated.has(relative) && !cryptoBuildOutput(relative)) {
37
41
  hash.update(relative + "\0").update(await readFile(path.join(root, relative))).update("\0");
38
42
  }
39
43
  }
@@ -43,12 +47,34 @@ export async function projectDigest(root, generated) {
43
47
  }
44
48
 
45
49
  export async function environmentDigest(root, report) {
46
- const parts = [JSON.stringify(report ?? null)];
50
+ // Coverage locations describe build output; native tools can discover more
51
+ // output directories after the first run. Build configuration is hashed below.
52
+ const {coverage: _coverage, ...toolchain} = report ?? {};
53
+ const parts = [JSON.stringify(report ? toolchain : null)];
47
54
  const names = (await readdir(root)).filter((name) => environmentFiles.includes(name) || name.endsWith(".cabal")).sort();
48
55
  for (const name of names) parts.push(name, await readFile(path.join(root, name), "utf8"));
49
56
  return digest(JSON.stringify(parts));
50
57
  }
51
58
 
59
+ // Every recorded value under recorded/, by content: "" when there are none.
60
+ export async function recordedDigest(folder) {
61
+ const hash = createHash("sha256");
62
+ let any = false;
63
+ async function walk(relative) {
64
+ const entries = await readdir(path.join(folder, relative), { withFileTypes: true }).catch(() => []);
65
+ for (const entry of entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0))) {
66
+ const next = relative ? `${relative}/${entry.name}` : entry.name;
67
+ if (entry.isDirectory()) await walk(next);
68
+ else if (entry.isFile()) {
69
+ any = true;
70
+ hash.update(next + "\0").update(await readFile(path.join(folder, next))).update("\0");
71
+ }
72
+ }
73
+ }
74
+ await walk("");
75
+ return any ? hash.digest("hex") : "";
76
+ }
77
+
52
78
  // The result key of each law in a target's test manifest.
53
79
  export function lawKeys({ build, target, machineBits, minify, tests, files, environment, project }) {
54
80
  const contents = new Map(files.map((file) => [file.path, file.content]));
@@ -67,37 +93,154 @@ const groupBy = (entries, key) => {
67
93
  return [...groups.entries()];
68
94
  };
69
95
 
96
+ // Rebar and Mix run their configured compile hooks. Gleam has no corresponding
97
+ // hook, so build its generated crypto bridge before starting a native test VM.
98
+ export function preparationInvocations(target, files) {
99
+ return target.language === "gleam" && files.some((file) => file.path === "priv/lawspec_crypto_native.c")
100
+ ? [{command: "escript", args: ["lawspec_crypto_build.escript"]}]
101
+ : [];
102
+ }
103
+
104
+ // The native invocations that run the harness's benchmarks (lawspec test
105
+ // --benchmarks): each target's test command, selecting the tests that
106
+ // measure them by the names the manifest gives (entry.name). Benchmarks are
107
+ // measured, never asserted; their timings reach the summary through the
108
+ // harness statistics.
109
+ export function benchmarkInvocations(target, entries, { offline = false } = {}) {
110
+ if (!entries.length) return [];
111
+ const language = target.language;
112
+ const testDir = target.testDir;
113
+ const relativeTo = (file, defaultDir) => {
114
+ const directory = testDir || defaultDir;
115
+ return file.startsWith(directory + "/") ? file.slice(directory.length + 1) : file;
116
+ };
117
+ const className = (file, defaultDir, extension) => without(relativeTo(file, defaultDir), extension).replaceAll("/", ".");
118
+ if (language === "erlang")
119
+ return entries.map((entry) => ({ command: "rebar3", args: ["eunit", "--generator",
120
+ `${path.posix.basename(entry.file, ".erl")}:${entry.name}_cases`] }));
121
+ if (language === "elixir")
122
+ return groupBy(entries, (entry) => entry.file).map(([file, marks]) => ({ command: "mix",
123
+ args: ["test", file, ...marks.flatMap((entry) => ["--only", `lawspec:${entry.name}`])] }));
124
+ if (language === "gleam")
125
+ return [{ command: "gleam", args: ["test", "--target", "erlang"], env: {
126
+ LAWSPEC_BEAM_BENCHMARKS: JSON.stringify(entries.map((entry) => ({
127
+ module: path.posix.basename(entry.file, ".gleam"), function: `${entry.name}__case_0_test`
128
+ })))
129
+ } }];
130
+ if (language === "python")
131
+ return groupBy(entries, (e) => e.file).map(([file, marks]) => ({ command: target.python || "python3",
132
+ args: ["-m", "pytest", "-q", "-s", file, "-k", marks.map((e) => e.name).join(" or ")] }));
133
+ if (language === "javascript" || language === "typescript") {
134
+ const files = [...new Set(entries.map((e) => language === "typescript" ? `dist/${without(e.file, ".ts")}.js` : e.file))];
135
+ const run = { command: process.execPath,
136
+ args: ["--test", `--test-name-pattern=^(${entries.map((e) => regex(e.name)).join("|")})$`, ...files] };
137
+ return language === "typescript" ? [{ command: "npm", args: ["exec", "--", "tsc", "-p", "tsconfig.json"] }, run] : [run];
138
+ }
139
+ if (language === "go")
140
+ return groupBy(entries, (e) => path.posix.dirname(e.file)).map(([directory, marks]) => ({ command: "go",
141
+ args: ["test", "-count=1", "-v", `./${directory}`, "-run", `^(${marks.map((e) => regex(e.name)).join("|")})$`] }));
142
+ if (language === "java") {
143
+ const byClass = groupBy(entries, (e) => className(e.file, "src/test/java", ".java"));
144
+ return [{ command: target.maven || "mvn", args: [...(offline ? ["-o"] : []), "-B", "test",
145
+ `-Dtest=${byClass.map(([name, marks]) => `${name}#${marks.map((e) => e.name).join("+")}`).join(",")}`] }];
146
+ }
147
+ if (language === "kotlin")
148
+ return groupBy(entries, (e) => e.file).map(([file, marks]) => {
149
+ const filter = `(${marks.map((e) => e.name).join("|")})`;
150
+ return { command: target.gradle || "gradle",
151
+ args: [...(offline ? ["--offline"] : []), "--console=plain", "test", "--rerun", "--tests", className(file, "src/test/kotlin", ".kt")],
152
+ env: { "kotest.filter.tests": filter, kotest_filter_tests: filter } };
153
+ });
154
+ if (language === "rust")
155
+ return groupBy(entries, (e) => e.file).map(([file, marks]) => ({ command: "cargo",
156
+ args: ["test", ...(offline ? ["--offline"] : []), "--test", path.posix.basename(file, ".rs"), "--", "--exact", "--nocapture", ...marks.map((e) => e.name)] }));
157
+ if (language === "haskell") {
158
+ const module = (e) => without(relativeTo(e.file, "test"), "Spec.hs").replaceAll("/", ".");
159
+ // hspec matches by substring, and a benchmark's name has spaces, so each
160
+ // module's benchmarks are selected together.
161
+ return [{ command: "stack", args: ["--no-terminal", "test", "--test-arguments",
162
+ [...new Set(entries.map((e) => `--match ${module(e)}/benchmark`))].join(" ")] }];
163
+ }
164
+ throw new Error(`lawspec test does not support ${language}`);
165
+ }
166
+
70
167
  // The native invocations that run exactly the given laws' tests, each with
71
168
  // the laws it covers and how to tell, from the runner's own report, which of
72
- // them ran. Names are matched whole, so law1 never selects law10. `scratch`
73
- // is a folder for reports.
74
- export function invocations(target, entries, { offline = false, scratch = "." } = {}) {
169
+ // them ran. A law's tests are named after its label (entry.name, see
170
+ // LawSpec.TestNames), then a kind after a separator no name contains, so one
171
+ // law's name never selects another's. `scratch` is a folder for reports;
172
+ // `coverage` asks each runner to measure coverage too (see coverageSetup).
173
+ export function invocations(target, entries, { offline = false, scratch = ".", coverage = null, xdist = false,
174
+ root = process.cwd(), coverageConfig } = {}) {
175
+ if (!entries.length) return [];
75
176
  const language = target.language;
76
177
  const testDir = target.testDir;
77
178
  const relativeTo = (file, defaultDir) => {
78
179
  const directory = testDir || defaultDir;
79
180
  return file.startsWith(directory + "/") ? file.slice(directory.length + 1) : file;
80
181
  };
81
- const tests = (entry) => [`law${entry.index}Example*`, `law${entry.index}Boundary*`, `law${entry.index}Property*`];
82
- const kinds = (entry) => new RegExp(`^law${entry.index}(Example|Boundary|Property)`);
182
+ const named = (entry) => new RegExp(`^${regex(entry.name)}_`);
83
183
  const className = (file, defaultDir, extension) => without(relativeTo(file, defaultDir), extension).replaceAll("/", ".");
184
+ if (["erlang", "elixir", "gleam"].includes(language)) {
185
+ const run = randomUUID();
186
+ const file = path.join(scratch, `beam-${run}.jsonl`);
187
+ const identity = (entry) => `${entry.unit}::${entry.name}`;
188
+ const env = { LAWSPEC_BEAM_REPORT: file, LAWSPEC_BEAM_RUN: run };
189
+ const measured = coverage ? {run, file: path.resolve(coverage, "runs", `${run}.coverdata`),
190
+ ...(language === "elixir" ? {beamDirectory: coverageConfig?.compilePath} : {}),
191
+ ...(language === "erlang" ? {tool: "rebar3"} : {}),
192
+ ...(language === "gleam" ? {metadata: path.resolve(coverage, "runs", `${run}.json`)} : {})} : null;
193
+ let command, args;
194
+ if (language === "elixir") {
195
+ command = "mix";
196
+ args = ["test", ...new Set(entries.map((entry) => entry.file)),
197
+ ...entries.flatMap((entry) => ["--only", `lawspec_identity:${identity(entry)}`])];
198
+ if (measured) {
199
+ if (coverageConfig?.tool !== "Mix.Tasks.Test.Coverage" || !coverageConfig.output || !coverageConfig.compilePath)
200
+ throw new Error("BEAM coverage requires Mix.Tasks.Test.Coverage and its effective output directory from lawspec doctor");
201
+ args.push("--cover", "--export-coverage",
202
+ path.relative(path.resolve(root, coverageConfig.output), without(measured.file, ".coverdata")));
203
+ }
204
+ if (offline) env.HEX_OFFLINE = "1";
205
+ } else {
206
+ env.LAWSPEC_BEAM_LAWS = JSON.stringify(entries.map(({unit, name}) => ({unit, name})));
207
+ if (language === "erlang") {
208
+ command = "rebar3";
209
+ args = ["eunit", "--generator", "lawspec_generated_tests:lawspec_test_"];
210
+ if (measured) args.push("--cover", "--cover_export_name", without(measured.file, ".coverdata"));
211
+ } else {
212
+ command = "gleam";
213
+ args = ["test", "--target", "erlang"];
214
+ if (measured) env.LAWSPEC_BEAM_COVERAGE = JSON.stringify(measured);
215
+ }
216
+ }
217
+ return [{ laws: entries, command, args, env, ...(measured ? {coverage: measured} : {}), report: {kind: "beam-events", file, run},
218
+ ran: (test) => test.status === "passed" ? entries.filter((entry) => test.identity === identity(entry)) : [] }];
219
+ }
84
220
  if (language === "python")
85
221
  return groupBy(entries, (e) => e.file).map(([file, laws], n) => {
86
222
  const report = path.join(scratch, `pytest-${n}.xml`);
223
+ // parallel: pytest-xdist runs the tests on several workers, when it
224
+ // is installed (lawspec test checks); otherwise one after another.
225
+ const workers = xdist && laws.some((e) => e.parallel) ? ["-p", "xdist", "-n", "auto"] : [];
226
+ const pytest = ["-m", "pytest", "-q", ...workers, file, "-k", laws.map((e) => `${e.name}__`).join(" or "), `--junitxml=${report}`];
87
227
  return { laws, command: target.python || "python3",
88
- args: ["-m", "pytest", "-q", file, "-k", laws.map((e) => `test_law${e.index}_`).join(" or "), `--junitxml=${report}`],
228
+ args: coverage ? ["-m", "coverage", "run", "--append", `--data-file=${path.join(coverage, ".coverage")}`, ...pytest.slice(1)] : pytest,
89
229
  report: { kind: "junit", files: [report] },
90
- ran: (test) => laws.filter((e) => test.name.startsWith(`test_law${e.index}_`)) };
230
+ ran: (test) => laws.filter((e) => test.name.startsWith(`${e.name}__`)) };
91
231
  });
92
232
  if (language === "javascript" || language === "typescript") {
93
- const pattern = (e) => `^${regex(e.label)}( example: .*| property)?$`;
233
+ const pattern = (e) => `^${regex(e.label)}( example: .*| property| known failing| skipped| replay| search)?$`;
94
234
  const files = [...new Set(entries.map((e) => language === "typescript"
95
235
  ? `dist/${without(e.file, ".ts")}.js` : e.file))];
96
236
  const report = path.join(scratch, "node.xml");
97
- const run = { laws: entries, command: process.execPath,
98
- args: ["--test", `--test-name-pattern=(${entries.map(pattern).join("|")})`,
99
- "--test-reporter=spec", "--test-reporter-destination=stdout",
100
- "--test-reporter=junit", `--test-reporter-destination=${report}`, ...files],
237
+ // parallel: test files run at the same time too.
238
+ const node = ["--test", ...(entries.some((e) => e.parallel) ? ["--test-concurrency=" + Math.max(2, (globalThis.navigator?.hardwareConcurrency ?? 2))] : []),
239
+ `--test-name-pattern=(${entries.map(pattern).join("|")})`,
240
+ "--test-reporter=spec", "--test-reporter-destination=stdout",
241
+ "--test-reporter=junit", `--test-reporter-destination=${report}`, ...files];
242
+ const run = { laws: entries, command: coverage ? "npx" : process.execPath,
243
+ args: coverage ? ["--no-install", "c8", "--reporter=text", `--reports-dir=${coverage}`, process.execPath, ...node] : node,
101
244
  report: { kind: "junit", files: [report] },
102
245
  ran: (test) => entries.filter((e) => new RegExp(pattern(e)).test(test.name)) };
103
246
  return language === "typescript"
@@ -107,15 +250,18 @@ export function invocations(target, entries, { offline = false, scratch = "." }
107
250
  if (language === "go")
108
251
  return groupBy(entries, (e) => path.posix.dirname(e.file)).map(([directory, laws]) => ({
109
252
  laws, command: "go",
110
- args: ["test", "-json", "-count=1", `./${directory}`, "-run", `^TestLaw(${laws.map((e) => e.index).join("|")})(Example|Boundary|Property)`],
253
+ args: ["test", "-json", "-count=1", ...(coverage ? ["-cover", `-coverprofile=${path.join(coverage, `go-${directory.replaceAll("/", "_") || "root"}.out`)}`] : []),
254
+ `./${directory}`, "-run", `^(${laws.map((e) => regex(e.name)).join("|")})_`],
111
255
  report: { kind: "go-json" },
112
- ran: (test) => laws.filter((e) => new RegExp(`^TestLaw${e.index}(Example|Boundary|Property)`).test(test.name)) }));
256
+ ran: (test) => laws.filter((e) => named(e).test(test.name)) }));
113
257
  if (language === "java") {
114
258
  const byClass = groupBy(entries, (e) => className(e.file, "src/test/java", ".java"));
115
259
  return [{ laws: entries, command: target.maven || "mvn",
116
- args: [...(offline ? ["-o"] : []), "-B", "test", `-Dtest=${byClass.map(([name, laws]) => `${name}#${laws.flatMap(tests).join("+")}`).join(",")}`],
260
+ args: [...(offline ? ["-o"] : []), "-B", ...(coverage ? ["org.jacoco:jacoco-maven-plugin:prepare-agent"] : []), "test",
261
+ ...(coverage ? ["org.jacoco:jacoco-maven-plugin:report"] : []),
262
+ `-Dtest=${byClass.map(([name, laws]) => `${name}#${laws.map((e) => `${e.name}_*`).join("+")}`).join(",")}`],
117
263
  report: { kind: "junit", directory: "target/surefire-reports" },
118
- ran: (test) => entries.filter((e) => test.classname === className(e.file, "src/test/java", ".java") && kinds(e).test(test.name)) }];
264
+ ran: (test) => entries.filter((e) => test.classname === className(e.file, "src/test/java", ".java") && named(e).test(test.name)) }];
119
265
  }
120
266
  // Kotest names tests by string, which Gradle's method filters cannot
121
267
  // select. Kotest's own filter selects them within each class: one pattern,
@@ -123,35 +269,152 @@ export function invocations(target, entries, { offline = false, scratch = "." }
123
269
  // patterns must all match, so alternatives go in one group).
124
270
  if (language === "kotlin")
125
271
  return groupBy(entries, (e) => e.file).map(([file, laws]) => {
126
- const filter = `(${laws.flatMap((e) => ["Example", "Boundary", "Property"].map((kind) => `law${e.index}${kind}`)).join("|")})*`;
272
+ const filter = `(${laws.map((e) => `${e.name}_`).join("|")})*`;
127
273
  const name = className(file, "src/test/kotlin", ".kt");
128
274
  return { laws, command: target.gradle || "gradle",
129
- args: [...(offline ? ["--offline"] : []), "--console=plain", "test", "--rerun", "--tests", name],
275
+ args: [...(offline ? ["--offline"] : []), "--console=plain", "test", ...(coverage ? ["koverXmlReport"] : []), "--rerun", "--tests", name],
130
276
  env: { "kotest.filter.tests": filter, kotest_filter_tests: filter },
131
277
  report: { kind: "junit", directory: "build/test-results/test" },
132
- ran: (test) => laws.filter((e) => test.classname === name && kinds(e).test(test.name)) };
278
+ ran: (test) => laws.filter((e) => test.classname === name && named(e).test(test.name)) };
133
279
  });
134
280
  if (language === "rust")
135
281
  return groupBy(entries, (e) => e.file).map(([file, laws]) => ({
136
282
  laws, command: "cargo",
137
- args: ["test", ...(offline ? ["--offline"] : []), "--test", path.posix.basename(file, ".rs"), "--", "--exact",
138
- ...laws.map((e) => `test_${e.index}`)],
139
- report: { kind: "lines", pattern: /^test (test_\d+) \.\.\. (ok|FAILED)/ },
140
- ran: (test) => laws.filter((e) => test.name === `test_${e.index}`) }));
283
+ args: [...(coverage ? ["llvm-cov", "--no-report"] : []), "test", ...(offline ? ["--offline"] : []), "--test", path.posix.basename(file, ".rs"), "--", "--exact",
284
+ ...laws.map((e) => e.name)],
285
+ report: { kind: "lines", pattern: /^test (\S+) \.\.\. (ok|FAILED|ignored)/ },
286
+ ran: (test) => laws.filter((e) => test.name === e.name) }));
141
287
  if (language === "haskell") {
142
288
  const module = (e) => without(relativeTo(e.file, "test"), "Spec.hs").replaceAll("/", ".");
143
- const matches = entries.flatMap((e) => ["Example", "Boundary", "Property"].map((kind) => `${module(e)}/law${e.index}${kind}`));
289
+ const matches = entries.map((e) => `${module(e)}/${e.name}_`);
144
290
  return [{ laws: entries, command: "stack",
145
- args: ["--no-terminal", "test", "--test-arguments", matches.map((m) => `--match ${m}`).join(" ")],
291
+ args: ["--no-terminal", "test", ...(coverage ? ["--coverage"] : []), "--test-arguments", matches.map((m) => `--match ${m}`).join(" ")],
146
292
  report: { kind: "hspec" },
147
- ran: (test) => entries.filter((e) => test.classname === module(e) && kinds(e).test(test.name)) }];
293
+ ran: (test) => entries.filter((e) => test.classname === module(e) && named(e).test(test.name)) }];
148
294
  }
149
295
  throw new Error(`lawspec test does not support ${language}`);
150
296
  }
151
297
 
298
+ // What --coverage needs on each target, and how to say it is missing.
299
+ export const coverageTools = {
300
+ erlang: { tool: "OTP cover", install: "install Erlang/OTP with the tools application" },
301
+ elixir: { tool: "OTP cover", install: "install Erlang/OTP with the tools application" },
302
+ gleam: { tool: "OTP cover", install: "install Erlang/OTP with the tools application" },
303
+ python: { tool: "coverage.py", check: ["-c", "import coverage"], install: "python -m pip install coverage" },
304
+ javascript: { tool: "c8", install: "npm install --save-dev c8" },
305
+ typescript: { tool: "c8", install: "npm install --save-dev c8" },
306
+ go: { tool: "go test -cover", install: "(built in)" },
307
+ java: { tool: "JaCoCo", install: "(resolved by Maven as org.jacoco:jacoco-maven-plugin)" },
308
+ kotlin: { tool: "Kover", install: "add id(\"org.jetbrains.kotlinx.kover\") to the plugins of build.gradle.kts" },
309
+ rust: { tool: "cargo-llvm-cov", install: "cargo install cargo-llvm-cov" },
310
+ haskell: { tool: "hpc", install: "(built into GHC; stack test --coverage)" },
311
+ };
312
+
313
+ // Laws selected by tag: every --tag, and none of --exclude-tag.
314
+ export function selectByTags(entries, include = [], exclude = []) {
315
+ return entries.filter((e) => (!include.length || include.some((t) => (e.tags ?? []).includes(t))) &&
316
+ !exclude.some((t) => (e.tags ?? []).includes(t)));
317
+ }
318
+
319
+ // A recorded failure takes precedence over any older passing cache entry,
320
+ // including when an adapter is repaired by restoring its previous contents.
321
+ export function testsToRun(entries, keys, previous, failures, {fresh = false, coverage = false, updateRecorded = false} = {}) {
322
+ return entries.filter((entry) => fresh || coverage || updateRecorded || failures[entry.law] ||
323
+ previous[entry.law]?.key !== keys.get(entry.law));
324
+ }
325
+
326
+ // Read complete outer elements, preserving nested suites and native payloads.
327
+ // Comments and CDATA can contain apparent XML tags; they are never elements.
328
+ function junitElements(xml, names) {
329
+ const elements = [];
330
+ let current;
331
+ const tags = /<!--[\s\S]*?-->|<!\[CDATA\[[\s\S]*?\]\]>|<\?[\s\S]*?\?>|<(\/?)([A-Za-z_][\w:.-]*)(?:[^"'<>]|"[^"]*"|'[^']*')*>/g;
332
+ for (const tag of xml.matchAll(tags)) {
333
+ const [, closing, name] = tag;
334
+ if (!name || !names.includes(name)) continue;
335
+ const empty = /\/\s*>$/.test(tag[0]);
336
+ if (!current) {
337
+ if (closing) continue;
338
+ current = {name, start: tag.index, opening: tag[0], depth: 0};
339
+ }
340
+ if (name !== current.name) continue;
341
+ current.depth += closing ? -1 : empty ? 0 : 1;
342
+ if (current.depth === 0) {
343
+ elements.push({...current, xml: xml.slice(current.start, tag.index + tag[0].length)});
344
+ current = undefined;
345
+ }
346
+ }
347
+ return elements;
348
+ }
349
+
350
+ function junitCounts(cases) {
351
+ const statuses = cases.map(test => new Set(junitElements(test.xml, ["failure", "error", "skipped"]).map(e => e.name)));
352
+ return {tests: cases.length, ...Object.fromEntries(["failure", "error", "skipped"].map(name =>
353
+ [name === "skipped" ? name : `${name}s`, statuses.filter(status => status.has(name)).length]))};
354
+ }
355
+
356
+ // One JUnit report from every target's: each target's suites, renamed with
357
+ // the target, in one <testsuites>.
358
+ export function mergeJunit(reports) {
359
+ const suites = [];
360
+ const totals = {tests: 0, failures: 0, errors: 0, skipped: 0};
361
+ for (const { target, xml } of reports) {
362
+ const elements = junitElements(xml, ["testsuite", "testcase"]);
363
+ const found = elements.filter(e => e.name === "testsuite");
364
+ // Node's native JUnit reporter puts top-level tests directly in testsuites.
365
+ // Keep those cases (and their native failure payloads) instead of silently
366
+ // dropping them because they have no intervening testsuite element.
367
+ const loose = elements.filter(e => e.name === "testcase");
368
+ if (loose.length) {
369
+ const opening = `<testsuite name="native" ${Object.entries(junitCounts(loose)).map(([key, value]) => `${key}="${value}"`).join(" ")}>`;
370
+ found.push({opening, xml: opening + "\n" + loose.map(e => e.xml).join("\n") + "\n</testsuite>"});
371
+ }
372
+ for (const suite of found) {
373
+ const attributes = [...suite.opening.matchAll(/\s([A-Za-z_:][\w:.-]*)\s*=\s*(?:"([^"]*)"|'([^']*)')/g)];
374
+ const values = Object.fromEntries(attributes.map(match => [match[1], match[2] ?? match[3]]));
375
+ const counts = junitCounts(junitElements(suite.xml, ["testcase"]));
376
+ for (const key of Object.keys(totals))
377
+ totals[key] += /^\d+$/.test(values[key] ?? "") ? Number(values[key]) : counts[key];
378
+ const name = attributes.find(match => match[1] === "name");
379
+ const opening = name
380
+ ? suite.opening.slice(0, name.index) + ` name="${escapeXml(target)}: ${values.name.replaceAll('"', '&quot;')}"` + suite.opening.slice(name.index + name[0].length)
381
+ : suite.opening.replace(/^<testsuite\b/, `<testsuite name="${escapeXml(target)}"`);
382
+ suites.push(opening + suite.xml.slice(suite.opening.length));
383
+ }
384
+ }
385
+ return `<?xml version="1.0" encoding="UTF-8"?>\n<testsuites tests="${totals.tests}" failures="${totals.failures}" errors="${totals.errors}" skipped="${totals.skipped}">\n` +
386
+ suites.join("\n") + "\n</testsuites>\n";
387
+ }
388
+
389
+ const escapeXml = (text) => text.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;").replaceAll('"', "&quot;");
390
+
391
+ // The harness statistics a run wrote (LAWSPEC_STATS): cover results, label
392
+ // counts, flaky and known-failing outcomes, benchmarks.
393
+ export async function harnessStatistics(directory) {
394
+ const names = await readdir(directory).catch(() => []);
395
+ const entries = [];
396
+ for (const name of names.filter((n) => n.endsWith(".json")).sort())
397
+ try { entries.push(JSON.parse(await readFile(path.join(directory, name), "utf8"))); } catch { /* partial */ }
398
+ return entries;
399
+ }
400
+
152
401
  // The tests a run executed, as {name, classname}, from the runner's report or
153
402
  // output. Skipped tests are not counted.
154
403
  export async function executedTests(report, output, root, since) {
404
+ if (report?.kind === "beam-events") {
405
+ // A native listener seals the invocation after its final event. Missing,
406
+ // stale, malformed or interrupted reports are not execution evidence.
407
+ const raw = await readFile(path.resolve(root, report.file), "utf8").catch(() => "");
408
+ let rows;
409
+ try { rows = raw.trim().split("\n").map((line) => JSON.parse(line)); } catch { return []; }
410
+ if (rows.length < 2 || rows[0]?.event !== "start" || rows.at(-1)?.event !== "end" ||
411
+ rows.some((row) => row?.run !== report.run)) return [];
412
+ const tests = rows.slice(1, -1);
413
+ if (tests.some((row) => row.event !== "test" || !["passed", "failed"].includes(row.status) ||
414
+ typeof row.name !== "string" || typeof row.classname !== "string" || typeof row.identity !== "string" ||
415
+ !Number.isFinite(row.time) || row.time < 0)) return [];
416
+ return tests;
417
+ }
155
418
  if (report?.kind === "junit") {
156
419
  const files = report.files ?? await reportFiles(path.join(root, report.directory), since);
157
420
  const tests = [];
@@ -169,13 +432,14 @@ export async function executedTests(report, output, root, since) {
169
432
  return output.split("\n").flatMap((line) => {
170
433
  try {
171
434
  const event = JSON.parse(line);
172
- return ["pass", "fail"].includes(event.Action) && event.Test ? [{ name: event.Test, classname: event.Package }] : [];
435
+ return ["pass", "fail"].includes(event.Action) && event.Test
436
+ ? [{ name: event.Test, classname: event.Package, status: event.Action === "pass" ? "passed" : "failed" }] : [];
173
437
  } catch { return []; }
174
438
  });
175
439
  if (report?.kind === "lines")
176
440
  return output.split("\n").flatMap((line) => {
177
441
  const match = line.match(report.pattern);
178
- return match ? [{ name: match[1], classname: "" }] : [];
442
+ return match && match[2] !== "ignored" ? [{ name: match[1], classname: "", status: match[2] === "ok" ? "passed" : "failed" }] : [];
179
443
  });
180
444
  if (report?.kind === "hspec") {
181
445
  // hspec prints each module, then its examples indented beneath it.
@@ -183,14 +447,31 @@ export async function executedTests(report, output, root, since) {
183
447
  let module = "";
184
448
  for (const line of output.split("\n")) {
185
449
  if (/^\S/.test(line) && !/^(Finished|Failures|Randomized|\d+ examples?)/.test(line)) module = line.trim();
186
- const example = line.match(/^\s+(law\d+\S*?):?(?:\s.*)?\s\[[✔✘]\]\s*$/);
187
- if (example) tests.push({ name: example[1], classname: module });
450
+ const example = line.match(/^\s+(law\S*?):?(?:\s.*)?\s\[([✔✘])\]\s*$/);
451
+ if (example) tests.push({ name: example[1], classname: module, status: example[2] === "✔" ? "passed" : "failed" });
188
452
  }
189
453
  return tests;
190
454
  }
191
455
  return [];
192
456
  }
193
457
 
458
+ // JUnit XML for a runner that writes none, from the tests it printed.
459
+ export function junitFromTests(tests) {
460
+ const failures = tests.filter((t) => t.status === "failed").length;
461
+ const skipped = tests.filter((t) => t.status === "skipped").length;
462
+ // A BEAM scheduler wraps native functions from several units. Its module
463
+ // is not the declaring class: retain the unit from the validated identity.
464
+ const classname = (test) => {
465
+ const separator = typeof test.identity === "string" ? test.identity.lastIndexOf("::") : -1;
466
+ return separator > 0 ? test.identity.slice(0, separator) : test.classname ?? "";
467
+ };
468
+ return `<testsuite name="lawspec" tests="${tests.length}" failures="${failures}"${skipped ? ` skipped="${skipped}"` : ""}>\n` +
469
+ tests.map((t) => ` <testcase classname="${escapeXml(classname(t))}" name="${escapeXml(t.name)}"` +
470
+ (Number.isFinite(t.time) ? ` time="${t.time}"` : "") +
471
+ (t.status === "failed" ? `><failure message="${escapeXml(t.failure ?? "failed")}"/></testcase>` :
472
+ t.status === "skipped" ? `><skipped message="${escapeXml(t.reason ?? "skipped")}"/></testcase>` : "/>")).join("\n") + "\n</testsuite>";
473
+ }
474
+
194
475
  async function reportFiles(directory, since) {
195
476
  const names = await readdir(directory).catch(() => []);
196
477
  const files = [];