lawspec 0.20.1 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +361 -2
- package/README.md +3 -3
- package/beam-coverage.mjs +160 -0
- package/beam-doctor.mjs +306 -0
- package/bin/lawspec.mjs +306 -38
- package/build.json +492 -106
- package/compatibility.json +17 -0
- package/core.wasm +0 -0
- package/core_jsffi.js +2 -2
- package/doctor.mjs +7 -1
- package/examples/native-payments/elixir/lawspec.json +149 -0
- package/examples/native-payments/elixir/lib/payments_domain.ex +54 -0
- package/examples/native-payments/elixir/test/support/payment_generators.ex +9 -0
- package/examples/native-payments/erlang/include/payments_domain.hrl +2 -0
- package/examples/native-payments/erlang/lawspec.json +148 -0
- package/examples/native-payments/erlang/src/payments_domain.erl +21 -0
- package/examples/native-payments/erlang/test/payment_generators.erl +10 -0
- package/examples/native-payments/gleam/lawspec.json +149 -0
- package/examples/native-payments/gleam/prepare.mjs +48 -0
- package/examples/native-payments/gleam/src/payments_domain.gleam +32 -0
- package/examples/native-payments/gleam/test/payment_generators.gleam +13 -0
- package/examples/specs/abilities.lawspec +103 -0
- package/examples/specs/ability_imports.lawspec +107 -0
- package/examples/specs/builtins.lawspec +121 -0
- package/examples/specs/crypto.lawspec +51 -0
- package/examples/specs/distribution.lawspec +29 -0
- package/examples/specs/failures.lawspec +96 -0
- package/examples/specs/handler_bindings.lawspec +65 -0
- package/examples/specs/harness.lawspec +75 -0
- package/examples/specs/harness_testing.lawspec +45 -0
- package/examples/specs/matchers.lawspec +112 -0
- package/examples/specs/resources.lawspec +116 -0
- package/examples/specs/scheduling_order.lawspec +18 -0
- package/examples/specs/scheduling_overlap.lawspec +14 -0
- package/examples/specs/tables.lawspec +64 -0
- package/failure-database.mjs +46 -0
- package/index.d.ts +12 -1
- package/native-examples.mjs +7 -2
- package/package.json +1 -1
- package/templates.mjs +105 -16
- package/test-command.mjs +316 -35
package/test-command.mjs
CHANGED
|
@@ -9,19 +9,23 @@
|
|
|
9
9
|
// a law that calls adapters, every file in the target project that LawSpec
|
|
10
10
|
// did not generate. A passing run records each law's key and seed; a law runs
|
|
11
11
|
// again when its key changes or with --fresh.
|
|
12
|
-
import { createHash } from "node:crypto";
|
|
12
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
13
13
|
import { readdir, readFile, stat } from "node:fs/promises";
|
|
14
14
|
import path from "node:path";
|
|
15
15
|
|
|
16
16
|
// Folders that hold dependencies, build output or caches, not code under test.
|
|
17
17
|
const skipped = new Set([".git", ".lawspec", "node_modules", "dist", "build", "target", "out", ".gradle",
|
|
18
|
-
".stack-work", "__pycache__", ".hypothesis", ".pytest_cache", ".venv", "venv", ".idea", ".vscode"]);
|
|
18
|
+
".stack-work", "_build", "deps", ".elixir_ls", "__pycache__", ".hypothesis", ".pytest_cache", ".venv", "venv", ".idea", ".vscode"]);
|
|
19
19
|
// Build and lock files, which choose the toolchain and dependencies.
|
|
20
20
|
const environmentFiles = ["package.json", "package-lock.json", "tsconfig.json", "pyproject.toml", "go.mod",
|
|
21
21
|
"go.sum", "pom.xml", "build.gradle", "build.gradle.kts", "settings.gradle", "settings.gradle.kts",
|
|
22
|
-
"Cargo.toml", "Cargo.lock", "stack.yaml", "stack.yaml.lock", "package.yaml"
|
|
22
|
+
"Cargo.toml", "Cargo.lock", "stack.yaml", "stack.yaml.lock", "package.yaml",
|
|
23
|
+
"rebar.config", "rebar.config.script", "rebar.lock", "mix.exs", "mix.lock", "gleam.toml", "manifest.toml"];
|
|
23
24
|
|
|
24
25
|
const digest = (value) => createHash("sha256").update(value).digest("hex");
|
|
26
|
+
// The crypto builder owns these outputs. Inputs and other priv/ assets still
|
|
27
|
+
// contribute to the adapter digest; rebuilding the NIF cannot invalidate a pass.
|
|
28
|
+
const cryptoBuildOutput = (file) => /^priv\/lawspec_crypto_native\.(?:so|dll)(?:\.build|\.\d+\.tmp)?$/.test(file);
|
|
25
29
|
|
|
26
30
|
// Every file in a project that LawSpec did not generate, by content.
|
|
27
31
|
export async function projectDigest(root, generated) {
|
|
@@ -33,7 +37,7 @@ export async function projectDigest(root, generated) {
|
|
|
33
37
|
const relative = folder ? `${folder}/${entry.name}` : entry.name;
|
|
34
38
|
if (entry.isDirectory()) {
|
|
35
39
|
if (!skipped.has(entry.name) && !relative.endsWith("testdata/rapid")) await walk(relative);
|
|
36
|
-
} else if (entry.isFile() && !generated.has(relative)) {
|
|
40
|
+
} else if (entry.isFile() && !generated.has(relative) && !cryptoBuildOutput(relative)) {
|
|
37
41
|
hash.update(relative + "\0").update(await readFile(path.join(root, relative))).update("\0");
|
|
38
42
|
}
|
|
39
43
|
}
|
|
@@ -43,12 +47,34 @@ export async function projectDigest(root, generated) {
|
|
|
43
47
|
}
|
|
44
48
|
|
|
45
49
|
export async function environmentDigest(root, report) {
|
|
46
|
-
|
|
50
|
+
// Coverage locations describe build output; native tools can discover more
|
|
51
|
+
// output directories after the first run. Build configuration is hashed below.
|
|
52
|
+
const {coverage: _coverage, ...toolchain} = report ?? {};
|
|
53
|
+
const parts = [JSON.stringify(report ? toolchain : null)];
|
|
47
54
|
const names = (await readdir(root)).filter((name) => environmentFiles.includes(name) || name.endsWith(".cabal")).sort();
|
|
48
55
|
for (const name of names) parts.push(name, await readFile(path.join(root, name), "utf8"));
|
|
49
56
|
return digest(JSON.stringify(parts));
|
|
50
57
|
}
|
|
51
58
|
|
|
59
|
+
// Every recorded value under recorded/, by content: "" when there are none.
|
|
60
|
+
export async function recordedDigest(folder) {
|
|
61
|
+
const hash = createHash("sha256");
|
|
62
|
+
let any = false;
|
|
63
|
+
async function walk(relative) {
|
|
64
|
+
const entries = await readdir(path.join(folder, relative), { withFileTypes: true }).catch(() => []);
|
|
65
|
+
for (const entry of entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0))) {
|
|
66
|
+
const next = relative ? `${relative}/${entry.name}` : entry.name;
|
|
67
|
+
if (entry.isDirectory()) await walk(next);
|
|
68
|
+
else if (entry.isFile()) {
|
|
69
|
+
any = true;
|
|
70
|
+
hash.update(next + "\0").update(await readFile(path.join(folder, next))).update("\0");
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
await walk("");
|
|
75
|
+
return any ? hash.digest("hex") : "";
|
|
76
|
+
}
|
|
77
|
+
|
|
52
78
|
// The result key of each law in a target's test manifest.
|
|
53
79
|
export function lawKeys({ build, target, machineBits, minify, tests, files, environment, project }) {
|
|
54
80
|
const contents = new Map(files.map((file) => [file.path, file.content]));
|
|
@@ -67,37 +93,154 @@ const groupBy = (entries, key) => {
|
|
|
67
93
|
return [...groups.entries()];
|
|
68
94
|
};
|
|
69
95
|
|
|
96
|
+
// Rebar and Mix run their configured compile hooks. Gleam has no corresponding
|
|
97
|
+
// hook, so build its generated crypto bridge before starting a native test VM.
|
|
98
|
+
export function preparationInvocations(target, files) {
|
|
99
|
+
return target.language === "gleam" && files.some((file) => file.path === "priv/lawspec_crypto_native.c")
|
|
100
|
+
? [{command: "escript", args: ["lawspec_crypto_build.escript"]}]
|
|
101
|
+
: [];
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// The native invocations that run the harness's benchmarks (lawspec test
|
|
105
|
+
// --benchmarks): each target's test command, selecting the tests that
|
|
106
|
+
// measure them by the names the manifest gives (entry.name). Benchmarks are
|
|
107
|
+
// measured, never asserted; their timings reach the summary through the
|
|
108
|
+
// harness statistics.
|
|
109
|
+
export function benchmarkInvocations(target, entries, { offline = false } = {}) {
|
|
110
|
+
if (!entries.length) return [];
|
|
111
|
+
const language = target.language;
|
|
112
|
+
const testDir = target.testDir;
|
|
113
|
+
const relativeTo = (file, defaultDir) => {
|
|
114
|
+
const directory = testDir || defaultDir;
|
|
115
|
+
return file.startsWith(directory + "/") ? file.slice(directory.length + 1) : file;
|
|
116
|
+
};
|
|
117
|
+
const className = (file, defaultDir, extension) => without(relativeTo(file, defaultDir), extension).replaceAll("/", ".");
|
|
118
|
+
if (language === "erlang")
|
|
119
|
+
return entries.map((entry) => ({ command: "rebar3", args: ["eunit", "--generator",
|
|
120
|
+
`${path.posix.basename(entry.file, ".erl")}:${entry.name}_cases`] }));
|
|
121
|
+
if (language === "elixir")
|
|
122
|
+
return groupBy(entries, (entry) => entry.file).map(([file, marks]) => ({ command: "mix",
|
|
123
|
+
args: ["test", file, ...marks.flatMap((entry) => ["--only", `lawspec:${entry.name}`])] }));
|
|
124
|
+
if (language === "gleam")
|
|
125
|
+
return [{ command: "gleam", args: ["test", "--target", "erlang"], env: {
|
|
126
|
+
LAWSPEC_BEAM_BENCHMARKS: JSON.stringify(entries.map((entry) => ({
|
|
127
|
+
module: path.posix.basename(entry.file, ".gleam"), function: `${entry.name}__case_0_test`
|
|
128
|
+
})))
|
|
129
|
+
} }];
|
|
130
|
+
if (language === "python")
|
|
131
|
+
return groupBy(entries, (e) => e.file).map(([file, marks]) => ({ command: target.python || "python3",
|
|
132
|
+
args: ["-m", "pytest", "-q", "-s", file, "-k", marks.map((e) => e.name).join(" or ")] }));
|
|
133
|
+
if (language === "javascript" || language === "typescript") {
|
|
134
|
+
const files = [...new Set(entries.map((e) => language === "typescript" ? `dist/${without(e.file, ".ts")}.js` : e.file))];
|
|
135
|
+
const run = { command: process.execPath,
|
|
136
|
+
args: ["--test", `--test-name-pattern=^(${entries.map((e) => regex(e.name)).join("|")})$`, ...files] };
|
|
137
|
+
return language === "typescript" ? [{ command: "npm", args: ["exec", "--", "tsc", "-p", "tsconfig.json"] }, run] : [run];
|
|
138
|
+
}
|
|
139
|
+
if (language === "go")
|
|
140
|
+
return groupBy(entries, (e) => path.posix.dirname(e.file)).map(([directory, marks]) => ({ command: "go",
|
|
141
|
+
args: ["test", "-count=1", "-v", `./${directory}`, "-run", `^(${marks.map((e) => regex(e.name)).join("|")})$`] }));
|
|
142
|
+
if (language === "java") {
|
|
143
|
+
const byClass = groupBy(entries, (e) => className(e.file, "src/test/java", ".java"));
|
|
144
|
+
return [{ command: target.maven || "mvn", args: [...(offline ? ["-o"] : []), "-B", "test",
|
|
145
|
+
`-Dtest=${byClass.map(([name, marks]) => `${name}#${marks.map((e) => e.name).join("+")}`).join(",")}`] }];
|
|
146
|
+
}
|
|
147
|
+
if (language === "kotlin")
|
|
148
|
+
return groupBy(entries, (e) => e.file).map(([file, marks]) => {
|
|
149
|
+
const filter = `(${marks.map((e) => e.name).join("|")})`;
|
|
150
|
+
return { command: target.gradle || "gradle",
|
|
151
|
+
args: [...(offline ? ["--offline"] : []), "--console=plain", "test", "--rerun", "--tests", className(file, "src/test/kotlin", ".kt")],
|
|
152
|
+
env: { "kotest.filter.tests": filter, kotest_filter_tests: filter } };
|
|
153
|
+
});
|
|
154
|
+
if (language === "rust")
|
|
155
|
+
return groupBy(entries, (e) => e.file).map(([file, marks]) => ({ command: "cargo",
|
|
156
|
+
args: ["test", ...(offline ? ["--offline"] : []), "--test", path.posix.basename(file, ".rs"), "--", "--exact", "--nocapture", ...marks.map((e) => e.name)] }));
|
|
157
|
+
if (language === "haskell") {
|
|
158
|
+
const module = (e) => without(relativeTo(e.file, "test"), "Spec.hs").replaceAll("/", ".");
|
|
159
|
+
// hspec matches by substring, and a benchmark's name has spaces, so each
|
|
160
|
+
// module's benchmarks are selected together.
|
|
161
|
+
return [{ command: "stack", args: ["--no-terminal", "test", "--test-arguments",
|
|
162
|
+
[...new Set(entries.map((e) => `--match ${module(e)}/benchmark`))].join(" ")] }];
|
|
163
|
+
}
|
|
164
|
+
throw new Error(`lawspec test does not support ${language}`);
|
|
165
|
+
}
|
|
166
|
+
|
|
70
167
|
// The native invocations that run exactly the given laws' tests, each with
|
|
71
168
|
// the laws it covers and how to tell, from the runner's own report, which of
|
|
72
|
-
// them ran.
|
|
73
|
-
//
|
|
74
|
-
|
|
169
|
+
// them ran. A law's tests are named after its label (entry.name, see
|
|
170
|
+
// LawSpec.TestNames), then a kind after a separator no name contains, so one
|
|
171
|
+
// law's name never selects another's. `scratch` is a folder for reports;
|
|
172
|
+
// `coverage` asks each runner to measure coverage too (see coverageSetup).
|
|
173
|
+
export function invocations(target, entries, { offline = false, scratch = ".", coverage = null, xdist = false,
|
|
174
|
+
root = process.cwd(), coverageConfig } = {}) {
|
|
175
|
+
if (!entries.length) return [];
|
|
75
176
|
const language = target.language;
|
|
76
177
|
const testDir = target.testDir;
|
|
77
178
|
const relativeTo = (file, defaultDir) => {
|
|
78
179
|
const directory = testDir || defaultDir;
|
|
79
180
|
return file.startsWith(directory + "/") ? file.slice(directory.length + 1) : file;
|
|
80
181
|
};
|
|
81
|
-
const
|
|
82
|
-
const kinds = (entry) => new RegExp(`^law${entry.index}(Example|Boundary|Property)`);
|
|
182
|
+
const named = (entry) => new RegExp(`^${regex(entry.name)}_`);
|
|
83
183
|
const className = (file, defaultDir, extension) => without(relativeTo(file, defaultDir), extension).replaceAll("/", ".");
|
|
184
|
+
if (["erlang", "elixir", "gleam"].includes(language)) {
|
|
185
|
+
const run = randomUUID();
|
|
186
|
+
const file = path.join(scratch, `beam-${run}.jsonl`);
|
|
187
|
+
const identity = (entry) => `${entry.unit}::${entry.name}`;
|
|
188
|
+
const env = { LAWSPEC_BEAM_REPORT: file, LAWSPEC_BEAM_RUN: run };
|
|
189
|
+
const measured = coverage ? {run, file: path.resolve(coverage, "runs", `${run}.coverdata`),
|
|
190
|
+
...(language === "elixir" ? {beamDirectory: coverageConfig?.compilePath} : {}),
|
|
191
|
+
...(language === "erlang" ? {tool: "rebar3"} : {}),
|
|
192
|
+
...(language === "gleam" ? {metadata: path.resolve(coverage, "runs", `${run}.json`)} : {})} : null;
|
|
193
|
+
let command, args;
|
|
194
|
+
if (language === "elixir") {
|
|
195
|
+
command = "mix";
|
|
196
|
+
args = ["test", ...new Set(entries.map((entry) => entry.file)),
|
|
197
|
+
...entries.flatMap((entry) => ["--only", `lawspec_identity:${identity(entry)}`])];
|
|
198
|
+
if (measured) {
|
|
199
|
+
if (coverageConfig?.tool !== "Mix.Tasks.Test.Coverage" || !coverageConfig.output || !coverageConfig.compilePath)
|
|
200
|
+
throw new Error("BEAM coverage requires Mix.Tasks.Test.Coverage and its effective output directory from lawspec doctor");
|
|
201
|
+
args.push("--cover", "--export-coverage",
|
|
202
|
+
path.relative(path.resolve(root, coverageConfig.output), without(measured.file, ".coverdata")));
|
|
203
|
+
}
|
|
204
|
+
if (offline) env.HEX_OFFLINE = "1";
|
|
205
|
+
} else {
|
|
206
|
+
env.LAWSPEC_BEAM_LAWS = JSON.stringify(entries.map(({unit, name}) => ({unit, name})));
|
|
207
|
+
if (language === "erlang") {
|
|
208
|
+
command = "rebar3";
|
|
209
|
+
args = ["eunit", "--generator", "lawspec_generated_tests:lawspec_test_"];
|
|
210
|
+
if (measured) args.push("--cover", "--cover_export_name", without(measured.file, ".coverdata"));
|
|
211
|
+
} else {
|
|
212
|
+
command = "gleam";
|
|
213
|
+
args = ["test", "--target", "erlang"];
|
|
214
|
+
if (measured) env.LAWSPEC_BEAM_COVERAGE = JSON.stringify(measured);
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
return [{ laws: entries, command, args, env, ...(measured ? {coverage: measured} : {}), report: {kind: "beam-events", file, run},
|
|
218
|
+
ran: (test) => test.status === "passed" ? entries.filter((entry) => test.identity === identity(entry)) : [] }];
|
|
219
|
+
}
|
|
84
220
|
if (language === "python")
|
|
85
221
|
return groupBy(entries, (e) => e.file).map(([file, laws], n) => {
|
|
86
222
|
const report = path.join(scratch, `pytest-${n}.xml`);
|
|
223
|
+
// parallel: pytest-xdist runs the tests on several workers, when it
|
|
224
|
+
// is installed (lawspec test checks); otherwise one after another.
|
|
225
|
+
const workers = xdist && laws.some((e) => e.parallel) ? ["-p", "xdist", "-n", "auto"] : [];
|
|
226
|
+
const pytest = ["-m", "pytest", "-q", ...workers, file, "-k", laws.map((e) => `${e.name}__`).join(" or "), `--junitxml=${report}`];
|
|
87
227
|
return { laws, command: target.python || "python3",
|
|
88
|
-
args: ["-m", "
|
|
228
|
+
args: coverage ? ["-m", "coverage", "run", "--append", `--data-file=${path.join(coverage, ".coverage")}`, ...pytest.slice(1)] : pytest,
|
|
89
229
|
report: { kind: "junit", files: [report] },
|
|
90
|
-
ran: (test) => laws.filter((e) => test.name.startsWith(
|
|
230
|
+
ran: (test) => laws.filter((e) => test.name.startsWith(`${e.name}__`)) };
|
|
91
231
|
});
|
|
92
232
|
if (language === "javascript" || language === "typescript") {
|
|
93
|
-
const pattern = (e) => `^${regex(e.label)}( example: .*| property)?$`;
|
|
233
|
+
const pattern = (e) => `^${regex(e.label)}( example: .*| property| known failing| skipped| replay| search)?$`;
|
|
94
234
|
const files = [...new Set(entries.map((e) => language === "typescript"
|
|
95
235
|
? `dist/${without(e.file, ".ts")}.js` : e.file))];
|
|
96
236
|
const report = path.join(scratch, "node.xml");
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
237
|
+
// parallel: test files run at the same time too.
|
|
238
|
+
const node = ["--test", ...(entries.some((e) => e.parallel) ? ["--test-concurrency=" + Math.max(2, (globalThis.navigator?.hardwareConcurrency ?? 2))] : []),
|
|
239
|
+
`--test-name-pattern=(${entries.map(pattern).join("|")})`,
|
|
240
|
+
"--test-reporter=spec", "--test-reporter-destination=stdout",
|
|
241
|
+
"--test-reporter=junit", `--test-reporter-destination=${report}`, ...files];
|
|
242
|
+
const run = { laws: entries, command: coverage ? "npx" : process.execPath,
|
|
243
|
+
args: coverage ? ["--no-install", "c8", "--reporter=text", `--reports-dir=${coverage}`, process.execPath, ...node] : node,
|
|
101
244
|
report: { kind: "junit", files: [report] },
|
|
102
245
|
ran: (test) => entries.filter((e) => new RegExp(pattern(e)).test(test.name)) };
|
|
103
246
|
return language === "typescript"
|
|
@@ -107,15 +250,18 @@ export function invocations(target, entries, { offline = false, scratch = "." }
|
|
|
107
250
|
if (language === "go")
|
|
108
251
|
return groupBy(entries, (e) => path.posix.dirname(e.file)).map(([directory, laws]) => ({
|
|
109
252
|
laws, command: "go",
|
|
110
|
-
args: ["test", "-json", "-count=1",
|
|
253
|
+
args: ["test", "-json", "-count=1", ...(coverage ? ["-cover", `-coverprofile=${path.join(coverage, `go-${directory.replaceAll("/", "_") || "root"}.out`)}`] : []),
|
|
254
|
+
`./${directory}`, "-run", `^(${laws.map((e) => regex(e.name)).join("|")})_`],
|
|
111
255
|
report: { kind: "go-json" },
|
|
112
|
-
ran: (test) => laws.filter((e) =>
|
|
256
|
+
ran: (test) => laws.filter((e) => named(e).test(test.name)) }));
|
|
113
257
|
if (language === "java") {
|
|
114
258
|
const byClass = groupBy(entries, (e) => className(e.file, "src/test/java", ".java"));
|
|
115
259
|
return [{ laws: entries, command: target.maven || "mvn",
|
|
116
|
-
args: [...(offline ? ["-o"] : []), "-B", "
|
|
260
|
+
args: [...(offline ? ["-o"] : []), "-B", ...(coverage ? ["org.jacoco:jacoco-maven-plugin:prepare-agent"] : []), "test",
|
|
261
|
+
...(coverage ? ["org.jacoco:jacoco-maven-plugin:report"] : []),
|
|
262
|
+
`-Dtest=${byClass.map(([name, laws]) => `${name}#${laws.map((e) => `${e.name}_*`).join("+")}`).join(",")}`],
|
|
117
263
|
report: { kind: "junit", directory: "target/surefire-reports" },
|
|
118
|
-
ran: (test) => entries.filter((e) => test.classname === className(e.file, "src/test/java", ".java") &&
|
|
264
|
+
ran: (test) => entries.filter((e) => test.classname === className(e.file, "src/test/java", ".java") && named(e).test(test.name)) }];
|
|
119
265
|
}
|
|
120
266
|
// Kotest names tests by string, which Gradle's method filters cannot
|
|
121
267
|
// select. Kotest's own filter selects them within each class: one pattern,
|
|
@@ -123,35 +269,152 @@ export function invocations(target, entries, { offline = false, scratch = "." }
|
|
|
123
269
|
// patterns must all match, so alternatives go in one group).
|
|
124
270
|
if (language === "kotlin")
|
|
125
271
|
return groupBy(entries, (e) => e.file).map(([file, laws]) => {
|
|
126
|
-
const filter = `(${laws.
|
|
272
|
+
const filter = `(${laws.map((e) => `${e.name}_`).join("|")})*`;
|
|
127
273
|
const name = className(file, "src/test/kotlin", ".kt");
|
|
128
274
|
return { laws, command: target.gradle || "gradle",
|
|
129
|
-
args: [...(offline ? ["--offline"] : []), "--console=plain", "test", "--rerun", "--tests", name],
|
|
275
|
+
args: [...(offline ? ["--offline"] : []), "--console=plain", "test", ...(coverage ? ["koverXmlReport"] : []), "--rerun", "--tests", name],
|
|
130
276
|
env: { "kotest.filter.tests": filter, kotest_filter_tests: filter },
|
|
131
277
|
report: { kind: "junit", directory: "build/test-results/test" },
|
|
132
|
-
ran: (test) => laws.filter((e) => test.classname === name &&
|
|
278
|
+
ran: (test) => laws.filter((e) => test.classname === name && named(e).test(test.name)) };
|
|
133
279
|
});
|
|
134
280
|
if (language === "rust")
|
|
135
281
|
return groupBy(entries, (e) => e.file).map(([file, laws]) => ({
|
|
136
282
|
laws, command: "cargo",
|
|
137
|
-
args: ["test", ...(offline ? ["--offline"] : []), "--test", path.posix.basename(file, ".rs"), "--", "--exact",
|
|
138
|
-
...laws.map((e) =>
|
|
139
|
-
report: { kind: "lines", pattern: /^test (
|
|
140
|
-
ran: (test) => laws.filter((e) => test.name ===
|
|
283
|
+
args: [...(coverage ? ["llvm-cov", "--no-report"] : []), "test", ...(offline ? ["--offline"] : []), "--test", path.posix.basename(file, ".rs"), "--", "--exact",
|
|
284
|
+
...laws.map((e) => e.name)],
|
|
285
|
+
report: { kind: "lines", pattern: /^test (\S+) \.\.\. (ok|FAILED|ignored)/ },
|
|
286
|
+
ran: (test) => laws.filter((e) => test.name === e.name) }));
|
|
141
287
|
if (language === "haskell") {
|
|
142
288
|
const module = (e) => without(relativeTo(e.file, "test"), "Spec.hs").replaceAll("/", ".");
|
|
143
|
-
const matches = entries.
|
|
289
|
+
const matches = entries.map((e) => `${module(e)}/${e.name}_`);
|
|
144
290
|
return [{ laws: entries, command: "stack",
|
|
145
|
-
args: ["--no-terminal", "test", "--test-arguments", matches.map((m) => `--match ${m}`).join(" ")],
|
|
291
|
+
args: ["--no-terminal", "test", ...(coverage ? ["--coverage"] : []), "--test-arguments", matches.map((m) => `--match ${m}`).join(" ")],
|
|
146
292
|
report: { kind: "hspec" },
|
|
147
|
-
ran: (test) => entries.filter((e) => test.classname === module(e) &&
|
|
293
|
+
ran: (test) => entries.filter((e) => test.classname === module(e) && named(e).test(test.name)) }];
|
|
148
294
|
}
|
|
149
295
|
throw new Error(`lawspec test does not support ${language}`);
|
|
150
296
|
}
|
|
151
297
|
|
|
298
|
+
// What --coverage needs on each target, and how to say it is missing.
|
|
299
|
+
export const coverageTools = {
|
|
300
|
+
erlang: { tool: "OTP cover", install: "install Erlang/OTP with the tools application" },
|
|
301
|
+
elixir: { tool: "OTP cover", install: "install Erlang/OTP with the tools application" },
|
|
302
|
+
gleam: { tool: "OTP cover", install: "install Erlang/OTP with the tools application" },
|
|
303
|
+
python: { tool: "coverage.py", check: ["-c", "import coverage"], install: "python -m pip install coverage" },
|
|
304
|
+
javascript: { tool: "c8", install: "npm install --save-dev c8" },
|
|
305
|
+
typescript: { tool: "c8", install: "npm install --save-dev c8" },
|
|
306
|
+
go: { tool: "go test -cover", install: "(built in)" },
|
|
307
|
+
java: { tool: "JaCoCo", install: "(resolved by Maven as org.jacoco:jacoco-maven-plugin)" },
|
|
308
|
+
kotlin: { tool: "Kover", install: "add id(\"org.jetbrains.kotlinx.kover\") to the plugins of build.gradle.kts" },
|
|
309
|
+
rust: { tool: "cargo-llvm-cov", install: "cargo install cargo-llvm-cov" },
|
|
310
|
+
haskell: { tool: "hpc", install: "(built into GHC; stack test --coverage)" },
|
|
311
|
+
};
|
|
312
|
+
|
|
313
|
+
// Laws selected by tag: every --tag, and none of --exclude-tag.
|
|
314
|
+
export function selectByTags(entries, include = [], exclude = []) {
|
|
315
|
+
return entries.filter((e) => (!include.length || include.some((t) => (e.tags ?? []).includes(t))) &&
|
|
316
|
+
!exclude.some((t) => (e.tags ?? []).includes(t)));
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
// A recorded failure takes precedence over any older passing cache entry,
|
|
320
|
+
// including when an adapter is repaired by restoring its previous contents.
|
|
321
|
+
export function testsToRun(entries, keys, previous, failures, {fresh = false, coverage = false, updateRecorded = false} = {}) {
|
|
322
|
+
return entries.filter((entry) => fresh || coverage || updateRecorded || failures[entry.law] ||
|
|
323
|
+
previous[entry.law]?.key !== keys.get(entry.law));
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
// Read complete outer elements, preserving nested suites and native payloads.
|
|
327
|
+
// Comments and CDATA can contain apparent XML tags; they are never elements.
|
|
328
|
+
function junitElements(xml, names) {
|
|
329
|
+
const elements = [];
|
|
330
|
+
let current;
|
|
331
|
+
const tags = /<!--[\s\S]*?-->|<!\[CDATA\[[\s\S]*?\]\]>|<\?[\s\S]*?\?>|<(\/?)([A-Za-z_][\w:.-]*)(?:[^"'<>]|"[^"]*"|'[^']*')*>/g;
|
|
332
|
+
for (const tag of xml.matchAll(tags)) {
|
|
333
|
+
const [, closing, name] = tag;
|
|
334
|
+
if (!name || !names.includes(name)) continue;
|
|
335
|
+
const empty = /\/\s*>$/.test(tag[0]);
|
|
336
|
+
if (!current) {
|
|
337
|
+
if (closing) continue;
|
|
338
|
+
current = {name, start: tag.index, opening: tag[0], depth: 0};
|
|
339
|
+
}
|
|
340
|
+
if (name !== current.name) continue;
|
|
341
|
+
current.depth += closing ? -1 : empty ? 0 : 1;
|
|
342
|
+
if (current.depth === 0) {
|
|
343
|
+
elements.push({...current, xml: xml.slice(current.start, tag.index + tag[0].length)});
|
|
344
|
+
current = undefined;
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
return elements;
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
function junitCounts(cases) {
|
|
351
|
+
const statuses = cases.map(test => new Set(junitElements(test.xml, ["failure", "error", "skipped"]).map(e => e.name)));
|
|
352
|
+
return {tests: cases.length, ...Object.fromEntries(["failure", "error", "skipped"].map(name =>
|
|
353
|
+
[name === "skipped" ? name : `${name}s`, statuses.filter(status => status.has(name)).length]))};
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
// One JUnit report from every target's: each target's suites, renamed with
|
|
357
|
+
// the target, in one <testsuites>.
|
|
358
|
+
export function mergeJunit(reports) {
|
|
359
|
+
const suites = [];
|
|
360
|
+
const totals = {tests: 0, failures: 0, errors: 0, skipped: 0};
|
|
361
|
+
for (const { target, xml } of reports) {
|
|
362
|
+
const elements = junitElements(xml, ["testsuite", "testcase"]);
|
|
363
|
+
const found = elements.filter(e => e.name === "testsuite");
|
|
364
|
+
// Node's native JUnit reporter puts top-level tests directly in testsuites.
|
|
365
|
+
// Keep those cases (and their native failure payloads) instead of silently
|
|
366
|
+
// dropping them because they have no intervening testsuite element.
|
|
367
|
+
const loose = elements.filter(e => e.name === "testcase");
|
|
368
|
+
if (loose.length) {
|
|
369
|
+
const opening = `<testsuite name="native" ${Object.entries(junitCounts(loose)).map(([key, value]) => `${key}="${value}"`).join(" ")}>`;
|
|
370
|
+
found.push({opening, xml: opening + "\n" + loose.map(e => e.xml).join("\n") + "\n</testsuite>"});
|
|
371
|
+
}
|
|
372
|
+
for (const suite of found) {
|
|
373
|
+
const attributes = [...suite.opening.matchAll(/\s([A-Za-z_:][\w:.-]*)\s*=\s*(?:"([^"]*)"|'([^']*)')/g)];
|
|
374
|
+
const values = Object.fromEntries(attributes.map(match => [match[1], match[2] ?? match[3]]));
|
|
375
|
+
const counts = junitCounts(junitElements(suite.xml, ["testcase"]));
|
|
376
|
+
for (const key of Object.keys(totals))
|
|
377
|
+
totals[key] += /^\d+$/.test(values[key] ?? "") ? Number(values[key]) : counts[key];
|
|
378
|
+
const name = attributes.find(match => match[1] === "name");
|
|
379
|
+
const opening = name
|
|
380
|
+
? suite.opening.slice(0, name.index) + ` name="${escapeXml(target)}: ${values.name.replaceAll('"', '"')}"` + suite.opening.slice(name.index + name[0].length)
|
|
381
|
+
: suite.opening.replace(/^<testsuite\b/, `<testsuite name="${escapeXml(target)}"`);
|
|
382
|
+
suites.push(opening + suite.xml.slice(suite.opening.length));
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
return `<?xml version="1.0" encoding="UTF-8"?>\n<testsuites tests="${totals.tests}" failures="${totals.failures}" errors="${totals.errors}" skipped="${totals.skipped}">\n` +
|
|
386
|
+
suites.join("\n") + "\n</testsuites>\n";
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
const escapeXml = (text) => text.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll('"', """);
|
|
390
|
+
|
|
391
|
+
// The harness statistics a run wrote (LAWSPEC_STATS): cover results, label
|
|
392
|
+
// counts, flaky and known-failing outcomes, benchmarks.
|
|
393
|
+
export async function harnessStatistics(directory) {
|
|
394
|
+
const names = await readdir(directory).catch(() => []);
|
|
395
|
+
const entries = [];
|
|
396
|
+
for (const name of names.filter((n) => n.endsWith(".json")).sort())
|
|
397
|
+
try { entries.push(JSON.parse(await readFile(path.join(directory, name), "utf8"))); } catch { /* partial */ }
|
|
398
|
+
return entries;
|
|
399
|
+
}
|
|
400
|
+
|
|
152
401
|
// The tests a run executed, as {name, classname}, from the runner's report or
|
|
153
402
|
// output. Skipped tests are not counted.
|
|
154
403
|
export async function executedTests(report, output, root, since) {
|
|
404
|
+
if (report?.kind === "beam-events") {
|
|
405
|
+
// A native listener seals the invocation after its final event. Missing,
|
|
406
|
+
// stale, malformed or interrupted reports are not execution evidence.
|
|
407
|
+
const raw = await readFile(path.resolve(root, report.file), "utf8").catch(() => "");
|
|
408
|
+
let rows;
|
|
409
|
+
try { rows = raw.trim().split("\n").map((line) => JSON.parse(line)); } catch { return []; }
|
|
410
|
+
if (rows.length < 2 || rows[0]?.event !== "start" || rows.at(-1)?.event !== "end" ||
|
|
411
|
+
rows.some((row) => row?.run !== report.run)) return [];
|
|
412
|
+
const tests = rows.slice(1, -1);
|
|
413
|
+
if (tests.some((row) => row.event !== "test" || !["passed", "failed"].includes(row.status) ||
|
|
414
|
+
typeof row.name !== "string" || typeof row.classname !== "string" || typeof row.identity !== "string" ||
|
|
415
|
+
!Number.isFinite(row.time) || row.time < 0)) return [];
|
|
416
|
+
return tests;
|
|
417
|
+
}
|
|
155
418
|
if (report?.kind === "junit") {
|
|
156
419
|
const files = report.files ?? await reportFiles(path.join(root, report.directory), since);
|
|
157
420
|
const tests = [];
|
|
@@ -169,13 +432,14 @@ export async function executedTests(report, output, root, since) {
|
|
|
169
432
|
return output.split("\n").flatMap((line) => {
|
|
170
433
|
try {
|
|
171
434
|
const event = JSON.parse(line);
|
|
172
|
-
return ["pass", "fail"].includes(event.Action) && event.Test
|
|
435
|
+
return ["pass", "fail"].includes(event.Action) && event.Test
|
|
436
|
+
? [{ name: event.Test, classname: event.Package, status: event.Action === "pass" ? "passed" : "failed" }] : [];
|
|
173
437
|
} catch { return []; }
|
|
174
438
|
});
|
|
175
439
|
if (report?.kind === "lines")
|
|
176
440
|
return output.split("\n").flatMap((line) => {
|
|
177
441
|
const match = line.match(report.pattern);
|
|
178
|
-
return match ? [{ name: match[1], classname: "" }] : [];
|
|
442
|
+
return match && match[2] !== "ignored" ? [{ name: match[1], classname: "", status: match[2] === "ok" ? "passed" : "failed" }] : [];
|
|
179
443
|
});
|
|
180
444
|
if (report?.kind === "hspec") {
|
|
181
445
|
// hspec prints each module, then its examples indented beneath it.
|
|
@@ -183,14 +447,31 @@ export async function executedTests(report, output, root, since) {
|
|
|
183
447
|
let module = "";
|
|
184
448
|
for (const line of output.split("\n")) {
|
|
185
449
|
if (/^\S/.test(line) && !/^(Finished|Failures|Randomized|\d+ examples?)/.test(line)) module = line.trim();
|
|
186
|
-
const example = line.match(/^\s+(law\
|
|
187
|
-
if (example) tests.push({ name: example[1], classname: module });
|
|
450
|
+
const example = line.match(/^\s+(law\S*?):?(?:\s.*)?\s\[([✔✘])\]\s*$/);
|
|
451
|
+
if (example) tests.push({ name: example[1], classname: module, status: example[2] === "✔" ? "passed" : "failed" });
|
|
188
452
|
}
|
|
189
453
|
return tests;
|
|
190
454
|
}
|
|
191
455
|
return [];
|
|
192
456
|
}
|
|
193
457
|
|
|
458
|
+
// JUnit XML for a runner that writes none, from the tests it printed.
|
|
459
|
+
export function junitFromTests(tests) {
|
|
460
|
+
const failures = tests.filter((t) => t.status === "failed").length;
|
|
461
|
+
const skipped = tests.filter((t) => t.status === "skipped").length;
|
|
462
|
+
// A BEAM scheduler wraps native functions from several units. Its module
|
|
463
|
+
// is not the declaring class: retain the unit from the validated identity.
|
|
464
|
+
const classname = (test) => {
|
|
465
|
+
const separator = typeof test.identity === "string" ? test.identity.lastIndexOf("::") : -1;
|
|
466
|
+
return separator > 0 ? test.identity.slice(0, separator) : test.classname ?? "";
|
|
467
|
+
};
|
|
468
|
+
return `<testsuite name="lawspec" tests="${tests.length}" failures="${failures}"${skipped ? ` skipped="${skipped}"` : ""}>\n` +
|
|
469
|
+
tests.map((t) => ` <testcase classname="${escapeXml(classname(t))}" name="${escapeXml(t.name)}"` +
|
|
470
|
+
(Number.isFinite(t.time) ? ` time="${t.time}"` : "") +
|
|
471
|
+
(t.status === "failed" ? `><failure message="${escapeXml(t.failure ?? "failed")}"/></testcase>` :
|
|
472
|
+
t.status === "skipped" ? `><skipped message="${escapeXml(t.reason ?? "skipped")}"/></testcase>` : "/>")).join("\n") + "\n</testsuite>";
|
|
473
|
+
}
|
|
474
|
+
|
|
194
475
|
async function reportFiles(directory, since) {
|
|
195
476
|
const names = await readdir(directory).catch(() => []);
|
|
196
477
|
const files = [];
|