lawspec 0.20.0 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/test-command.mjs CHANGED
@@ -49,6 +49,25 @@ export async function environmentDigest(root, report) {
49
49
  return digest(JSON.stringify(parts));
50
50
  }
51
51
 
52
+ // Every recorded value under recorded/, by content: "" when there are none.
53
+ export async function recordedDigest(folder) {
54
+ const hash = createHash("sha256");
55
+ let any = false;
56
+ async function walk(relative) {
57
+ const entries = await readdir(path.join(folder, relative), { withFileTypes: true }).catch(() => []);
58
+ for (const entry of entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0))) {
59
+ const next = relative ? `${relative}/${entry.name}` : entry.name;
60
+ if (entry.isDirectory()) await walk(next);
61
+ else if (entry.isFile()) {
62
+ any = true;
63
+ hash.update(next + "\0").update(await readFile(path.join(folder, next))).update("\0");
64
+ }
65
+ }
66
+ }
67
+ await walk("");
68
+ return any ? hash.digest("hex") : "";
69
+ }
70
+
52
71
  // The result key of each law in a target's test manifest.
53
72
  export function lawKeys({ build, target, machineBits, minify, tests, files, environment, project }) {
54
73
  const contents = new Map(files.map((file) => [file.path, file.content]));
@@ -67,37 +86,96 @@ const groupBy = (entries, key) => {
67
86
  return [...groups.entries()];
68
87
  };
69
88
 
89
+ // The native invocations that run the harness's benchmarks (lawspec test
90
+ // --benchmarks): each target's test command, selecting the tests that
91
+ // measure them by the names the manifest gives (entry.name). Benchmarks are
92
+ // measured, never asserted; their timings reach the summary through the
93
+ // harness statistics.
94
+ export function benchmarkInvocations(target, entries, { offline = false } = {}) {
95
+ if (!entries.length) return [];
96
+ const language = target.language;
97
+ const testDir = target.testDir;
98
+ const relativeTo = (file, defaultDir) => {
99
+ const directory = testDir || defaultDir;
100
+ return file.startsWith(directory + "/") ? file.slice(directory.length + 1) : file;
101
+ };
102
+ const className = (file, defaultDir, extension) => without(relativeTo(file, defaultDir), extension).replaceAll("/", ".");
103
+ if (language === "python")
104
+ return groupBy(entries, (e) => e.file).map(([file, marks]) => ({ command: target.python || "python3",
105
+ args: ["-m", "pytest", "-q", "-s", file, "-k", marks.map((e) => e.name).join(" or ")] }));
106
+ if (language === "javascript" || language === "typescript") {
107
+ const files = [...new Set(entries.map((e) => language === "typescript" ? `dist/${without(e.file, ".ts")}.js` : e.file))];
108
+ const run = { command: process.execPath,
109
+ args: ["--test", `--test-name-pattern=^(${entries.map((e) => regex(e.name)).join("|")})$`, ...files] };
110
+ return language === "typescript" ? [{ command: "npm", args: ["exec", "--", "tsc", "-p", "tsconfig.json"] }, run] : [run];
111
+ }
112
+ if (language === "go")
113
+ return groupBy(entries, (e) => path.posix.dirname(e.file)).map(([directory, marks]) => ({ command: "go",
114
+ args: ["test", "-count=1", "-v", `./${directory}`, "-run", `^(${marks.map((e) => regex(e.name)).join("|")})$`] }));
115
+ if (language === "java") {
116
+ const byClass = groupBy(entries, (e) => className(e.file, "src/test/java", ".java"));
117
+ return [{ command: target.maven || "mvn", args: [...(offline ? ["-o"] : []), "-B", "test",
118
+ `-Dtest=${byClass.map(([name, marks]) => `${name}#${marks.map((e) => e.name).join("+")}`).join(",")}`] }];
119
+ }
120
+ if (language === "kotlin")
121
+ return groupBy(entries, (e) => e.file).map(([file, marks]) => {
122
+ const filter = `(${marks.map((e) => e.name).join("|")})`;
123
+ return { command: target.gradle || "gradle",
124
+ args: [...(offline ? ["--offline"] : []), "--console=plain", "test", "--rerun", "--tests", className(file, "src/test/kotlin", ".kt")],
125
+ env: { "kotest.filter.tests": filter, kotest_filter_tests: filter } };
126
+ });
127
+ if (language === "rust")
128
+ return groupBy(entries, (e) => e.file).map(([file, marks]) => ({ command: "cargo",
129
+ args: ["test", ...(offline ? ["--offline"] : []), "--test", path.posix.basename(file, ".rs"), "--", "--exact", "--nocapture", ...marks.map((e) => e.name)] }));
130
+ if (language === "haskell") {
131
+ const module = (e) => without(relativeTo(e.file, "test"), "Spec.hs").replaceAll("/", ".");
132
+ // hspec matches by substring, and a benchmark's name has spaces, so each
133
+ // module's benchmarks are selected together.
134
+ return [{ command: "stack", args: ["--no-terminal", "test", "--test-arguments",
135
+ [...new Set(entries.map((e) => `--match ${module(e)}/benchmark`))].join(" ")] }];
136
+ }
137
+ throw new Error(`lawspec test does not support ${language}`);
138
+ }
139
+
70
140
  // The native invocations that run exactly the given laws' tests, each with
71
141
  // the laws it covers and how to tell, from the runner's own report, which of
72
- // them ran. Names are matched whole, so law1 never selects law10. `scratch`
73
- // is a folder for reports.
74
- export function invocations(target, entries, { offline = false, scratch = "." } = {}) {
142
+ // them ran. A law's tests are named after its label (entry.name, see
143
+ // LawSpec.TestNames), then a kind after a separator no name contains, so one
144
+ // law's name never selects another's. `scratch` is a folder for reports;
145
+ // `coverage` asks each runner to measure coverage too (see coverageSetup).
146
+ export function invocations(target, entries, { offline = false, scratch = ".", coverage = null, xdist = false } = {}) {
75
147
  const language = target.language;
76
148
  const testDir = target.testDir;
77
149
  const relativeTo = (file, defaultDir) => {
78
150
  const directory = testDir || defaultDir;
79
151
  return file.startsWith(directory + "/") ? file.slice(directory.length + 1) : file;
80
152
  };
81
- const tests = (entry) => [`law${entry.index}Example*`, `law${entry.index}Boundary*`, `law${entry.index}Property*`];
82
- const kinds = (entry) => new RegExp(`^law${entry.index}(Example|Boundary|Property)`);
153
+ const named = (entry) => new RegExp(`^${regex(entry.name)}_`);
83
154
  const className = (file, defaultDir, extension) => without(relativeTo(file, defaultDir), extension).replaceAll("/", ".");
84
155
  if (language === "python")
85
156
  return groupBy(entries, (e) => e.file).map(([file, laws], n) => {
86
157
  const report = path.join(scratch, `pytest-${n}.xml`);
158
+ // parallel: pytest-xdist runs the tests on several workers, when it
159
+ // is installed (lawspec test checks); otherwise one after another.
160
+ const workers = xdist && laws.some((e) => e.parallel) ? ["-p", "xdist", "-n", "auto"] : [];
161
+ const pytest = ["-m", "pytest", "-q", ...workers, file, "-k", laws.map((e) => `${e.name}__`).join(" or "), `--junitxml=${report}`];
87
162
  return { laws, command: target.python || "python3",
88
- args: ["-m", "pytest", "-q", file, "-k", laws.map((e) => `test_law${e.index}_`).join(" or "), `--junitxml=${report}`],
163
+ args: coverage ? ["-m", "coverage", "run", "--append", `--data-file=${path.join(coverage, ".coverage")}`, ...pytest.slice(1)] : pytest,
89
164
  report: { kind: "junit", files: [report] },
90
- ran: (test) => laws.filter((e) => test.name.startsWith(`test_law${e.index}_`)) };
165
+ ran: (test) => laws.filter((e) => test.name.startsWith(`${e.name}__`)) };
91
166
  });
92
167
  if (language === "javascript" || language === "typescript") {
93
- const pattern = (e) => `^${regex(e.label)}( example: .*| property)?$`;
168
+ const pattern = (e) => `^${regex(e.label)}( example: .*| property| known failing| skipped| replay| search)?$`;
94
169
  const files = [...new Set(entries.map((e) => language === "typescript"
95
170
  ? `dist/${without(e.file, ".ts")}.js` : e.file))];
96
171
  const report = path.join(scratch, "node.xml");
97
- const run = { laws: entries, command: process.execPath,
98
- args: ["--test", `--test-name-pattern=(${entries.map(pattern).join("|")})`,
99
- "--test-reporter=spec", "--test-reporter-destination=stdout",
100
- "--test-reporter=junit", `--test-reporter-destination=${report}`, ...files],
172
+ // parallel: test files run at the same time too.
173
+ const node = ["--test", ...(entries.some((e) => e.parallel) ? ["--test-concurrency=" + Math.max(2, (globalThis.navigator?.hardwareConcurrency ?? 2))] : []),
174
+ `--test-name-pattern=(${entries.map(pattern).join("|")})`,
175
+ "--test-reporter=spec", "--test-reporter-destination=stdout",
176
+ "--test-reporter=junit", `--test-reporter-destination=${report}`, ...files];
177
+ const run = { laws: entries, command: coverage ? "npx" : process.execPath,
178
+ args: coverage ? ["--no-install", "c8", "--reporter=text", `--reports-dir=${coverage}`, process.execPath, ...node] : node,
101
179
  report: { kind: "junit", files: [report] },
102
180
  ran: (test) => entries.filter((e) => new RegExp(pattern(e)).test(test.name)) };
103
181
  return language === "typescript"
@@ -107,15 +185,18 @@ export function invocations(target, entries, { offline = false, scratch = "." }
107
185
  if (language === "go")
108
186
  return groupBy(entries, (e) => path.posix.dirname(e.file)).map(([directory, laws]) => ({
109
187
  laws, command: "go",
110
- args: ["test", "-json", "-count=1", `./${directory}`, "-run", `^TestLaw(${laws.map((e) => e.index).join("|")})(Example|Boundary|Property)`],
188
+ args: ["test", "-json", "-count=1", ...(coverage ? ["-cover", `-coverprofile=${path.join(coverage, `go-${directory.replaceAll("/", "_") || "root"}.out`)}`] : []),
189
+ `./${directory}`, "-run", `^(${laws.map((e) => regex(e.name)).join("|")})_`],
111
190
  report: { kind: "go-json" },
112
- ran: (test) => laws.filter((e) => new RegExp(`^TestLaw${e.index}(Example|Boundary|Property)`).test(test.name)) }));
191
+ ran: (test) => laws.filter((e) => named(e).test(test.name)) }));
113
192
  if (language === "java") {
114
193
  const byClass = groupBy(entries, (e) => className(e.file, "src/test/java", ".java"));
115
194
  return [{ laws: entries, command: target.maven || "mvn",
116
- args: [...(offline ? ["-o"] : []), "-B", "test", `-Dtest=${byClass.map(([name, laws]) => `${name}#${laws.flatMap(tests).join("+")}`).join(",")}`],
195
+ args: [...(offline ? ["-o"] : []), "-B", ...(coverage ? ["org.jacoco:jacoco-maven-plugin:prepare-agent"] : []), "test",
196
+ ...(coverage ? ["org.jacoco:jacoco-maven-plugin:report"] : []),
197
+ `-Dtest=${byClass.map(([name, laws]) => `${name}#${laws.map((e) => `${e.name}_*`).join("+")}`).join(",")}`],
117
198
  report: { kind: "junit", directory: "target/surefire-reports" },
118
- ran: (test) => entries.filter((e) => test.classname === className(e.file, "src/test/java", ".java") && kinds(e).test(test.name)) }];
199
+ ran: (test) => entries.filter((e) => test.classname === className(e.file, "src/test/java", ".java") && named(e).test(test.name)) }];
119
200
  }
120
201
  // Kotest names tests by string, which Gradle's method filters cannot
121
202
  // select. Kotest's own filter selects them within each class: one pattern,
@@ -123,32 +204,77 @@ export function invocations(target, entries, { offline = false, scratch = "." }
123
204
  // patterns must all match, so alternatives go in one group).
124
205
  if (language === "kotlin")
125
206
  return groupBy(entries, (e) => e.file).map(([file, laws]) => {
126
- const filter = `(${laws.flatMap((e) => ["Example", "Boundary", "Property"].map((kind) => `law${e.index}${kind}`)).join("|")})*`;
207
+ const filter = `(${laws.map((e) => `${e.name}_`).join("|")})*`;
127
208
  const name = className(file, "src/test/kotlin", ".kt");
128
209
  return { laws, command: target.gradle || "gradle",
129
- args: [...(offline ? ["--offline"] : []), "--console=plain", "test", "--rerun", "--tests", name],
210
+ args: [...(offline ? ["--offline"] : []), "--console=plain", "test", ...(coverage ? ["koverXmlReport"] : []), "--rerun", "--tests", name],
130
211
  env: { "kotest.filter.tests": filter, kotest_filter_tests: filter },
131
212
  report: { kind: "junit", directory: "build/test-results/test" },
132
- ran: (test) => laws.filter((e) => test.classname === name && kinds(e).test(test.name)) };
213
+ ran: (test) => laws.filter((e) => test.classname === name && named(e).test(test.name)) };
133
214
  });
134
215
  if (language === "rust")
135
216
  return groupBy(entries, (e) => e.file).map(([file, laws]) => ({
136
217
  laws, command: "cargo",
137
- args: ["test", ...(offline ? ["--offline"] : []), "--test", path.posix.basename(file, ".rs"), "--", "--exact",
138
- ...laws.map((e) => `test_${e.index}`)],
139
- report: { kind: "lines", pattern: /^test (test_\d+) \.\.\. (ok|FAILED)/ },
140
- ran: (test) => laws.filter((e) => test.name === `test_${e.index}`) }));
218
+ args: [...(coverage ? ["llvm-cov", "--no-report"] : []), "test", ...(offline ? ["--offline"] : []), "--test", path.posix.basename(file, ".rs"), "--", "--exact",
219
+ ...laws.map((e) => e.name)],
220
+ report: { kind: "lines", pattern: /^test (\S+) \.\.\. (ok|FAILED|ignored)/ },
221
+ ran: (test) => laws.filter((e) => test.name === e.name) }));
141
222
  if (language === "haskell") {
142
223
  const module = (e) => without(relativeTo(e.file, "test"), "Spec.hs").replaceAll("/", ".");
143
- const matches = entries.flatMap((e) => ["Example", "Boundary", "Property"].map((kind) => `${module(e)}/law${e.index}${kind}`));
224
+ const matches = entries.map((e) => `${module(e)}/${e.name}_`);
144
225
  return [{ laws: entries, command: "stack",
145
- args: ["--no-terminal", "test", "--test-arguments", matches.map((m) => `--match ${m}`).join(" ")],
226
+ args: ["--no-terminal", "test", ...(coverage ? ["--coverage"] : []), "--test-arguments", matches.map((m) => `--match ${m}`).join(" ")],
146
227
  report: { kind: "hspec" },
147
- ran: (test) => entries.filter((e) => test.classname === module(e) && kinds(e).test(test.name)) }];
228
+ ran: (test) => entries.filter((e) => test.classname === module(e) && named(e).test(test.name)) }];
148
229
  }
149
230
  throw new Error(`lawspec test does not support ${language}`);
150
231
  }
151
232
 
233
+ // What --coverage needs on each target, and how to say it is missing.
234
+ export const coverageTools = {
235
+ python: { tool: "coverage.py", check: ["-c", "import coverage"], install: "python -m pip install coverage" },
236
+ javascript: { tool: "c8", install: "npm install --save-dev c8" },
237
+ typescript: { tool: "c8", install: "npm install --save-dev c8" },
238
+ go: { tool: "go test -cover", install: "(built in)" },
239
+ java: { tool: "JaCoCo", install: "(resolved by Maven as org.jacoco:jacoco-maven-plugin)" },
240
+ kotlin: { tool: "Kover", install: "add id(\"org.jetbrains.kotlinx.kover\") to the plugins of build.gradle.kts" },
241
+ rust: { tool: "cargo-llvm-cov", install: "cargo install cargo-llvm-cov" },
242
+ haskell: { tool: "hpc", install: "(built into GHC; stack test --coverage)" },
243
+ };
244
+
245
+ // Laws selected by tag: every --tag, and none of --exclude-tag.
246
+ export function selectByTags(entries, include = [], exclude = []) {
247
+ return entries.filter((e) => (!include.length || include.some((t) => (e.tags ?? []).includes(t))) &&
248
+ !exclude.some((t) => (e.tags ?? []).includes(t)));
249
+ }
250
+
251
+ // One JUnit report from every target's: each target's suites, renamed with
252
+ // the target, in one <testsuites>.
253
+ export function mergeJunit(reports) {
254
+ const suites = [];
255
+ for (const { target, xml } of reports) {
256
+ const found = [...xml.matchAll(/<testsuite\b[\s\S]*?<\/testsuite>|<testsuite\b[^>]*\/>/g)].map((m) => m[0]);
257
+ for (const suite of found)
258
+ suites.push(suite.replace(/<testsuite\b([^>]*?)\bname="([^"]*)"/, (whole, before, name) => `<testsuite${before}name="${escapeXml(target)}: ${name}"`)
259
+ .replace(/^<testsuite\b(?![^>]*\bname=)/, `<testsuite name="${escapeXml(target)}"`));
260
+ }
261
+ const count = (attribute) => suites.reduce((n, suite) => n + Number(suite.match(new RegExp(`\\b${attribute}="(\\d+)"`))?.[1] ?? 0), 0);
262
+ return `<?xml version="1.0" encoding="UTF-8"?>\n<testsuites tests="${count("tests")}" failures="${count("failures")}" errors="${count("errors")}" skipped="${count("skipped")}">\n` +
263
+ suites.join("\n") + "\n</testsuites>\n";
264
+ }
265
+
266
+ const escapeXml = (text) => text.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;").replaceAll('"', "&quot;");
267
+
268
+ // The harness statistics a run wrote (LAWSPEC_STATS): cover results, label
269
+ // counts, flaky and known-failing outcomes, benchmarks.
270
+ export async function harnessStatistics(directory) {
271
+ const names = await readdir(directory).catch(() => []);
272
+ const entries = [];
273
+ for (const name of names.filter((n) => n.endsWith(".json")).sort())
274
+ try { entries.push(JSON.parse(await readFile(path.join(directory, name), "utf8"))); } catch { /* partial */ }
275
+ return entries;
276
+ }
277
+
152
278
  // The tests a run executed, as {name, classname}, from the runner's report or
153
279
  // output. Skipped tests are not counted.
154
280
  export async function executedTests(report, output, root, since) {
@@ -169,13 +295,14 @@ export async function executedTests(report, output, root, since) {
169
295
  return output.split("\n").flatMap((line) => {
170
296
  try {
171
297
  const event = JSON.parse(line);
172
- return ["pass", "fail"].includes(event.Action) && event.Test ? [{ name: event.Test, classname: event.Package }] : [];
298
+ return ["pass", "fail"].includes(event.Action) && event.Test
299
+ ? [{ name: event.Test, classname: event.Package, status: event.Action === "pass" ? "passed" : "failed" }] : [];
173
300
  } catch { return []; }
174
301
  });
175
302
  if (report?.kind === "lines")
176
303
  return output.split("\n").flatMap((line) => {
177
304
  const match = line.match(report.pattern);
178
- return match ? [{ name: match[1], classname: "" }] : [];
305
+ return match && match[2] !== "ignored" ? [{ name: match[1], classname: "", status: match[2] === "ok" ? "passed" : "failed" }] : [];
179
306
  });
180
307
  if (report?.kind === "hspec") {
181
308
  // hspec prints each module, then its examples indented beneath it.
@@ -183,14 +310,22 @@ export async function executedTests(report, output, root, since) {
183
310
  let module = "";
184
311
  for (const line of output.split("\n")) {
185
312
  if (/^\S/.test(line) && !/^(Finished|Failures|Randomized|\d+ examples?)/.test(line)) module = line.trim();
186
- const example = line.match(/^\s+(law\d+\S*?):?(?:\s.*)?\s\[[✔✘]\]\s*$/);
187
- if (example) tests.push({ name: example[1], classname: module });
313
+ const example = line.match(/^\s+(law\S*?):?(?:\s.*)?\s\[([✔✘])\]\s*$/);
314
+ if (example) tests.push({ name: example[1], classname: module, status: example[2] === "✔" ? "passed" : "failed" });
188
315
  }
189
316
  return tests;
190
317
  }
191
318
  return [];
192
319
  }
193
320
 
321
+ // JUnit XML for a runner that writes none, from the tests it printed.
322
+ export function junitFromTests(tests) {
323
+ const failures = tests.filter((t) => t.status === "failed").length;
324
+ return `<testsuite name="lawspec" tests="${tests.length}" failures="${failures}">\n` +
325
+ tests.map((t) => ` <testcase classname="${escapeXml(t.classname ?? "")}" name="${escapeXml(t.name)}"` +
326
+ (t.status === "failed" ? `><failure message="failed"/></testcase>` : "/>")).join("\n") + "\n</testsuite>";
327
+ }
328
+
194
329
  async function reportFiles(directory, since) {
195
330
  const names = await readdir(directory).catch(() => []);
196
331
  const files = [];