lawspec 0.20.1 → 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/CHANGELOG.md +361 -2
  2. package/README.md +3 -3
  3. package/beam-coverage.mjs +160 -0
  4. package/beam-doctor.mjs +306 -0
  5. package/bin/lawspec.mjs +306 -38
  6. package/build.json +492 -106
  7. package/compatibility.json +17 -0
  8. package/core.wasm +0 -0
  9. package/core_jsffi.js +2 -2
  10. package/doctor.mjs +7 -1
  11. package/examples/native-payments/elixir/lawspec.json +149 -0
  12. package/examples/native-payments/elixir/lib/payments_domain.ex +54 -0
  13. package/examples/native-payments/elixir/test/support/payment_generators.ex +9 -0
  14. package/examples/native-payments/erlang/include/payments_domain.hrl +2 -0
  15. package/examples/native-payments/erlang/lawspec.json +148 -0
  16. package/examples/native-payments/erlang/src/payments_domain.erl +21 -0
  17. package/examples/native-payments/erlang/test/payment_generators.erl +10 -0
  18. package/examples/native-payments/gleam/lawspec.json +149 -0
  19. package/examples/native-payments/gleam/prepare.mjs +48 -0
  20. package/examples/native-payments/gleam/src/payments_domain.gleam +32 -0
  21. package/examples/native-payments/gleam/test/payment_generators.gleam +13 -0
  22. package/examples/specs/abilities.lawspec +103 -0
  23. package/examples/specs/ability_imports.lawspec +107 -0
  24. package/examples/specs/builtins.lawspec +121 -0
  25. package/examples/specs/crypto.lawspec +51 -0
  26. package/examples/specs/distribution.lawspec +29 -0
  27. package/examples/specs/failures.lawspec +96 -0
  28. package/examples/specs/handler_bindings.lawspec +65 -0
  29. package/examples/specs/harness.lawspec +75 -0
  30. package/examples/specs/harness_testing.lawspec +45 -0
  31. package/examples/specs/matchers.lawspec +112 -0
  32. package/examples/specs/resources.lawspec +116 -0
  33. package/examples/specs/scheduling_order.lawspec +18 -0
  34. package/examples/specs/scheduling_overlap.lawspec +14 -0
  35. package/examples/specs/tables.lawspec +64 -0
  36. package/failure-database.mjs +46 -0
  37. package/index.d.ts +12 -1
  38. package/native-examples.mjs +7 -2
  39. package/package.json +1 -1
  40. package/templates.mjs +105 -16
  41. package/test-command.mjs +316 -35
package/bin/lawspec.mjs CHANGED
@@ -8,8 +8,11 @@ import { targets, templates, commands, setup } from "../templates.mjs";
8
8
  import { generateExamples } from "../examples-command.mjs";
9
9
  import { showScalar } from "../scalars.mjs";
10
10
  import { doctor } from "../doctor.mjs";
11
- import { spawn } from "node:child_process";
12
- import { environmentDigest, executedTests, invocations, lawKeys, projectDigest } from "../test-command.mjs";
11
+ import { beamCoverageRequirement, collectBeamCoverage } from "../beam-coverage.mjs";
12
+ import { failureDatabase, projectKey } from "../failure-database.mjs";
13
+ import { spawn, spawnSync } from "node:child_process";
14
+ import { benchmarkInvocations, preparationInvocations, environmentDigest, executedTests, invocations, lawKeys, projectDigest, recordedDigest, selectByTags, mergeJunit,
15
+ harnessStatistics, coverageTools, junitFromTests, testsToRun } from "../test-command.mjs";
13
16
  import {
14
17
  readOptional,
15
18
  planWrites,
@@ -17,18 +20,23 @@ import {
17
20
  atomicWrite,
18
21
  safePath,
19
22
  } from "../files.mjs";
20
- const VERSION = "0.20.1";
23
+ const VERSION = "0.22.0";
21
24
  const args = process.argv.slice(2);
22
25
  const verb = args.shift();
23
26
  const options = {};
24
27
  const positional = [];
25
28
  for (let i = 0; i < args.length; i++) {
26
29
  const arg = args[i];
27
- if (["--target", "--project", "--config", "--output", "--machine-bits", "--example", "--seed"].includes(arg)) {
30
+ if (["--target", "--project", "--config", "--output", "--machine-bits", "--example", "--seed", "--report"].includes(arg)) {
28
31
  if (!args[i + 1] || args[i + 1].startsWith("--"))
29
32
  throw new Error(`Missing value for ${arg}`);
30
33
  options[arg.slice(2)] = args[++i];
31
- } else if (["--dry-run", "--check", "--json", "--minify", "--no-cache", "--fresh"].includes(arg))
34
+ } else if (["--tag", "--exclude-tag"].includes(arg)) {
35
+ // Repeatable, and each may list several: --tag a,b --tag c.
36
+ if (!args[i + 1] || args[i + 1].startsWith("--"))
37
+ throw new Error(`Missing value for ${arg}`);
38
+ options[arg.slice(2)] = [...(options[arg.slice(2)] ?? []), ...args[++i].split(",").filter(Boolean)];
39
+ } else if (["--dry-run", "--check", "--json", "--minify", "--no-cache", "--fresh", "--coverage", "--update-recorded", "--benchmarks"].includes(arg))
32
40
  options[arg.slice(2)] = true;
33
41
  else if (arg.startsWith("--")) throw new Error(`Unknown option: ${arg}`);
34
42
  else positional.push(arg);
@@ -191,6 +199,9 @@ async function init() {
191
199
  "build.gradle",
192
200
  "build.gradle.kts",
193
201
  "settings.gradle.kts",
202
+ "rebar.config",
203
+ "mix.exs",
204
+ "gleam.toml",
194
205
  ];
195
206
  const hasBuild = (await readdir(root)).some(
196
207
  (f) => buildFiles.includes(f) || f.endsWith(".cabal"),
@@ -236,6 +247,24 @@ function showAssertion(assertion) {
236
247
  if (assertion.kind === "implies") return `${assertion.guard.text} implies ${showAssertion(assertion.body)}`;
237
248
  return assertion.items.map(showAssertion).join(" and ");
238
249
  }
250
+ // An ability key, as the spec names it: example.shop::ability::Store(Int32)
251
+ // is Store Int32, lawspec::ability::Fail(example.shop::type::E) is Fail E.
252
+ function abilityName(key) {
253
+ const short = (text) => text.replace(/[A-Za-z0-9_.]+::(ability|type)::/g, "");
254
+ const match = /^(.*?)(\((.*)\))?$/.exec(short(key));
255
+ return match[3] ? `${match[1]} ${match[3].replace(/[()]/g, " ").replace(/;/g, " ").trim()}` : match[1];
256
+ }
257
+ // The handlers a law runs under, and the ability row of each function its
258
+ // expansion calls.
259
+ function explainAbilities(law, expansion, units) {
260
+ const unit = (units || []).find((u) => u.id === law.owner);
261
+ const called = (unit ? unit.declarations : []).filter(
262
+ (d) => d.uses && new RegExp(`\\b${d.name}\\b`).test(expansion),
263
+ );
264
+ const handlers = (law.handlers || []).map((h) => `\n ${abilityName(h.ability)}: ${h.handler}`).join("");
265
+ const rows = called.map((d) => `\n ${d.name} uses ${d.uses.map(abilityName).join(", ")}`).join("");
266
+ return (handlers ? `\nhandlers${handlers}` : "") + (rows ? `\nrows${rows}` : "");
267
+ }
239
268
  function explainExamples(law) {
240
269
  return law.examples.map(ex =>
241
270
  `\nexample ${JSON.stringify(ex.name)}\n` +
@@ -243,6 +272,50 @@ function explainExamples(law) {
243
272
  ex.expectations.map(e => ` expect ${showAssertion(e)}`).join("\n")
244
273
  ).join("\n");
245
274
  }
275
+ // What lawspec test last recorded for each law, across targets.
276
+ async function lastRuns() {
277
+ const folder = path.join(configRoot, ".lawspec", "results");
278
+ const runs = new Map();
279
+ for (const name of (await readOptionalDirectory(folder)) ?? []) {
280
+ if (!name.endsWith(".json")) continue;
281
+ const recorded = JSON.parse((await readOptional(path.join(folder, name))) ?? "{}");
282
+ for (const [law, run] of Object.entries(recorded.laws ?? {})) {
283
+ const known = runs.get(law) ?? {};
284
+ runs.set(law, { flaky: known.flaky || run.flaky, adequacy: run.adequacy ?? known.adequacy });
285
+ }
286
+ }
287
+ return runs;
288
+ }
289
+ // One law's harness, as lawspec evidence shows it.
290
+ function harnessText(item) {
291
+ const h = item.harness ?? {};
292
+ const parts = [];
293
+ if (h.unit) parts.push(`harness ${h.unit}`);
294
+ if (h.tags?.length) parts.push(`tags ${h.tags.join(", ")}`);
295
+ if (h.strategies?.length) parts.push(h.strategies.map((s) => `${s.input} drawn by ${s.strategy}`).join(", "));
296
+ for (const c of h.cover ?? []) parts.push(`cover ${c.percent}% "${c.label}" when ${c.when.text}`);
297
+ for (const c of h.classify ?? []) parts.push(`classify ${c.when.text} as "${c.label}"`);
298
+ for (const l of h.labels ?? []) parts.push(`label ${l.text}`);
299
+ if (h.target) parts.push(`target maximize ${h.target.text}`);
300
+ if (h.timeoutMilliseconds) parts.push(`timeout ${h.timeoutMilliseconds} ms`);
301
+ if (h.repeat > 1) parts.push(`repeat ${h.repeat}`);
302
+ if (h.retries) parts.push(`retry flaky ${h.retries}`);
303
+ if (h.skip) parts.push(`skip "${h.skip}"`);
304
+ if (h.knownFailing) parts.push(`known failing "${h.knownFailing}"`);
305
+ if (h.orderRandom) parts.push("order random");
306
+ // Python runs a parallel unit's tests at the same time only with
307
+ // pytest-xdist; without it, parallel changes nothing there.
308
+ if (h.parallel) parts.push("parallel (on Python only with pytest-xdist; otherwise one after another)");
309
+ const lines = [` ${item.declaration.replace("::law::", "::")}: ${parts.join("; ") || "no harness settings"}`];
310
+ for (const run of item.adequacy ?? []) {
311
+ lines.push(` last run: ${run.cases} generated case(s)` +
312
+ (run.cover ?? []).map((c) => `; cover "${c.label}" ${c.observed}% (needs ${c.required}%)${c.met ? "" : " NOT MET"}`).join(""));
313
+ const counts = { ...(run.classes ?? {}), ...Object.fromEntries(Object.entries(run.labels ?? {}).map(([k, v]) => [`label ${k}`, v])) };
314
+ if (Object.keys(counts).length)
315
+ lines.push(` ${Object.entries(counts).map(([k, v]) => `${k}: ${run.cases ? (100 * v / run.cases).toFixed(1) : 0}%`).join(", ")}`);
316
+ }
317
+ return lines.join("\n");
318
+ }
246
319
  // The compiler keeps work between runs in .lawspec/cache, in one folder per
247
320
  // compiler build, so a different build never reads another's entries. The
248
321
  // WebAssembly compiler sees only the working directory. A command that fails
@@ -272,12 +345,19 @@ async function buildDigest() {
272
345
  return buildDigestValue;
273
346
  }
274
347
  // lawspec test: run the tests of the laws whose results may have changed,
275
- // and record each passing law's key and seed (see test-command.mjs).
348
+ // and record each passing law's key and seed (see test-command.mjs). The
349
+ // harness plane chooses which: --tag and --exclude-tag select laws by their
350
+ // harness tags, and a skipped law runs nothing. A law whose last run failed
351
+ // runs first, and its failing inputs are replayed first (.lawspec/failures).
276
352
  async function runTests(compiler, input, selected, roots, config) {
353
+ // Recorded values live beside lawspec.json, under recorded/<unit>/<name>.
354
+ const recordedFolder = path.join(configRoot, "recorded");
277
355
  const seed = options.seed ?? String(1 + Math.floor(Math.random() * 2147483646));
278
356
  const offline = process.env.LAWSPEC_OFFLINE === "1";
279
357
  const build = await buildDigest();
280
358
  const summaries = [];
359
+ const junit = [];
360
+ const report = options.report === undefined ? null : parseReport(options.report);
281
361
  for (const [i, target] of selected.entries()) {
282
362
  const root = roots[i];
283
363
  const planned = diagnostics(await compiler.planGeneration({
@@ -287,53 +367,214 @@ async function runTests(compiler, input, selected, roots, config) {
287
367
  const pending = await planWrites(root, planned.files);
288
368
  if (pending.changes.length)
289
369
  throw new Error(`${target.language}: generated files are out of date; run lawspec generate${options.minify ? " --minify" : ""}`);
290
- const report = await doctor(target, root, planned.files);
291
- if (!report.ok) throw new Error(`${target.language}: ${report.message}\n${report.instructions}`);
370
+ const report_ = await doctor(target, root, planned.files);
371
+ if (!report_.ok) throw new Error(`${target.language}: ${report_.message}\n${report_.instructions}`);
292
372
  const generated = new Set(planned.files.filter((f) => f.ownership === "generated").map((f) => f.path));
293
373
  const keys = lawKeys({
294
374
  build, target, machineBits: input.machineBits, minify: options.minify === true,
295
375
  tests: planned.tests, files: planned.files,
296
- environment: await environmentDigest(root, report),
376
+ // Recorded values are spec data: a changed recording runs its laws again.
377
+ environment: await environmentDigest(root, report_) + (await recordedDigest(recordedFolder)),
297
378
  project: await projectDigest(root, generated),
298
379
  });
299
- const resultsFile = path.join(configRoot, ".lawspec", "results",
300
- `${target.language}-${createHash("sha256").update(root).digest("hex").slice(0, 12)}.json`);
380
+ const identity = projectKey(root);
381
+ const resultsFile = path.join(configRoot, ".lawspec", "results", `${target.language}-${identity}.json`);
301
382
  const previous = JSON.parse((await readOptional(resultsFile)) ?? '{"version":1,"laws":{}}');
302
- const stale = planned.tests.filter((entry) => options.fresh || previous.laws[entry.law]?.key !== keys.get(entry.law));
383
+ const {directory: failures, file: databaseFile, database} = await failureDatabase(configRoot, target.language, root);
384
+ // Recording again runs every law, so each records what it sees now.
385
+ const chosen = selectByTags(planned.tests, options.tag ?? [], options["exclude-tag"] ?? []);
386
+ const stale = testsToRun(chosen, keys, previous.laws, database,
387
+ {fresh: options.fresh, coverage: options.coverage, updateRecorded: options["update-recorded"]});
388
+ // The failure database: each law whose last run failed, with the seed
389
+ // that exposed it. Those laws run first, with that seed, so the failing
390
+ // inputs are generated again; the runtimes keep their own counterexample
391
+ // databases beside it (Hypothesis, proptest).
392
+ const replayed = options.seed === undefined ? stale.filter((entry) => database[entry.law]) : [];
393
+ const batches = [
394
+ ...[...new Set(replayed.map((entry) => database[entry.law].seed))].map((s) =>
395
+ ({ seed: String(s), entries: replayed.filter((entry) => database[entry.law].seed === s) })),
396
+ { seed, entries: stale.filter((entry) => !replayed.includes(entry)) },
397
+ ].filter((batch) => batch.entries.length);
303
398
  const passed = [];
304
399
  const unrun = [];
305
400
  let failed = false;
306
- const scratch = path.join(configRoot, ".lawspec", "reports", target.language);
401
+ const scratch = path.join(configRoot, ".lawspec", "reports", target.language, identity);
402
+ const stats = path.join(scratch, "statistics");
403
+ const beam = ["erlang", "elixir", "gleam"].includes(target.language);
404
+ const coverage = options.coverage ? path.join(configRoot, ".lawspec", "coverage", target.language, identity) : null;
405
+ const coverageRuns = [];
406
+ let coverageWritten = false, coverageError;
307
407
  await rm(scratch, { recursive: true, force: true });
308
- await mkdir(scratch, { recursive: true });
309
- await writeFile(path.join(path.dirname(scratch), ".gitignore"), "*\n");
310
- for (const run of stale.length ? invocations(target, stale, { offline, scratch }) : []) {
311
- const since = Date.now() - 1000;
312
- const { ok, output } = await spawned(run.command, run.args, root,
313
- { ...process.env, ...run.env, LAWSPEC_SEED: seed, HSPEC_SEED: seed }, run.report?.kind === "go-json");
314
- if (!ok) { failed = true; break; }
315
- // A runner whose filter matched nothing reports success, so a law
316
- // counts as passed only if the runner's report shows its tests ran.
317
- const executed = new Set((await executedTests(run.report, output, root, since)).flatMap(run.ran ?? (() => [])));
318
- passed.push(...run.laws.filter((law) => executed.has(law)));
319
- unrun.push(...run.laws.filter((law) => !executed.has(law)));
408
+ await mkdir(stats, { recursive: true });
409
+ await mkdir(failures, { recursive: true });
410
+ await writeFile(path.join(configRoot, ".lawspec", "reports", ".gitignore"), "*\n");
411
+ const missingCoverage = coverage ? await coverageMissing(target, root, report_) : null;
412
+ const useCoverage = coverage && !missingCoverage;
413
+ if (missingCoverage)
414
+ console.error(`${target.language}: --coverage needs ${missingCoverage.tool}, which is not available; ${missingCoverage.install}. Running without coverage.`);
415
+ if (useCoverage) {
416
+ if (beam) await rm(coverage, {recursive: true, force: true});
417
+ await mkdir(beam ? path.join(coverage, "runs") : coverage, { recursive: true });
418
+ }
419
+ // parallel on Python needs pytest-xdist; without it the tests run one
420
+ // after another, and lawspec test says so.
421
+ let xdist = false;
422
+ if (target.language === "python" && stale.some((entry) => entry.parallel)) {
423
+ xdist = spawnSync(target.python || "python3", ["-c", "import xdist"], { cwd: root, stdio: "ignore" }).status === 0;
424
+ if (!xdist) console.error("python: `parallel` runs tests at the same time only with pytest-xdist (pip install pytest-xdist); without it they run one after another.");
425
+ }
426
+ // A skipped law runs nothing. BEAM known-failing checks finish as native
427
+ // passing tests, so their completion must appear in the native report.
428
+ const expected = (entry) => !entry.skip && (!entry.knownFailing || ["erlang", "elixir", "gleam"].includes(target.language));
429
+ const benchmarking = options.benchmarks ? (planned.benchmarks ?? []) : [];
430
+ if (batches.length || benchmarking.length) {
431
+ for (const preparation of preparationInvocations(target, planned.files)) {
432
+ const {ok} = await spawned(preparation.command, preparation.args, root, process.env);
433
+ if (!ok) throw new Error(`${target.language}: native test preparation failed`);
434
+ }
435
+ }
436
+ const seedOf = new Map();
437
+ batches: for (const batch of batches) {
438
+ for (const run of invocations(target, batch.entries, { offline, scratch, coverage: useCoverage ? coverage : null,
439
+ xdist, root, coverageConfig: report_.coverage })) {
440
+ const since = Date.now() - 1000;
441
+ const { ok, output } = await spawned(run.command, run.args, root,
442
+ { ...process.env, ...run.env, LAWSPEC_SEED: batch.seed, HSPEC_SEED: batch.seed,
443
+ LAWSPEC_STATS: stats, LAWSPEC_FAILURES: failures, LAWSPEC_RECORDED: recordedFolder,
444
+ ...(options["update-recorded"] ? { LAWSPEC_UPDATE_RECORDED: "1" } : {}) }, run.report?.kind === "go-json");
445
+ if (run.coverage) coverageRuns.push(run.coverage);
446
+ const executed = await executedTests(run.report, output, root, since);
447
+ for (const law of run.laws) seedOf.set(law.law, batch.seed);
448
+ if (report) junit.push({ target: `${target.language} (${target.root})`, xml: await junitOf(run, executed, root, since) });
449
+ if (!ok || executed.some((test) => test.status === "failed")) { failed = true; break batches; }
450
+ // A runner whose filter matched nothing reports success, so a law
451
+ // counts as passed only if the runner's report shows its tests ran.
452
+ const ran = new Set(executed.flatMap(run.ran ?? (() => [])));
453
+ passed.push(...run.laws.filter((law) => ran.has(law) || !expected(law)));
454
+ unrun.push(...run.laws.filter((law) => !ran.has(law) && expected(law)));
455
+ }
320
456
  }
321
457
  if (unrun.length) failed = true;
458
+ if (useCoverage && beam) {
459
+ try {
460
+ coverageWritten = !!(await collectBeamCoverage({root, directory: coverage, runs: coverageRuns}));
461
+ } catch (error) {
462
+ failed = true;
463
+ coverageError = error.message;
464
+ }
465
+ }
466
+ // --benchmarks: the harness's benchmarks run after the laws, every time;
467
+ // they are measured, never asserted, and never cached.
468
+ if (!failed) for (const run of benchmarkInvocations(target, benchmarking, { offline })) {
469
+ const { ok } = await spawned(run.command, run.args, root,
470
+ { ...process.env, ...run.env, LAWSPEC_SEED: seed, HSPEC_SEED: seed, LAWSPEC_STATS: stats, LAWSPEC_RECORDED: recordedFolder }, false);
471
+ if (!ok) { failed = true; break; }
472
+ }
473
+ const statistics = await harnessStatistics(stats);
322
474
  const laws = Object.fromEntries(planned.tests.filter((entry) => previous.laws[entry.law]).map((entry) => [entry.law, previous.laws[entry.law]]));
323
- for (const entry of passed) laws[entry.law] = { key: keys.get(entry.law), seed: Number(seed), passed: new Date().toISOString() };
475
+ for (const entry of passed) {
476
+ const label = entry.label;
477
+ const observed = statistics.filter((s) => s.law === label);
478
+ laws[entry.law] = { key: keys.get(entry.law), seed: Number(seedOf.get(entry.law) ?? seed), passed: new Date().toISOString(),
479
+ ...(observed.some((s) => s.outcome === "flaky") ? { flaky: true } : {}),
480
+ ...(observed.some((s) => s.cover || s.labels || s.classes) ? { adequacy: observed.filter((s) => s.cover || s.labels || s.classes)
481
+ .map(({ test, cases, cover, classes, labels }) => ({ test, cases, cover, classes, labels })) } : {}) };
482
+ }
483
+ // A law that failed is recorded with its seed; one that passed leaves.
484
+ for (const entry of stale) {
485
+ if (passed.includes(entry)) delete database[entry.law];
486
+ else if (seedOf.has(entry.law)) {
487
+ delete laws[entry.law];
488
+ database[entry.law] = { seed: Number(seedOf.get(entry.law)), failed: new Date().toISOString() };
489
+ }
490
+ }
491
+ await writeFile(path.join(configRoot, ".lawspec", "failures", ".gitignore"), "*\n");
492
+ await writeFile(databaseFile, JSON.stringify(database, null, 2) + "\n");
324
493
  await mkdir(path.dirname(resultsFile), { recursive: true });
325
494
  await writeFile(path.join(path.dirname(resultsFile), ".gitignore"), "*\n");
326
495
  await writeFile(resultsFile, JSON.stringify({ version: 1, laws }, null, 2) + "\n");
327
- summaries.push({ target: target.language, seed: Number(seed), ran: stale.map((e) => e.law),
328
- unchanged: planned.tests.length - stale.length, ok: !failed,
329
- ...(unrun.length ? { unrun: unrun.map((e) => e.law) } : {}) });
496
+ const flaky = statistics.filter((s) => s.outcome === "flaky").map((s) => s.law);
497
+ const unmet = statistics.flatMap((s) => (s.cover ?? []).filter((c) => !c.met).map((c) => `${s.law}: cover ${c.required}% "${c.label}" (${c.observed}%)`));
498
+ const benchmarks = statistics.filter((s) => s.benchmark);
499
+ // How each parallel unit's tests actually ran at the same time.
500
+ const parallelism = statistics.filter((s) => s.parallel).map(({ parallel, mode, workers }) => ({ unit: parallel, mode, workers }));
501
+ summaries.push({ target: target.language, root: target.root, seed: Number(seed), ran: stale.map((e) => e.law),
502
+ unchanged: chosen.length - stale.length, ok: !failed,
503
+ ...(chosen.length !== planned.tests.length ? { deselected: planned.tests.length - chosen.length } : {}),
504
+ ...(unrun.length ? { unrun: unrun.map((e) => e.law) } : {}),
505
+ ...(flaky.length ? { flaky: [...new Set(flaky)] } : {}),
506
+ ...(unmet.length ? { unmetCover: unmet } : {}),
507
+ ...(benchmarks.length ? { benchmarks } : {}),
508
+ ...(parallelism.length ? { parallelism } : {}),
509
+ ...(coverageError ? {coverageError} : {}),
510
+ ...(useCoverage && (!beam || coverageWritten) ? { coverage: path.relative(process.cwd(), coverage) || "." } : {}) });
511
+ }
512
+ if (report) {
513
+ await mkdir(path.dirname(path.resolve(report.path)), { recursive: true });
514
+ await writeFile(path.resolve(report.path), mergeJunit(junit));
330
515
  }
331
516
  output(options.json ? summaries : summaries.map((s) =>
332
- `${s.target}: ${s.ran.length ? `ran ${s.ran.length} law(s) with seed ${s.seed}` : "nothing to run"}` +
517
+ `${s.target}${summaries.filter(other => other.target === s.target).length > 1 ? ` (${s.root})` : ""}: ${s.ran.length ? `ran ${s.ran.length} law(s) with seed ${s.seed}` : "nothing to run"}` +
333
518
  `, ${s.unchanged} unchanged since their last passing run.${s.ok ? "" : " FAILED"}` +
334
- (s.unrun ? `\nNo tests ran for ${s.unrun.join(", ")}; the runner matched none of their tests.` : "")).join("\n"));
519
+ (s.deselected ? `\n${s.deselected} law(s) not selected by --tag or --exclude-tag.` : "") +
520
+ (s.unrun ? `\nNo tests ran for ${s.unrun.join(", ")}; the runner matched none of their tests.` : "") +
521
+ (s.flaky ? `\nFlaky (failed, then passed on a retry): ${s.flaky.join(", ")}` : "") +
522
+ (s.unmetCover ? `\nCover not met: ${s.unmetCover.join("; ")}` : "") +
523
+ (s.parallelism ? "\n" + s.parallelism.map((p) => `parallel ${p.unit}: ${p.mode}, ${p.workers} worker(s)`).join("\n") : "") +
524
+ (s.benchmarks ? "\n" + s.benchmarks.map((b) => `benchmark ${b.benchmark}: mean ${(b.mean_ns / 1000).toFixed(2)} us over ${b.iterations} iteration(s)`).join("\n") : "") +
525
+ (s.coverageError ? `\nCoverage failed: ${s.coverageError}` : "") +
526
+ (s.coverage ? `\nCoverage written to ${s.coverage}.` : "")).join("\n"));
335
527
  if (summaries.some((s) => !s.ok)) process.exitCode = 1;
336
528
  }
529
+ // --report junit=path: one JUnit report merged across targets.
530
+ function parseReport(value) {
531
+ const match = value.match(/^junit=(.+)$/);
532
+ if (!match) throw new Error("--report takes junit=<path>");
533
+ return { kind: "junit", path: match[1] };
534
+ }
535
+ // A run's JUnit XML: the runner's own where it writes one, otherwise made
536
+ // from what it printed.
537
+ async function junitOf(run, executed, root, since) {
538
+ if (run.report?.kind === "junit") {
539
+ const files = run.report.files ?? (await readdir(path.join(root, run.report.directory)).catch(() => []))
540
+ .filter((n) => n.endsWith(".xml")).map((n) => path.join(root, run.report.directory, n));
541
+ const xml = [];
542
+ for (const file of files) {
543
+ const info = await stat(path.resolve(root, file)).catch(() => null);
544
+ if (info && info.mtimeMs >= since) xml.push(await readFile(path.resolve(root, file), "utf8"));
545
+ }
546
+ return xml.join("\n");
547
+ }
548
+ // BEAM reports only executed cases. Intentional harness skips remain
549
+ // reportable obligations; they are never evidence that a test passed.
550
+ const skipped = run.report?.kind === "beam-events" ? run.laws.filter((entry) => entry.skip)
551
+ .map((entry) => ({name: entry.label, classname: entry.unit, status: "skipped", reason: entry.skip})) : [];
552
+ return junitFromTests([...executed, ...skipped]);
553
+ }
554
+ // The tool --coverage needs, if it is missing.
555
+ async function coverageMissing(target, root, report) {
556
+ const tool = coverageTools[target.language];
557
+ if (!tool) return { tool: "a coverage tool", install: "no coverage tool is known for this target" };
558
+ const probe = (command, args) => new Promise((resolve) => {
559
+ const child = spawn(command, args, { cwd: root, stdio: "ignore" });
560
+ child.on("error", () => resolve(false));
561
+ child.on("close", (code) => resolve(code === 0));
562
+ });
563
+ if (["erlang", "elixir", "gleam"].includes(target.language)) {
564
+ const requirement = beamCoverageRequirement(target, report);
565
+ if (requirement) return requirement;
566
+ return (await probe("erl", ["-noshell", "-eval",
567
+ 'case code:ensure_loaded(cover) of {module,cover} -> halt(0); _ -> halt(1) end.'])) ? null : tool;
568
+ }
569
+ if (target.language === "python") return (await probe(target.python || "python3", tool.check)) ? null : tool;
570
+ if (["javascript", "typescript"].includes(target.language)) return (await probe("npx", ["--no-install", "c8", "--version"])) ? null : tool;
571
+ if (target.language === "rust") return (await probe("cargo", ["llvm-cov", "--version"])) ? null : tool;
572
+ if (target.language === "kotlin") {
573
+ const build = await readFile(path.join(root, "build.gradle.kts"), "utf8").catch(() => "");
574
+ return build.includes("kover") ? null : tool;
575
+ }
576
+ return null;
577
+ }
337
578
  // Run a native test command, showing its output (on stderr with --json) and
338
579
  // keeping it. Go's JSON events are shown as the text they carry.
339
580
  function spawned(command, args, cwd, env, goJson = false) {
@@ -374,7 +615,7 @@ async function main() {
374
615
  }
375
616
  if (!verb || ["help", "--help", "-h"].includes(verb)) {
376
617
  output(
377
- "LawSpec " + VERSION + "\nUsage: lawspec init --target <language> [--project <directory>] [--minify]\n lawspec check | doctor | explain <unit>::<law> | generate\n lawspec evidence [<unit> | <unit>::<declaration>]\n lawspec test [--target <language>] [--fresh] [--seed <n>]\n lawspec package [--project <package directory>]\n lawspec examples [--example payments] [--target <language>] [--output <directory>]\nOptions: --config <path>, --target <language>, --machine-bits <32|64>, --json, --no-cache\nGeneration: --dry-run, --check, --minify\nTargets: " +
618
+ "LawSpec " + VERSION + "\nUsage: lawspec init --target <language> [--project <directory>] [--minify]\n lawspec check | doctor | explain <unit>::<law> | generate\n lawspec evidence [<unit> | <unit>::<declaration>]\n lawspec test [--target <language>] [--fresh] [--seed <n>] [--update-recorded] [--tag <t>] [--exclude-tag <t>] [--report junit=<path>] [--coverage] [--benchmarks]\n lawspec package [--project <package directory>]\n lawspec examples [--example payments] [--target <language>] [--output <directory>]\nOptions: --config <path>, --target <language>, --machine-bits <32|64>, --json, --no-cache\nGeneration: --dry-run, --check, --minify\nTargets: " +
378
619
  targets.join(", "),
379
620
  );
380
621
  return;
@@ -418,8 +659,9 @@ async function main() {
418
659
  if (options.example) throw new Error("--example is only supported by examples");
419
660
  if (options.minify && !["init", "generate", "examples", "test"].includes(verb))
420
661
  throw new Error("--minify applies to init, generate, test and examples");
421
- if ((options.fresh || options.seed !== undefined) && verb !== "test")
422
- throw new Error("--fresh and --seed apply to test");
662
+ if ((options.fresh || options.seed !== undefined || options.tag || options["exclude-tag"] ||
663
+ options.report !== undefined || options.coverage || options["update-recorded"] || options.benchmarks) && verb !== "test")
664
+ throw new Error("--fresh, --seed, --update-recorded, --tag, --exclude-tag, --report, --coverage and --benchmarks apply to test");
423
665
  if (options.seed !== undefined && !/^[0-9]+$/.test(options.seed))
424
666
  throw new Error("--seed must be a whole number");
425
667
  if (verb === "init") return init();
@@ -476,13 +718,24 @@ async function main() {
476
718
  ["proved", "PROVED"],
477
719
  ["exhaustively-checked", "EXHAUSTIVELY CHECKED"],
478
720
  ["property-tested", "PROPERTY TESTED"],
721
+ ["measured", "MEASURED"],
479
722
  ["runtime-checked", "RUNTIME CHECKED"],
723
+ ["default-handler", "DEFAULT HANDLER"],
480
724
  ["assumed", "ASSUMED / EXTERNAL"],
725
+ // The harness plane: a law it marks as known to fail, a law whose last
726
+ // run was flaky, and a law whose tests it skips.
727
+ ["known-failing", "KNOWN FAILING"],
728
+ ["flaky", "FLAKY"],
729
+ ["skipped", "SKIPPED"],
481
730
  ];
482
731
  const evidenceSummary = (evidence) => {
483
732
  if (!evidence.length) return "";
484
733
  const count = (status) => evidence.filter((item) => item.status === status).length;
485
- return ` Evidence: ${statuses.map(([status, label]) => `${count(status)} ${label.toLowerCase()}`).join(", ")}.`;
734
+ // The harness statuses (after assumed) are counted only when a harness
735
+ // gives some.
736
+ const harnessFrom = statuses.findIndex(([status]) => status === "known-failing");
737
+ const shown = statuses.filter(([status], i) => i < harnessFrom || count(status));
738
+ return ` Evidence: ${shown.map(([status, label]) => `${count(status)} ${label.toLowerCase()}`).join(", ")}.`;
486
739
  };
487
740
  const obligationName = (item) => item.declaration.replace("::law::", "::");
488
741
  if (verb === "check") {
@@ -512,10 +765,21 @@ async function main() {
512
765
  evidence = evidence.filter((item) => !positional[0] ||
513
766
  obligationName(item) === positional[0] || item.owner === positional[0]);
514
767
  if (positional[0] && !evidence.length) throw new Error("No matching obligation");
768
+ // The last run's outcome (lawspec test): a law that failed, then passed
769
+ // on a retry, is flaky; what its generated cases covered is its adequacy.
770
+ const runs = await lastRuns();
771
+ evidence = evidence.map((item) => {
772
+ const run = item.stage === "law" ? runs.get(item.declaration) : undefined;
773
+ if (!run) return item;
774
+ return { ...item, ...(run.adequacy ? { adequacy: run.adequacy } : {}),
775
+ ...(run.flaky && ["property-tested", "exhaustively-checked"].includes(item.status)
776
+ ? { status: "flaky", reason: `${item.reason}; its last run failed, then passed on a retry` } : {}) };
777
+ });
778
+ const harnessed = evidence.filter((item) => item.harness || item.adequacy);
515
779
  output(
516
780
  options.json
517
781
  ? evidence
518
- : statuses
782
+ : [...statuses
519
783
  .map(([status, label]) => {
520
784
  const items = evidence.filter((item) => item.status === status);
521
785
  if (!items.length) return null;
@@ -523,7 +787,10 @@ async function main() {
523
787
  ` ${item.stage} ${obligationName(item)}${item.target ? ` [${item.target}]` : ""}` +
524
788
  (item.claim ? `: ${item.claim.text}` : "") + `\n ${item.reason}`).join("\n");
525
789
  })
526
- .filter((section) => section !== null)
790
+ .filter((section) => section !== null),
791
+ // How each law is discharged: the harness plane, apart from what
792
+ // the laws say.
793
+ ...(harnessed.length ? [`HARNESS (${harnessed.length})\n` + harnessed.map(harnessText).join("\n")] : [])]
527
794
  .join("\n\n"),
528
795
  );
529
796
  return;
@@ -542,7 +809,8 @@ async function main() {
542
809
  : indices
543
810
  .map(
544
811
  ({ e, i }) =>
545
- `${e.owner}::${e.name}\n${e.trace.join("\n=> ")}\n=> ${result.expansions[i]}${explainExamples(e)}`,
812
+ `${e.owner}::${e.name}\n${e.trace.join("\n=> ")}\n=> ${result.expansions[i]}` +
813
+ explainAbilities(e, result.expansions[i], result.units) + explainExamples(e),
546
814
  )
547
815
  .join("\n\n"),
548
816
  );