lawspec 0.20.0 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -30,7 +30,7 @@ Node 22+ and the selected target's build tools are required. No Haskell
30
30
  toolchain is needed. The reference platforms are macOS and Linux.
31
31
 
32
32
  ```sh
33
- npm install --save-dev lawspec@0.20.0
33
+ npm install --save-dev lawspec@0.21.0
34
34
  npx lawspec --version
35
35
  ```
36
36
 
@@ -40,8 +40,8 @@ LawSpec can create `package.json` and its test script:
40
40
  ```sh
41
41
  mkdir lawspec-example
42
42
  cd lawspec-example
43
- npm exec --package=lawspec@0.20.0 -- lawspec init --target javascript
44
- npm install --save-dev lawspec@0.20.0
43
+ npm exec --package=lawspec@0.21.0 -- lawspec init --target javascript
44
+ npm install --save-dev lawspec@0.21.0
45
45
  npx lawspec check
46
46
  npx lawspec explain 'example.atoi_codec::itoa and then atoi yields a'
47
47
  npx lawspec doctor
@@ -113,7 +113,7 @@ See the [target guides](docs/how-to/targets/index.md).
113
113
  The [documentation](docs/index.md) has tutorials, how-to guides, a reference
114
114
  for the language, CLI, configuration and API, and explanations of the design.
115
115
  Release history is in the [changelog](CHANGELOG.md). To work on LawSpec itself,
116
- see [contributing](CONTRIBUTING.md).
116
+ see [contributing](docs/how-to/contribute.md).
117
117
 
118
118
  Syntax highlighting is available as a [VS Code extension](editors/vscode/README.md).
119
119
 
package/bin/lawspec.mjs CHANGED
@@ -8,8 +8,9 @@ import { targets, templates, commands, setup } from "../templates.mjs";
8
8
  import { generateExamples } from "../examples-command.mjs";
9
9
  import { showScalar } from "../scalars.mjs";
10
10
  import { doctor } from "../doctor.mjs";
11
- import { spawn } from "node:child_process";
12
- import { environmentDigest, executedTests, invocations, lawKeys, projectDigest } from "../test-command.mjs";
11
+ import { spawn, spawnSync } from "node:child_process";
12
+ import { benchmarkInvocations, environmentDigest, executedTests, invocations, lawKeys, projectDigest, recordedDigest, selectByTags, mergeJunit,
13
+ harnessStatistics, coverageTools, junitFromTests } from "../test-command.mjs";
13
14
  import {
14
15
  readOptional,
15
16
  planWrites,
@@ -17,18 +18,23 @@ import {
17
18
  atomicWrite,
18
19
  safePath,
19
20
  } from "../files.mjs";
20
- const VERSION = "0.20.0";
21
+ const VERSION = "0.21.0";
21
22
  const args = process.argv.slice(2);
22
23
  const verb = args.shift();
23
24
  const options = {};
24
25
  const positional = [];
25
26
  for (let i = 0; i < args.length; i++) {
26
27
  const arg = args[i];
27
- if (["--target", "--project", "--config", "--output", "--machine-bits", "--example", "--seed"].includes(arg)) {
28
+ if (["--target", "--project", "--config", "--output", "--machine-bits", "--example", "--seed", "--report"].includes(arg)) {
28
29
  if (!args[i + 1] || args[i + 1].startsWith("--"))
29
30
  throw new Error(`Missing value for ${arg}`);
30
31
  options[arg.slice(2)] = args[++i];
31
- } else if (["--dry-run", "--check", "--json", "--minify", "--no-cache", "--fresh"].includes(arg))
32
+ } else if (["--tag", "--exclude-tag"].includes(arg)) {
33
+ // Repeatable, and each may list several: --tag a,b --tag c.
34
+ if (!args[i + 1] || args[i + 1].startsWith("--"))
35
+ throw new Error(`Missing value for ${arg}`);
36
+ options[arg.slice(2)] = [...(options[arg.slice(2)] ?? []), ...args[++i].split(",").filter(Boolean)];
37
+ } else if (["--dry-run", "--check", "--json", "--minify", "--no-cache", "--fresh", "--coverage", "--update-recorded", "--benchmarks"].includes(arg))
32
38
  options[arg.slice(2)] = true;
33
39
  else if (arg.startsWith("--")) throw new Error(`Unknown option: ${arg}`);
34
40
  else positional.push(arg);
@@ -236,6 +242,24 @@ function showAssertion(assertion) {
236
242
  if (assertion.kind === "implies") return `${assertion.guard.text} implies ${showAssertion(assertion.body)}`;
237
243
  return assertion.items.map(showAssertion).join(" and ");
238
244
  }
245
+ // An ability key, as the spec names it: example.shop::ability::Store(Int32)
246
+ // is Store Int32, lawspec::ability::Fail(example.shop::type::E) is Fail E.
247
+ function abilityName(key) {
248
+ const short = (text) => text.replace(/[A-Za-z0-9_.]+::(ability|type)::/g, "");
249
+ const match = /^(.*?)(\((.*)\))?$/.exec(short(key));
250
+ return match[3] ? `${match[1]} ${match[3].replace(/[()]/g, " ").replace(/;/g, " ").trim()}` : match[1];
251
+ }
252
+ // The handlers a law runs under, and the ability row of each function its
253
+ // expansion calls.
254
+ function explainAbilities(law, expansion, units) {
255
+ const unit = (units || []).find((u) => u.id === law.owner);
256
+ const called = (unit ? unit.declarations : []).filter(
257
+ (d) => d.uses && new RegExp(`\\b${d.name}\\b`).test(expansion),
258
+ );
259
+ const handlers = (law.handlers || []).map((h) => `\n ${abilityName(h.ability)}: ${h.handler}`).join("");
260
+ const rows = called.map((d) => `\n ${d.name} uses ${d.uses.map(abilityName).join(", ")}`).join("");
261
+ return (handlers ? `\nhandlers${handlers}` : "") + (rows ? `\nrows${rows}` : "");
262
+ }
239
263
  function explainExamples(law) {
240
264
  return law.examples.map(ex =>
241
265
  `\nexample ${JSON.stringify(ex.name)}\n` +
@@ -243,6 +267,50 @@ function explainExamples(law) {
243
267
  ex.expectations.map(e => ` expect ${showAssertion(e)}`).join("\n")
244
268
  ).join("\n");
245
269
  }
270
+ // What lawspec test last recorded for each law, across targets.
271
+ async function lastRuns() {
272
+ const folder = path.join(configRoot, ".lawspec", "results");
273
+ const runs = new Map();
274
+ for (const name of (await readOptionalDirectory(folder)) ?? []) {
275
+ if (!name.endsWith(".json")) continue;
276
+ const recorded = JSON.parse((await readOptional(path.join(folder, name))) ?? "{}");
277
+ for (const [law, run] of Object.entries(recorded.laws ?? {})) {
278
+ const known = runs.get(law) ?? {};
279
+ runs.set(law, { flaky: known.flaky || run.flaky, adequacy: run.adequacy ?? known.adequacy });
280
+ }
281
+ }
282
+ return runs;
283
+ }
284
+ // One law's harness, as lawspec evidence shows it.
285
+ function harnessText(item) {
286
+ const h = item.harness ?? {};
287
+ const parts = [];
288
+ if (h.unit) parts.push(`harness ${h.unit}`);
289
+ if (h.tags?.length) parts.push(`tags ${h.tags.join(", ")}`);
290
+ if (h.strategies?.length) parts.push(h.strategies.map((s) => `${s.input} drawn by ${s.strategy}`).join(", "));
291
+ for (const c of h.cover ?? []) parts.push(`cover ${c.percent}% "${c.label}" when ${c.when.text}`);
292
+ for (const c of h.classify ?? []) parts.push(`classify ${c.when.text} as "${c.label}"`);
293
+ for (const l of h.labels ?? []) parts.push(`label ${l.text}`);
294
+ if (h.target) parts.push(`target maximize ${h.target.text}`);
295
+ if (h.timeoutMilliseconds) parts.push(`timeout ${h.timeoutMilliseconds} ms`);
296
+ if (h.repeat > 1) parts.push(`repeat ${h.repeat}`);
297
+ if (h.retries) parts.push(`retry flaky ${h.retries}`);
298
+ if (h.skip) parts.push(`skip "${h.skip}"`);
299
+ if (h.knownFailing) parts.push(`known failing "${h.knownFailing}"`);
300
+ if (h.orderRandom) parts.push("order random");
301
+ // Python runs a parallel unit's tests at the same time only with
302
+ // pytest-xdist; without it, parallel changes nothing there.
303
+ if (h.parallel) parts.push("parallel (on Python only with pytest-xdist; otherwise one after another)");
304
+ const lines = [` ${item.declaration.replace("::law::", "::")}: ${parts.join("; ") || "no harness settings"}`];
305
+ for (const run of item.adequacy ?? []) {
306
+ lines.push(` last run: ${run.cases} generated case(s)` +
307
+ (run.cover ?? []).map((c) => `; cover "${c.label}" ${c.observed}% (needs ${c.required}%)${c.met ? "" : " NOT MET"}`).join(""));
308
+ const counts = { ...(run.classes ?? {}), ...Object.fromEntries(Object.entries(run.labels ?? {}).map(([k, v]) => [`label ${k}`, v])) };
309
+ if (Object.keys(counts).length)
310
+ lines.push(` ${Object.entries(counts).map(([k, v]) => `${k}: ${run.cases ? (100 * v / run.cases).toFixed(1) : 0}%`).join(", ")}`);
311
+ }
312
+ return lines.join("\n");
313
+ }
246
314
  // The compiler keeps work between runs in .lawspec/cache, in one folder per
247
315
  // compiler build, so a different build never reads another's entries. The
248
316
  // WebAssembly compiler sees only the working directory. A command that fails
@@ -272,12 +340,19 @@ async function buildDigest() {
272
340
  return buildDigestValue;
273
341
  }
274
342
  // lawspec test: run the tests of the laws whose results may have changed,
275
- // and record each passing law's key and seed (see test-command.mjs).
343
+ // and record each passing law's key and seed (see test-command.mjs). The
344
+ // harness plane chooses which: --tag and --exclude-tag select laws by their
345
+ // harness tags, and a skipped law runs nothing. A law whose last run failed
346
+ // runs first, and its failing inputs are replayed first (.lawspec/failures).
276
347
  async function runTests(compiler, input, selected, roots, config) {
348
+ // Recorded values live beside lawspec.json, under recorded/<unit>/<name>.
349
+ const recordedFolder = path.join(configRoot, "recorded");
277
350
  const seed = options.seed ?? String(1 + Math.floor(Math.random() * 2147483646));
278
351
  const offline = process.env.LAWSPEC_OFFLINE === "1";
279
352
  const build = await buildDigest();
280
353
  const summaries = [];
354
+ const junit = [];
355
+ const report = options.report === undefined ? null : parseReport(options.report);
281
356
  for (const [i, target] of selected.entries()) {
282
357
  const root = roots[i];
283
358
  const planned = diagnostics(await compiler.planGeneration({
@@ -287,53 +362,179 @@ async function runTests(compiler, input, selected, roots, config) {
287
362
  const pending = await planWrites(root, planned.files);
288
363
  if (pending.changes.length)
289
364
  throw new Error(`${target.language}: generated files are out of date; run lawspec generate${options.minify ? " --minify" : ""}`);
290
- const report = await doctor(target, root, planned.files);
291
- if (!report.ok) throw new Error(`${target.language}: ${report.message}\n${report.instructions}`);
365
+ const report_ = await doctor(target, root, planned.files);
366
+ if (!report_.ok) throw new Error(`${target.language}: ${report_.message}\n${report_.instructions}`);
292
367
  const generated = new Set(planned.files.filter((f) => f.ownership === "generated").map((f) => f.path));
293
368
  const keys = lawKeys({
294
369
  build, target, machineBits: input.machineBits, minify: options.minify === true,
295
370
  tests: planned.tests, files: planned.files,
296
- environment: await environmentDigest(root, report),
371
+ // Recorded values are spec data: a changed recording runs its laws again.
372
+ environment: await environmentDigest(root, report_) + (await recordedDigest(recordedFolder)),
297
373
  project: await projectDigest(root, generated),
298
374
  });
299
375
  const resultsFile = path.join(configRoot, ".lawspec", "results",
300
376
  `${target.language}-${createHash("sha256").update(root).digest("hex").slice(0, 12)}.json`);
301
377
  const previous = JSON.parse((await readOptional(resultsFile)) ?? '{"version":1,"laws":{}}');
302
- const stale = planned.tests.filter((entry) => options.fresh || previous.laws[entry.law]?.key !== keys.get(entry.law));
378
+ // Recording again runs every law, so each records what it sees now.
379
+ const chosen = selectByTags(planned.tests, options.tag ?? [], options["exclude-tag"] ?? []);
380
+ const stale = chosen.filter((entry) => options.fresh || options.coverage || options["update-recorded"] || previous.laws[entry.law]?.key !== keys.get(entry.law));
381
+ // The failure database: each law whose last run failed, with the seed
382
+ // that exposed it. Those laws run first, with that seed, so the failing
383
+ // inputs are generated again; the runtimes keep their own counterexample
384
+ // databases beside it (Hypothesis, proptest).
385
+ const failures = path.join(configRoot, ".lawspec", "failures", target.language);
386
+ const databaseFile = path.join(failures, "laws.json");
387
+ const database = JSON.parse((await readOptional(databaseFile)) ?? "{}");
388
+ const replayed = options.seed === undefined ? stale.filter((entry) => database[entry.law]) : [];
389
+ const batches = [
390
+ ...[...new Set(replayed.map((entry) => database[entry.law].seed))].map((s) =>
391
+ ({ seed: String(s), entries: replayed.filter((entry) => database[entry.law].seed === s) })),
392
+ { seed, entries: stale.filter((entry) => !replayed.includes(entry)) },
393
+ ].filter((batch) => batch.entries.length);
303
394
  const passed = [];
304
395
  const unrun = [];
305
396
  let failed = false;
306
397
  const scratch = path.join(configRoot, ".lawspec", "reports", target.language);
398
+ const stats = path.join(scratch, "statistics");
399
+ const coverage = options.coverage ? path.join(configRoot, ".lawspec", "coverage", target.language) : null;
307
400
  await rm(scratch, { recursive: true, force: true });
308
- await mkdir(scratch, { recursive: true });
401
+ await mkdir(stats, { recursive: true });
402
+ await mkdir(failures, { recursive: true });
309
403
  await writeFile(path.join(path.dirname(scratch), ".gitignore"), "*\n");
310
- for (const run of stale.length ? invocations(target, stale, { offline, scratch }) : []) {
311
- const since = Date.now() - 1000;
312
- const { ok, output } = await spawned(run.command, run.args, root,
313
- { ...process.env, ...run.env, LAWSPEC_SEED: seed, HSPEC_SEED: seed }, run.report?.kind === "go-json");
314
- if (!ok) { failed = true; break; }
315
- // A runner whose filter matched nothing reports success, so a law
316
- // counts as passed only if the runner's report shows its tests ran.
317
- const executed = new Set((await executedTests(run.report, output, root, since)).flatMap(run.ran ?? (() => [])));
318
- passed.push(...run.laws.filter((law) => executed.has(law)));
319
- unrun.push(...run.laws.filter((law) => !executed.has(law)));
404
+ if (coverage) {
405
+ await mkdir(coverage, { recursive: true });
406
+ const missing = await coverageMissing(target, root);
407
+ if (missing) console.error(`${target.language}: --coverage needs ${missing.tool}, which is not available; ${missing.install}. Running without coverage.`);
408
+ }
409
+ const useCoverage = coverage && !(await coverageMissing(target, root));
410
+ // parallel on Python needs pytest-xdist; without it the tests run one
411
+ // after another, and lawspec test says so.
412
+ let xdist = false;
413
+ if (target.language === "python" && stale.some((entry) => entry.parallel)) {
414
+ xdist = spawnSync(target.python || "python3", ["-c", "import xdist"], { cwd: root, stdio: "ignore" }).status === 0;
415
+ if (!xdist) console.error("python: `parallel` runs tests at the same time only with pytest-xdist (pip install pytest-xdist); without it they run one after another.");
416
+ }
417
+ // A skipped law runs nothing, and a known-failing law's one test is
418
+ // expected to fail, so neither is required to show as run.
419
+ const expected = (entry) => !entry.skip && !entry.knownFailing;
420
+ const seedOf = new Map();
421
+ batches: for (const batch of batches) {
422
+ for (const run of invocations(target, batch.entries, { offline, scratch, coverage: useCoverage ? coverage : null, xdist })) {
423
+ const since = Date.now() - 1000;
424
+ const { ok, output } = await spawned(run.command, run.args, root,
425
+ { ...process.env, ...run.env, LAWSPEC_SEED: batch.seed, HSPEC_SEED: batch.seed,
426
+ LAWSPEC_STATS: stats, LAWSPEC_FAILURES: failures, LAWSPEC_RECORDED: recordedFolder,
427
+ ...(options["update-recorded"] ? { LAWSPEC_UPDATE_RECORDED: "1" } : {}) }, run.report?.kind === "go-json");
428
+ const executed = await executedTests(run.report, output, root, since);
429
+ for (const law of run.laws) seedOf.set(law.law, batch.seed);
430
+ if (report) junit.push({ target: target.language, xml: await junitOf(run, executed, root, since) });
431
+ if (!ok) { failed = true; break batches; }
432
+ // A runner whose filter matched nothing reports success, so a law
433
+ // counts as passed only if the runner's report shows its tests ran.
434
+ const ran = new Set(executed.flatMap(run.ran ?? (() => [])));
435
+ passed.push(...run.laws.filter((law) => ran.has(law) || !expected(law)));
436
+ unrun.push(...run.laws.filter((law) => !ran.has(law) && expected(law)));
437
+ }
320
438
  }
321
439
  if (unrun.length) failed = true;
440
+ // --benchmarks: the harness's benchmarks run after the laws, every time;
441
+ // they are measured, never asserted, and never cached.
442
+ const benchmarking = options.benchmarks ? (planned.benchmarks ?? []) : [];
443
+ if (!failed) for (const run of benchmarkInvocations(target, benchmarking, { offline })) {
444
+ const { ok } = await spawned(run.command, run.args, root,
445
+ { ...process.env, ...run.env, LAWSPEC_SEED: seed, HSPEC_SEED: seed, LAWSPEC_STATS: stats, LAWSPEC_RECORDED: recordedFolder }, false);
446
+ if (!ok) { failed = true; break; }
447
+ }
448
+ const statistics = await harnessStatistics(stats);
322
449
  const laws = Object.fromEntries(planned.tests.filter((entry) => previous.laws[entry.law]).map((entry) => [entry.law, previous.laws[entry.law]]));
323
- for (const entry of passed) laws[entry.law] = { key: keys.get(entry.law), seed: Number(seed), passed: new Date().toISOString() };
450
+ for (const entry of passed) {
451
+ const label = entry.label;
452
+ const observed = statistics.filter((s) => s.law === label);
453
+ laws[entry.law] = { key: keys.get(entry.law), seed: Number(seedOf.get(entry.law) ?? seed), passed: new Date().toISOString(),
454
+ ...(observed.some((s) => s.outcome === "flaky") ? { flaky: true } : {}),
455
+ ...(observed.some((s) => s.cover || s.labels || s.classes) ? { adequacy: observed.filter((s) => s.cover || s.labels || s.classes)
456
+ .map(({ test, cases, cover, classes, labels }) => ({ test, cases, cover, classes, labels })) } : {}) };
457
+ }
458
+ // A law that failed is recorded with its seed; one that passed leaves.
459
+ for (const entry of stale) {
460
+ if (passed.includes(entry)) delete database[entry.law];
461
+ else if (seedOf.has(entry.law)) database[entry.law] = { seed: Number(seedOf.get(entry.law)), failed: new Date().toISOString() };
462
+ }
463
+ await writeFile(path.join(path.dirname(failures), ".gitignore"), "*\n");
464
+ await writeFile(databaseFile, JSON.stringify(database, null, 2) + "\n");
324
465
  await mkdir(path.dirname(resultsFile), { recursive: true });
325
466
  await writeFile(path.join(path.dirname(resultsFile), ".gitignore"), "*\n");
326
467
  await writeFile(resultsFile, JSON.stringify({ version: 1, laws }, null, 2) + "\n");
468
+ const flaky = statistics.filter((s) => s.outcome === "flaky").map((s) => s.law);
469
+ const unmet = statistics.flatMap((s) => (s.cover ?? []).filter((c) => !c.met).map((c) => `${s.law}: cover ${c.required}% "${c.label}" (${c.observed}%)`));
470
+ const benchmarks = statistics.filter((s) => s.benchmark);
471
+ // How each parallel unit's tests actually ran at the same time.
472
+ const parallelism = statistics.filter((s) => s.parallel).map(({ parallel, mode, workers }) => ({ unit: parallel, mode, workers }));
327
473
  summaries.push({ target: target.language, seed: Number(seed), ran: stale.map((e) => e.law),
328
- unchanged: planned.tests.length - stale.length, ok: !failed,
329
- ...(unrun.length ? { unrun: unrun.map((e) => e.law) } : {}) });
474
+ unchanged: chosen.length - stale.length, ok: !failed,
475
+ ...(chosen.length !== planned.tests.length ? { deselected: planned.tests.length - chosen.length } : {}),
476
+ ...(unrun.length ? { unrun: unrun.map((e) => e.law) } : {}),
477
+ ...(flaky.length ? { flaky: [...new Set(flaky)] } : {}),
478
+ ...(unmet.length ? { unmetCover: unmet } : {}),
479
+ ...(benchmarks.length ? { benchmarks } : {}),
480
+ ...(parallelism.length ? { parallelism } : {}),
481
+ ...(useCoverage ? { coverage: path.relative(process.cwd(), coverage) || "." } : {}) });
482
+ }
483
+ if (report) {
484
+ await mkdir(path.dirname(path.resolve(report.path)), { recursive: true });
485
+ await writeFile(path.resolve(report.path), mergeJunit(junit));
330
486
  }
331
487
  output(options.json ? summaries : summaries.map((s) =>
332
488
  `${s.target}: ${s.ran.length ? `ran ${s.ran.length} law(s) with seed ${s.seed}` : "nothing to run"}` +
333
489
  `, ${s.unchanged} unchanged since their last passing run.${s.ok ? "" : " FAILED"}` +
334
- (s.unrun ? `\nNo tests ran for ${s.unrun.join(", ")}; the runner matched none of their tests.` : "")).join("\n"));
490
+ (s.deselected ? `\n${s.deselected} law(s) not selected by --tag or --exclude-tag.` : "") +
491
+ (s.unrun ? `\nNo tests ran for ${s.unrun.join(", ")}; the runner matched none of their tests.` : "") +
492
+ (s.flaky ? `\nFlaky (failed, then passed on a retry): ${s.flaky.join(", ")}` : "") +
493
+ (s.unmetCover ? `\nCover not met: ${s.unmetCover.join("; ")}` : "") +
494
+ (s.parallelism ? "\n" + s.parallelism.map((p) => `parallel ${p.unit}: ${p.mode}, ${p.workers} worker(s)`).join("\n") : "") +
495
+ (s.benchmarks ? "\n" + s.benchmarks.map((b) => `benchmark ${b.benchmark}: mean ${(b.mean_ns / 1000).toFixed(2)} us over ${b.iterations} iteration(s)`).join("\n") : "") +
496
+ (s.coverage ? `\nCoverage written to ${s.coverage}.` : "")).join("\n"));
335
497
  if (summaries.some((s) => !s.ok)) process.exitCode = 1;
336
498
  }
499
+ // --report junit=path: one JUnit report merged across targets.
500
+ function parseReport(value) {
501
+ const match = value.match(/^junit=(.+)$/);
502
+ if (!match) throw new Error("--report takes junit=<path>");
503
+ return { kind: "junit", path: match[1] };
504
+ }
505
+ // A run's JUnit XML: the runner's own where it writes one, otherwise made
506
+ // from what it printed.
507
+ async function junitOf(run, executed, root, since) {
508
+ if (run.report?.kind === "junit") {
509
+ const files = run.report.files ?? (await readdir(path.join(root, run.report.directory)).catch(() => []))
510
+ .filter((n) => n.endsWith(".xml")).map((n) => path.join(root, run.report.directory, n));
511
+ const xml = [];
512
+ for (const file of files) {
513
+ const info = await stat(path.resolve(root, file)).catch(() => null);
514
+ if (info && info.mtimeMs >= since) xml.push(await readFile(path.resolve(root, file), "utf8"));
515
+ }
516
+ return xml.join("\n");
517
+ }
518
+ return junitFromTests(executed);
519
+ }
520
+ // The tool --coverage needs, if it is missing.
521
+ async function coverageMissing(target, root) {
522
+ const tool = coverageTools[target.language];
523
+ if (!tool) return { tool: "a coverage tool", install: "no coverage tool is known for this target" };
524
+ const probe = (command, args) => new Promise((resolve) => {
525
+ const child = spawn(command, args, { cwd: root, stdio: "ignore" });
526
+ child.on("error", () => resolve(false));
527
+ child.on("close", (code) => resolve(code === 0));
528
+ });
529
+ if (target.language === "python") return (await probe(target.python || "python3", tool.check)) ? null : tool;
530
+ if (["javascript", "typescript"].includes(target.language)) return (await probe("npx", ["--no-install", "c8", "--version"])) ? null : tool;
531
+ if (target.language === "rust") return (await probe("cargo", ["llvm-cov", "--version"])) ? null : tool;
532
+ if (target.language === "kotlin") {
533
+ const build = await readFile(path.join(root, "build.gradle.kts"), "utf8").catch(() => "");
534
+ return build.includes("kover") ? null : tool;
535
+ }
536
+ return null;
537
+ }
337
538
  // Run a native test command, showing its output (on stderr with --json) and
338
539
  // keeping it. Go's JSON events are shown as the text they carry.
339
540
  function spawned(command, args, cwd, env, goJson = false) {
@@ -374,7 +575,7 @@ async function main() {
374
575
  }
375
576
  if (!verb || ["help", "--help", "-h"].includes(verb)) {
376
577
  output(
377
- "LawSpec " + VERSION + "\nUsage: lawspec init --target <language> [--project <directory>] [--minify]\n lawspec check | doctor | explain <unit>::<law> | generate\n lawspec evidence [<unit> | <unit>::<declaration>]\n lawspec test [--target <language>] [--fresh] [--seed <n>]\n lawspec package [--project <package directory>]\n lawspec examples [--example payments] [--target <language>] [--output <directory>]\nOptions: --config <path>, --target <language>, --machine-bits <32|64>, --json, --no-cache\nGeneration: --dry-run, --check, --minify\nTargets: " +
578
+ "LawSpec " + VERSION + "\nUsage: lawspec init --target <language> [--project <directory>] [--minify]\n lawspec check | doctor | explain <unit>::<law> | generate\n lawspec evidence [<unit> | <unit>::<declaration>]\n lawspec test [--target <language>] [--fresh] [--seed <n>] [--update-recorded] [--tag <t>] [--exclude-tag <t>] [--report junit=<path>] [--coverage] [--benchmarks]\n lawspec package [--project <package directory>]\n lawspec examples [--example payments] [--target <language>] [--output <directory>]\nOptions: --config <path>, --target <language>, --machine-bits <32|64>, --json, --no-cache\nGeneration: --dry-run, --check, --minify\nTargets: " +
378
579
  targets.join(", "),
379
580
  );
380
581
  return;
@@ -418,8 +619,9 @@ async function main() {
418
619
  if (options.example) throw new Error("--example is only supported by examples");
419
620
  if (options.minify && !["init", "generate", "examples", "test"].includes(verb))
420
621
  throw new Error("--minify applies to init, generate, test and examples");
421
- if ((options.fresh || options.seed !== undefined) && verb !== "test")
422
- throw new Error("--fresh and --seed apply to test");
622
+ if ((options.fresh || options.seed !== undefined || options.tag || options["exclude-tag"] ||
623
+ options.report !== undefined || options.coverage || options["update-recorded"] || options.benchmarks) && verb !== "test")
624
+ throw new Error("--fresh, --seed, --update-recorded, --tag, --exclude-tag, --report, --coverage and --benchmarks apply to test");
423
625
  if (options.seed !== undefined && !/^[0-9]+$/.test(options.seed))
424
626
  throw new Error("--seed must be a whole number");
425
627
  if (verb === "init") return init();
@@ -476,13 +678,24 @@ async function main() {
476
678
  ["proved", "PROVED"],
477
679
  ["exhaustively-checked", "EXHAUSTIVELY CHECKED"],
478
680
  ["property-tested", "PROPERTY TESTED"],
681
+ ["measured", "MEASURED"],
479
682
  ["runtime-checked", "RUNTIME CHECKED"],
683
+ ["default-handler", "DEFAULT HANDLER"],
480
684
  ["assumed", "ASSUMED / EXTERNAL"],
685
+ // The harness plane: a law it marks as known to fail, a law whose last
686
+ // run was flaky, and a law whose tests it skips.
687
+ ["known-failing", "KNOWN FAILING"],
688
+ ["flaky", "FLAKY"],
689
+ ["skipped", "SKIPPED"],
481
690
  ];
482
691
  const evidenceSummary = (evidence) => {
483
692
  if (!evidence.length) return "";
484
693
  const count = (status) => evidence.filter((item) => item.status === status).length;
485
- return ` Evidence: ${statuses.map(([status, label]) => `${count(status)} ${label.toLowerCase()}`).join(", ")}.`;
694
+ // The harness statuses (after assumed) are counted only when a harness
695
+ // gives some.
696
+ const harnessFrom = statuses.findIndex(([status]) => status === "known-failing");
697
+ const shown = statuses.filter(([status], i) => i < harnessFrom || count(status));
698
+ return ` Evidence: ${shown.map(([status, label]) => `${count(status)} ${label.toLowerCase()}`).join(", ")}.`;
486
699
  };
487
700
  const obligationName = (item) => item.declaration.replace("::law::", "::");
488
701
  if (verb === "check") {
@@ -512,10 +725,21 @@ async function main() {
512
725
  evidence = evidence.filter((item) => !positional[0] ||
513
726
  obligationName(item) === positional[0] || item.owner === positional[0]);
514
727
  if (positional[0] && !evidence.length) throw new Error("No matching obligation");
728
+ // The last run's outcome (lawspec test): a law that failed, then passed
729
+ // on a retry, is flaky; what its generated cases covered is its adequacy.
730
+ const runs = await lastRuns();
731
+ evidence = evidence.map((item) => {
732
+ const run = item.stage === "law" ? runs.get(item.declaration) : undefined;
733
+ if (!run) return item;
734
+ return { ...item, ...(run.adequacy ? { adequacy: run.adequacy } : {}),
735
+ ...(run.flaky && ["property-tested", "exhaustively-checked"].includes(item.status)
736
+ ? { status: "flaky", reason: `${item.reason}; its last run failed, then passed on a retry` } : {}) };
737
+ });
738
+ const harnessed = evidence.filter((item) => item.harness || item.adequacy);
515
739
  output(
516
740
  options.json
517
741
  ? evidence
518
- : statuses
742
+ : [...statuses
519
743
  .map(([status, label]) => {
520
744
  const items = evidence.filter((item) => item.status === status);
521
745
  if (!items.length) return null;
@@ -523,7 +747,10 @@ async function main() {
523
747
  ` ${item.stage} ${obligationName(item)}${item.target ? ` [${item.target}]` : ""}` +
524
748
  (item.claim ? `: ${item.claim.text}` : "") + `\n ${item.reason}`).join("\n");
525
749
  })
526
- .filter((section) => section !== null)
750
+ .filter((section) => section !== null),
751
+ // How each law is discharged: the harness plane, apart from what
752
+ // the laws say.
753
+ ...(harnessed.length ? [`HARNESS (${harnessed.length})\n` + harnessed.map(harnessText).join("\n")] : [])]
527
754
  .join("\n\n"),
528
755
  );
529
756
  return;
@@ -542,7 +769,8 @@ async function main() {
542
769
  : indices
543
770
  .map(
544
771
  ({ e, i }) =>
545
- `${e.owner}::${e.name}\n${e.trace.join("\n=> ")}\n=> ${result.expansions[i]}${explainExamples(e)}`,
772
+ `${e.owner}::${e.name}\n${e.trace.join("\n=> ")}\n=> ${result.expansions[i]}` +
773
+ explainAbilities(e, result.expansions[i], result.units) + explainExamples(e),
546
774
  )
547
775
  .join("\n\n"),
548
776
  );