lawspec 0.21.0 → 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/bin/lawspec.mjs CHANGED
@@ -8,9 +8,11 @@ import { targets, templates, commands, setup } from "../templates.mjs";
8
8
  import { generateExamples } from "../examples-command.mjs";
9
9
  import { showScalar } from "../scalars.mjs";
10
10
  import { doctor } from "../doctor.mjs";
11
+ import { beamCoverageRequirement, collectBeamCoverage } from "../beam-coverage.mjs";
12
+ import { failureDatabase, projectKey } from "../failure-database.mjs";
11
13
  import { spawn, spawnSync } from "node:child_process";
12
- import { benchmarkInvocations, environmentDigest, executedTests, invocations, lawKeys, projectDigest, recordedDigest, selectByTags, mergeJunit,
13
- harnessStatistics, coverageTools, junitFromTests } from "../test-command.mjs";
14
+ import { benchmarkInvocations, preparationInvocations, environmentDigest, executedTests, invocations, lawKeys, projectDigest, recordedDigest, selectByTags, mergeJunit,
15
+ harnessStatistics, coverageTools, junitFromTests, testsToRun } from "../test-command.mjs";
14
16
  import {
15
17
  readOptional,
16
18
  planWrites,
@@ -18,7 +20,7 @@ import {
18
20
  atomicWrite,
19
21
  safePath,
20
22
  } from "../files.mjs";
21
- const VERSION = "0.21.0";
23
+ const VERSION = "0.22.0";
22
24
  const args = process.argv.slice(2);
23
25
  const verb = args.shift();
24
26
  const options = {};
@@ -197,6 +199,9 @@ async function init() {
197
199
  "build.gradle",
198
200
  "build.gradle.kts",
199
201
  "settings.gradle.kts",
202
+ "rebar.config",
203
+ "mix.exs",
204
+ "gleam.toml",
200
205
  ];
201
206
  const hasBuild = (await readdir(root)).some(
202
207
  (f) => buildFiles.includes(f) || f.endsWith(".cabal"),
@@ -372,19 +377,18 @@ async function runTests(compiler, input, selected, roots, config) {
372
377
  environment: await environmentDigest(root, report_) + (await recordedDigest(recordedFolder)),
373
378
  project: await projectDigest(root, generated),
374
379
  });
375
- const resultsFile = path.join(configRoot, ".lawspec", "results",
376
- `${target.language}-${createHash("sha256").update(root).digest("hex").slice(0, 12)}.json`);
380
+ const identity = projectKey(root);
381
+ const resultsFile = path.join(configRoot, ".lawspec", "results", `${target.language}-${identity}.json`);
377
382
  const previous = JSON.parse((await readOptional(resultsFile)) ?? '{"version":1,"laws":{}}');
383
+ const {directory: failures, file: databaseFile, database} = await failureDatabase(configRoot, target.language, root);
378
384
  // Recording again runs every law, so each records what it sees now.
379
385
  const chosen = selectByTags(planned.tests, options.tag ?? [], options["exclude-tag"] ?? []);
380
- const stale = chosen.filter((entry) => options.fresh || options.coverage || options["update-recorded"] || previous.laws[entry.law]?.key !== keys.get(entry.law));
386
+ const stale = testsToRun(chosen, keys, previous.laws, database,
387
+ {fresh: options.fresh, coverage: options.coverage, updateRecorded: options["update-recorded"]});
381
388
  // The failure database: each law whose last run failed, with the seed
382
389
  // that exposed it. Those laws run first, with that seed, so the failing
383
390
  // inputs are generated again; the runtimes keep their own counterexample
384
391
  // databases beside it (Hypothesis, proptest).
385
- const failures = path.join(configRoot, ".lawspec", "failures", target.language);
386
- const databaseFile = path.join(failures, "laws.json");
387
- const database = JSON.parse((await readOptional(databaseFile)) ?? "{}");
388
392
  const replayed = options.seed === undefined ? stale.filter((entry) => database[entry.law]) : [];
389
393
  const batches = [
390
394
  ...[...new Set(replayed.map((entry) => database[entry.law].seed))].map((s) =>
@@ -394,19 +398,24 @@ async function runTests(compiler, input, selected, roots, config) {
394
398
  const passed = [];
395
399
  const unrun = [];
396
400
  let failed = false;
397
- const scratch = path.join(configRoot, ".lawspec", "reports", target.language);
401
+ const scratch = path.join(configRoot, ".lawspec", "reports", target.language, identity);
398
402
  const stats = path.join(scratch, "statistics");
399
- const coverage = options.coverage ? path.join(configRoot, ".lawspec", "coverage", target.language) : null;
403
+ const beam = ["erlang", "elixir", "gleam"].includes(target.language);
404
+ const coverage = options.coverage ? path.join(configRoot, ".lawspec", "coverage", target.language, identity) : null;
405
+ const coverageRuns = [];
406
+ let coverageWritten = false, coverageError;
400
407
  await rm(scratch, { recursive: true, force: true });
401
408
  await mkdir(stats, { recursive: true });
402
409
  await mkdir(failures, { recursive: true });
403
- await writeFile(path.join(path.dirname(scratch), ".gitignore"), "*\n");
404
- if (coverage) {
405
- await mkdir(coverage, { recursive: true });
406
- const missing = await coverageMissing(target, root);
407
- if (missing) console.error(`${target.language}: --coverage needs ${missing.tool}, which is not available; ${missing.install}. Running without coverage.`);
410
+ await writeFile(path.join(configRoot, ".lawspec", "reports", ".gitignore"), "*\n");
411
+ const missingCoverage = coverage ? await coverageMissing(target, root, report_) : null;
412
+ const useCoverage = coverage && !missingCoverage;
413
+ if (missingCoverage)
414
+ console.error(`${target.language}: --coverage needs ${missingCoverage.tool}, which is not available; ${missingCoverage.install}. Running without coverage.`);
415
+ if (useCoverage) {
416
+ if (beam) await rm(coverage, {recursive: true, force: true});
417
+ await mkdir(beam ? path.join(coverage, "runs") : coverage, { recursive: true });
408
418
  }
409
- const useCoverage = coverage && !(await coverageMissing(target, root));
410
419
  // parallel on Python needs pytest-xdist; without it the tests run one
411
420
  // after another, and lawspec test says so.
412
421
  let xdist = false;
@@ -414,21 +423,30 @@ async function runTests(compiler, input, selected, roots, config) {
414
423
  xdist = spawnSync(target.python || "python3", ["-c", "import xdist"], { cwd: root, stdio: "ignore" }).status === 0;
415
424
  if (!xdist) console.error("python: `parallel` runs tests at the same time only with pytest-xdist (pip install pytest-xdist); without it they run one after another.");
416
425
  }
417
- // A skipped law runs nothing, and a known-failing law's one test is
418
- // expected to fail, so neither is required to show as run.
419
- const expected = (entry) => !entry.skip && !entry.knownFailing;
426
+ // A skipped law runs nothing. BEAM known-failing checks finish as native
427
+ // passing tests, so their completion must appear in the native report.
428
+ const expected = (entry) => !entry.skip && (!entry.knownFailing || ["erlang", "elixir", "gleam"].includes(target.language));
429
+ const benchmarking = options.benchmarks ? (planned.benchmarks ?? []) : [];
430
+ if (batches.length || benchmarking.length) {
431
+ for (const preparation of preparationInvocations(target, planned.files)) {
432
+ const {ok} = await spawned(preparation.command, preparation.args, root, process.env);
433
+ if (!ok) throw new Error(`${target.language}: native test preparation failed`);
434
+ }
435
+ }
420
436
  const seedOf = new Map();
421
437
  batches: for (const batch of batches) {
422
- for (const run of invocations(target, batch.entries, { offline, scratch, coverage: useCoverage ? coverage : null, xdist })) {
438
+ for (const run of invocations(target, batch.entries, { offline, scratch, coverage: useCoverage ? coverage : null,
439
+ xdist, root, coverageConfig: report_.coverage })) {
423
440
  const since = Date.now() - 1000;
424
441
  const { ok, output } = await spawned(run.command, run.args, root,
425
442
  { ...process.env, ...run.env, LAWSPEC_SEED: batch.seed, HSPEC_SEED: batch.seed,
426
443
  LAWSPEC_STATS: stats, LAWSPEC_FAILURES: failures, LAWSPEC_RECORDED: recordedFolder,
427
444
  ...(options["update-recorded"] ? { LAWSPEC_UPDATE_RECORDED: "1" } : {}) }, run.report?.kind === "go-json");
445
+ if (run.coverage) coverageRuns.push(run.coverage);
428
446
  const executed = await executedTests(run.report, output, root, since);
429
447
  for (const law of run.laws) seedOf.set(law.law, batch.seed);
430
- if (report) junit.push({ target: target.language, xml: await junitOf(run, executed, root, since) });
431
- if (!ok) { failed = true; break batches; }
448
+ if (report) junit.push({ target: `${target.language} (${target.root})`, xml: await junitOf(run, executed, root, since) });
449
+ if (!ok || executed.some((test) => test.status === "failed")) { failed = true; break batches; }
432
450
  // A runner whose filter matched nothing reports success, so a law
433
451
  // counts as passed only if the runner's report shows its tests ran.
434
452
  const ran = new Set(executed.flatMap(run.ran ?? (() => [])));
@@ -437,9 +455,16 @@ async function runTests(compiler, input, selected, roots, config) {
437
455
  }
438
456
  }
439
457
  if (unrun.length) failed = true;
458
+ if (useCoverage && beam) {
459
+ try {
460
+ coverageWritten = !!(await collectBeamCoverage({root, directory: coverage, runs: coverageRuns}));
461
+ } catch (error) {
462
+ failed = true;
463
+ coverageError = error.message;
464
+ }
465
+ }
440
466
  // --benchmarks: the harness's benchmarks run after the laws, every time;
441
467
  // they are measured, never asserted, and never cached.
442
- const benchmarking = options.benchmarks ? (planned.benchmarks ?? []) : [];
443
468
  if (!failed) for (const run of benchmarkInvocations(target, benchmarking, { offline })) {
444
469
  const { ok } = await spawned(run.command, run.args, root,
445
470
  { ...process.env, ...run.env, LAWSPEC_SEED: seed, HSPEC_SEED: seed, LAWSPEC_STATS: stats, LAWSPEC_RECORDED: recordedFolder }, false);
@@ -458,9 +483,12 @@ async function runTests(compiler, input, selected, roots, config) {
458
483
  // A law that failed is recorded with its seed; one that passed leaves.
459
484
  for (const entry of stale) {
460
485
  if (passed.includes(entry)) delete database[entry.law];
461
- else if (seedOf.has(entry.law)) database[entry.law] = { seed: Number(seedOf.get(entry.law)), failed: new Date().toISOString() };
486
+ else if (seedOf.has(entry.law)) {
487
+ delete laws[entry.law];
488
+ database[entry.law] = { seed: Number(seedOf.get(entry.law)), failed: new Date().toISOString() };
489
+ }
462
490
  }
463
- await writeFile(path.join(path.dirname(failures), ".gitignore"), "*\n");
491
+ await writeFile(path.join(configRoot, ".lawspec", "failures", ".gitignore"), "*\n");
464
492
  await writeFile(databaseFile, JSON.stringify(database, null, 2) + "\n");
465
493
  await mkdir(path.dirname(resultsFile), { recursive: true });
466
494
  await writeFile(path.join(path.dirname(resultsFile), ".gitignore"), "*\n");
@@ -470,7 +498,7 @@ async function runTests(compiler, input, selected, roots, config) {
470
498
  const benchmarks = statistics.filter((s) => s.benchmark);
471
499
  // How each parallel unit's tests actually ran at the same time.
472
500
  const parallelism = statistics.filter((s) => s.parallel).map(({ parallel, mode, workers }) => ({ unit: parallel, mode, workers }));
473
- summaries.push({ target: target.language, seed: Number(seed), ran: stale.map((e) => e.law),
501
+ summaries.push({ target: target.language, root: target.root, seed: Number(seed), ran: stale.map((e) => e.law),
474
502
  unchanged: chosen.length - stale.length, ok: !failed,
475
503
  ...(chosen.length !== planned.tests.length ? { deselected: planned.tests.length - chosen.length } : {}),
476
504
  ...(unrun.length ? { unrun: unrun.map((e) => e.law) } : {}),
@@ -478,14 +506,15 @@ async function runTests(compiler, input, selected, roots, config) {
478
506
  ...(unmet.length ? { unmetCover: unmet } : {}),
479
507
  ...(benchmarks.length ? { benchmarks } : {}),
480
508
  ...(parallelism.length ? { parallelism } : {}),
481
- ...(useCoverage ? { coverage: path.relative(process.cwd(), coverage) || "." } : {}) });
509
+ ...(coverageError ? {coverageError} : {}),
510
+ ...(useCoverage && (!beam || coverageWritten) ? { coverage: path.relative(process.cwd(), coverage) || "." } : {}) });
482
511
  }
483
512
  if (report) {
484
513
  await mkdir(path.dirname(path.resolve(report.path)), { recursive: true });
485
514
  await writeFile(path.resolve(report.path), mergeJunit(junit));
486
515
  }
487
516
  output(options.json ? summaries : summaries.map((s) =>
488
- `${s.target}: ${s.ran.length ? `ran ${s.ran.length} law(s) with seed ${s.seed}` : "nothing to run"}` +
517
+ `${s.target}${summaries.filter(other => other.target === s.target).length > 1 ? ` (${s.root})` : ""}: ${s.ran.length ? `ran ${s.ran.length} law(s) with seed ${s.seed}` : "nothing to run"}` +
489
518
  `, ${s.unchanged} unchanged since their last passing run.${s.ok ? "" : " FAILED"}` +
490
519
  (s.deselected ? `\n${s.deselected} law(s) not selected by --tag or --exclude-tag.` : "") +
491
520
  (s.unrun ? `\nNo tests ran for ${s.unrun.join(", ")}; the runner matched none of their tests.` : "") +
@@ -493,6 +522,7 @@ async function runTests(compiler, input, selected, roots, config) {
493
522
  (s.unmetCover ? `\nCover not met: ${s.unmetCover.join("; ")}` : "") +
494
523
  (s.parallelism ? "\n" + s.parallelism.map((p) => `parallel ${p.unit}: ${p.mode}, ${p.workers} worker(s)`).join("\n") : "") +
495
524
  (s.benchmarks ? "\n" + s.benchmarks.map((b) => `benchmark ${b.benchmark}: mean ${(b.mean_ns / 1000).toFixed(2)} us over ${b.iterations} iteration(s)`).join("\n") : "") +
525
+ (s.coverageError ? `\nCoverage failed: ${s.coverageError}` : "") +
496
526
  (s.coverage ? `\nCoverage written to ${s.coverage}.` : "")).join("\n"));
497
527
  if (summaries.some((s) => !s.ok)) process.exitCode = 1;
498
528
  }
@@ -515,10 +545,14 @@ async function junitOf(run, executed, root, since) {
515
545
  }
516
546
  return xml.join("\n");
517
547
  }
518
- return junitFromTests(executed);
548
+ // BEAM reports only executed cases. Intentional harness skips remain
549
+ // reportable obligations; they are never evidence that a test passed.
550
+ const skipped = run.report?.kind === "beam-events" ? run.laws.filter((entry) => entry.skip)
551
+ .map((entry) => ({name: entry.label, classname: entry.unit, status: "skipped", reason: entry.skip})) : [];
552
+ return junitFromTests([...executed, ...skipped]);
519
553
  }
520
554
  // The tool --coverage needs, if it is missing.
521
- async function coverageMissing(target, root) {
555
+ async function coverageMissing(target, root, report) {
522
556
  const tool = coverageTools[target.language];
523
557
  if (!tool) return { tool: "a coverage tool", install: "no coverage tool is known for this target" };
524
558
  const probe = (command, args) => new Promise((resolve) => {
@@ -526,6 +560,12 @@ async function coverageMissing(target, root) {
526
560
  child.on("error", () => resolve(false));
527
561
  child.on("close", (code) => resolve(code === 0));
528
562
  });
563
+ if (["erlang", "elixir", "gleam"].includes(target.language)) {
564
+ const requirement = beamCoverageRequirement(target, report);
565
+ if (requirement) return requirement;
566
+ return (await probe("erl", ["-noshell", "-eval",
567
+ 'case code:ensure_loaded(cover) of {module,cover} -> halt(0); _ -> halt(1) end.'])) ? null : tool;
568
+ }
529
569
  if (target.language === "python") return (await probe(target.python || "python3", tool.check)) ? null : tool;
530
570
  if (["javascript", "typescript"].includes(target.language)) return (await probe("npx", ["--no-install", "c8", "--version"])) ? null : tool;
531
571
  if (target.language === "rust") return (await probe("cargo", ["llvm-cov", "--version"])) ? null : tool;