lawspec 0.21.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +65 -1
- package/README.md +3 -3
- package/beam-coverage.mjs +160 -0
- package/beam-doctor.mjs +306 -0
- package/bin/lawspec.mjs +71 -31
- package/build.json +232 -38
- package/compatibility.json +17 -0
- package/core.wasm +2 -2
- package/doctor.mjs +7 -1
- package/examples/native-payments/elixir/lawspec.json +149 -0
- package/examples/native-payments/elixir/lib/payments_domain.ex +54 -0
- package/examples/native-payments/elixir/test/support/payment_generators.ex +9 -0
- package/examples/native-payments/erlang/include/payments_domain.hrl +2 -0
- package/examples/native-payments/erlang/lawspec.json +148 -0
- package/examples/native-payments/erlang/src/payments_domain.erl +21 -0
- package/examples/native-payments/erlang/test/payment_generators.erl +10 -0
- package/examples/native-payments/gleam/lawspec.json +149 -0
- package/examples/native-payments/gleam/prepare.mjs +48 -0
- package/examples/native-payments/gleam/src/payments_domain.gleam +32 -0
- package/examples/native-payments/gleam/test/payment_generators.gleam +13 -0
- package/failure-database.mjs +46 -0
- package/index.d.ts +10 -1
- package/native-examples.mjs +7 -2
- package/package.json +1 -1
- package/templates.mjs +92 -3
- package/test-command.mjs +161 -15
package/bin/lawspec.mjs
CHANGED
|
@@ -8,9 +8,11 @@ import { targets, templates, commands, setup } from "../templates.mjs";
|
|
|
8
8
|
import { generateExamples } from "../examples-command.mjs";
|
|
9
9
|
import { showScalar } from "../scalars.mjs";
|
|
10
10
|
import { doctor } from "../doctor.mjs";
|
|
11
|
+
import { beamCoverageRequirement, collectBeamCoverage } from "../beam-coverage.mjs";
|
|
12
|
+
import { failureDatabase, projectKey } from "../failure-database.mjs";
|
|
11
13
|
import { spawn, spawnSync } from "node:child_process";
|
|
12
|
-
import { benchmarkInvocations, environmentDigest, executedTests, invocations, lawKeys, projectDigest, recordedDigest, selectByTags, mergeJunit,
|
|
13
|
-
harnessStatistics, coverageTools, junitFromTests } from "../test-command.mjs";
|
|
14
|
+
import { benchmarkInvocations, preparationInvocations, environmentDigest, executedTests, invocations, lawKeys, projectDigest, recordedDigest, selectByTags, mergeJunit,
|
|
15
|
+
harnessStatistics, coverageTools, junitFromTests, testsToRun } from "../test-command.mjs";
|
|
14
16
|
import {
|
|
15
17
|
readOptional,
|
|
16
18
|
planWrites,
|
|
@@ -18,7 +20,7 @@ import {
|
|
|
18
20
|
atomicWrite,
|
|
19
21
|
safePath,
|
|
20
22
|
} from "../files.mjs";
|
|
21
|
-
const VERSION = "0.
|
|
23
|
+
const VERSION = "0.22.0";
|
|
22
24
|
const args = process.argv.slice(2);
|
|
23
25
|
const verb = args.shift();
|
|
24
26
|
const options = {};
|
|
@@ -197,6 +199,9 @@ async function init() {
|
|
|
197
199
|
"build.gradle",
|
|
198
200
|
"build.gradle.kts",
|
|
199
201
|
"settings.gradle.kts",
|
|
202
|
+
"rebar.config",
|
|
203
|
+
"mix.exs",
|
|
204
|
+
"gleam.toml",
|
|
200
205
|
];
|
|
201
206
|
const hasBuild = (await readdir(root)).some(
|
|
202
207
|
(f) => buildFiles.includes(f) || f.endsWith(".cabal"),
|
|
@@ -372,19 +377,18 @@ async function runTests(compiler, input, selected, roots, config) {
|
|
|
372
377
|
environment: await environmentDigest(root, report_) + (await recordedDigest(recordedFolder)),
|
|
373
378
|
project: await projectDigest(root, generated),
|
|
374
379
|
});
|
|
375
|
-
const
|
|
376
|
-
|
|
380
|
+
const identity = projectKey(root);
|
|
381
|
+
const resultsFile = path.join(configRoot, ".lawspec", "results", `${target.language}-${identity}.json`);
|
|
377
382
|
const previous = JSON.parse((await readOptional(resultsFile)) ?? '{"version":1,"laws":{}}');
|
|
383
|
+
const {directory: failures, file: databaseFile, database} = await failureDatabase(configRoot, target.language, root);
|
|
378
384
|
// Recording again runs every law, so each records what it sees now.
|
|
379
385
|
const chosen = selectByTags(planned.tests, options.tag ?? [], options["exclude-tag"] ?? []);
|
|
380
|
-
const stale = chosen
|
|
386
|
+
const stale = testsToRun(chosen, keys, previous.laws, database,
|
|
387
|
+
{fresh: options.fresh, coverage: options.coverage, updateRecorded: options["update-recorded"]});
|
|
381
388
|
// The failure database: each law whose last run failed, with the seed
|
|
382
389
|
// that exposed it. Those laws run first, with that seed, so the failing
|
|
383
390
|
// inputs are generated again; the runtimes keep their own counterexample
|
|
384
391
|
// databases beside it (Hypothesis, proptest).
|
|
385
|
-
const failures = path.join(configRoot, ".lawspec", "failures", target.language);
|
|
386
|
-
const databaseFile = path.join(failures, "laws.json");
|
|
387
|
-
const database = JSON.parse((await readOptional(databaseFile)) ?? "{}");
|
|
388
392
|
const replayed = options.seed === undefined ? stale.filter((entry) => database[entry.law]) : [];
|
|
389
393
|
const batches = [
|
|
390
394
|
...[...new Set(replayed.map((entry) => database[entry.law].seed))].map((s) =>
|
|
@@ -394,19 +398,24 @@ async function runTests(compiler, input, selected, roots, config) {
|
|
|
394
398
|
const passed = [];
|
|
395
399
|
const unrun = [];
|
|
396
400
|
let failed = false;
|
|
397
|
-
const scratch = path.join(configRoot, ".lawspec", "reports", target.language);
|
|
401
|
+
const scratch = path.join(configRoot, ".lawspec", "reports", target.language, identity);
|
|
398
402
|
const stats = path.join(scratch, "statistics");
|
|
399
|
-
const
|
|
403
|
+
const beam = ["erlang", "elixir", "gleam"].includes(target.language);
|
|
404
|
+
const coverage = options.coverage ? path.join(configRoot, ".lawspec", "coverage", target.language, identity) : null;
|
|
405
|
+
const coverageRuns = [];
|
|
406
|
+
let coverageWritten = false, coverageError;
|
|
400
407
|
await rm(scratch, { recursive: true, force: true });
|
|
401
408
|
await mkdir(stats, { recursive: true });
|
|
402
409
|
await mkdir(failures, { recursive: true });
|
|
403
|
-
await writeFile(path.join(
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
410
|
+
await writeFile(path.join(configRoot, ".lawspec", "reports", ".gitignore"), "*\n");
|
|
411
|
+
const missingCoverage = coverage ? await coverageMissing(target, root, report_) : null;
|
|
412
|
+
const useCoverage = coverage && !missingCoverage;
|
|
413
|
+
if (missingCoverage)
|
|
414
|
+
console.error(`${target.language}: --coverage needs ${missingCoverage.tool}, which is not available; ${missingCoverage.install}. Running without coverage.`);
|
|
415
|
+
if (useCoverage) {
|
|
416
|
+
if (beam) await rm(coverage, {recursive: true, force: true});
|
|
417
|
+
await mkdir(beam ? path.join(coverage, "runs") : coverage, { recursive: true });
|
|
408
418
|
}
|
|
409
|
-
const useCoverage = coverage && !(await coverageMissing(target, root));
|
|
410
419
|
// parallel on Python needs pytest-xdist; without it the tests run one
|
|
411
420
|
// after another, and lawspec test says so.
|
|
412
421
|
let xdist = false;
|
|
@@ -414,21 +423,30 @@ async function runTests(compiler, input, selected, roots, config) {
|
|
|
414
423
|
xdist = spawnSync(target.python || "python3", ["-c", "import xdist"], { cwd: root, stdio: "ignore" }).status === 0;
|
|
415
424
|
if (!xdist) console.error("python: `parallel` runs tests at the same time only with pytest-xdist (pip install pytest-xdist); without it they run one after another.");
|
|
416
425
|
}
|
|
417
|
-
// A skipped law runs nothing
|
|
418
|
-
//
|
|
419
|
-
const expected = (entry) => !entry.skip && !entry.knownFailing;
|
|
426
|
+
// A skipped law runs nothing. BEAM known-failing checks finish as native
|
|
427
|
+
// passing tests, so their completion must appear in the native report.
|
|
428
|
+
const expected = (entry) => !entry.skip && (!entry.knownFailing || ["erlang", "elixir", "gleam"].includes(target.language));
|
|
429
|
+
const benchmarking = options.benchmarks ? (planned.benchmarks ?? []) : [];
|
|
430
|
+
if (batches.length || benchmarking.length) {
|
|
431
|
+
for (const preparation of preparationInvocations(target, planned.files)) {
|
|
432
|
+
const {ok} = await spawned(preparation.command, preparation.args, root, process.env);
|
|
433
|
+
if (!ok) throw new Error(`${target.language}: native test preparation failed`);
|
|
434
|
+
}
|
|
435
|
+
}
|
|
420
436
|
const seedOf = new Map();
|
|
421
437
|
batches: for (const batch of batches) {
|
|
422
|
-
for (const run of invocations(target, batch.entries, { offline, scratch, coverage: useCoverage ? coverage : null,
|
|
438
|
+
for (const run of invocations(target, batch.entries, { offline, scratch, coverage: useCoverage ? coverage : null,
|
|
439
|
+
xdist, root, coverageConfig: report_.coverage })) {
|
|
423
440
|
const since = Date.now() - 1000;
|
|
424
441
|
const { ok, output } = await spawned(run.command, run.args, root,
|
|
425
442
|
{ ...process.env, ...run.env, LAWSPEC_SEED: batch.seed, HSPEC_SEED: batch.seed,
|
|
426
443
|
LAWSPEC_STATS: stats, LAWSPEC_FAILURES: failures, LAWSPEC_RECORDED: recordedFolder,
|
|
427
444
|
...(options["update-recorded"] ? { LAWSPEC_UPDATE_RECORDED: "1" } : {}) }, run.report?.kind === "go-json");
|
|
445
|
+
if (run.coverage) coverageRuns.push(run.coverage);
|
|
428
446
|
const executed = await executedTests(run.report, output, root, since);
|
|
429
447
|
for (const law of run.laws) seedOf.set(law.law, batch.seed);
|
|
430
|
-
if (report) junit.push({ target: target.language
|
|
431
|
-
if (!ok) { failed = true; break batches; }
|
|
448
|
+
if (report) junit.push({ target: `${target.language} (${target.root})`, xml: await junitOf(run, executed, root, since) });
|
|
449
|
+
if (!ok || executed.some((test) => test.status === "failed")) { failed = true; break batches; }
|
|
432
450
|
// A runner whose filter matched nothing reports success, so a law
|
|
433
451
|
// counts as passed only if the runner's report shows its tests ran.
|
|
434
452
|
const ran = new Set(executed.flatMap(run.ran ?? (() => [])));
|
|
@@ -437,9 +455,16 @@ async function runTests(compiler, input, selected, roots, config) {
|
|
|
437
455
|
}
|
|
438
456
|
}
|
|
439
457
|
if (unrun.length) failed = true;
|
|
458
|
+
if (useCoverage && beam) {
|
|
459
|
+
try {
|
|
460
|
+
coverageWritten = !!(await collectBeamCoverage({root, directory: coverage, runs: coverageRuns}));
|
|
461
|
+
} catch (error) {
|
|
462
|
+
failed = true;
|
|
463
|
+
coverageError = error.message;
|
|
464
|
+
}
|
|
465
|
+
}
|
|
440
466
|
// --benchmarks: the harness's benchmarks run after the laws, every time;
|
|
441
467
|
// they are measured, never asserted, and never cached.
|
|
442
|
-
const benchmarking = options.benchmarks ? (planned.benchmarks ?? []) : [];
|
|
443
468
|
if (!failed) for (const run of benchmarkInvocations(target, benchmarking, { offline })) {
|
|
444
469
|
const { ok } = await spawned(run.command, run.args, root,
|
|
445
470
|
{ ...process.env, ...run.env, LAWSPEC_SEED: seed, HSPEC_SEED: seed, LAWSPEC_STATS: stats, LAWSPEC_RECORDED: recordedFolder }, false);
|
|
@@ -458,9 +483,12 @@ async function runTests(compiler, input, selected, roots, config) {
|
|
|
458
483
|
// A law that failed is recorded with its seed; one that passed leaves.
|
|
459
484
|
for (const entry of stale) {
|
|
460
485
|
if (passed.includes(entry)) delete database[entry.law];
|
|
461
|
-
else if (seedOf.has(entry.law))
|
|
486
|
+
else if (seedOf.has(entry.law)) {
|
|
487
|
+
delete laws[entry.law];
|
|
488
|
+
database[entry.law] = { seed: Number(seedOf.get(entry.law)), failed: new Date().toISOString() };
|
|
489
|
+
}
|
|
462
490
|
}
|
|
463
|
-
await writeFile(path.join(
|
|
491
|
+
await writeFile(path.join(configRoot, ".lawspec", "failures", ".gitignore"), "*\n");
|
|
464
492
|
await writeFile(databaseFile, JSON.stringify(database, null, 2) + "\n");
|
|
465
493
|
await mkdir(path.dirname(resultsFile), { recursive: true });
|
|
466
494
|
await writeFile(path.join(path.dirname(resultsFile), ".gitignore"), "*\n");
|
|
@@ -470,7 +498,7 @@ async function runTests(compiler, input, selected, roots, config) {
|
|
|
470
498
|
const benchmarks = statistics.filter((s) => s.benchmark);
|
|
471
499
|
// How each parallel unit's tests actually ran at the same time.
|
|
472
500
|
const parallelism = statistics.filter((s) => s.parallel).map(({ parallel, mode, workers }) => ({ unit: parallel, mode, workers }));
|
|
473
|
-
summaries.push({ target: target.language, seed: Number(seed), ran: stale.map((e) => e.law),
|
|
501
|
+
summaries.push({ target: target.language, root: target.root, seed: Number(seed), ran: stale.map((e) => e.law),
|
|
474
502
|
unchanged: chosen.length - stale.length, ok: !failed,
|
|
475
503
|
...(chosen.length !== planned.tests.length ? { deselected: planned.tests.length - chosen.length } : {}),
|
|
476
504
|
...(unrun.length ? { unrun: unrun.map((e) => e.law) } : {}),
|
|
@@ -478,14 +506,15 @@ async function runTests(compiler, input, selected, roots, config) {
|
|
|
478
506
|
...(unmet.length ? { unmetCover: unmet } : {}),
|
|
479
507
|
...(benchmarks.length ? { benchmarks } : {}),
|
|
480
508
|
...(parallelism.length ? { parallelism } : {}),
|
|
481
|
-
...(
|
|
509
|
+
...(coverageError ? {coverageError} : {}),
|
|
510
|
+
...(useCoverage && (!beam || coverageWritten) ? { coverage: path.relative(process.cwd(), coverage) || "." } : {}) });
|
|
482
511
|
}
|
|
483
512
|
if (report) {
|
|
484
513
|
await mkdir(path.dirname(path.resolve(report.path)), { recursive: true });
|
|
485
514
|
await writeFile(path.resolve(report.path), mergeJunit(junit));
|
|
486
515
|
}
|
|
487
516
|
output(options.json ? summaries : summaries.map((s) =>
|
|
488
|
-
`${s.target}: ${s.ran.length ? `ran ${s.ran.length} law(s) with seed ${s.seed}` : "nothing to run"}` +
|
|
517
|
+
`${s.target}${summaries.filter(other => other.target === s.target).length > 1 ? ` (${s.root})` : ""}: ${s.ran.length ? `ran ${s.ran.length} law(s) with seed ${s.seed}` : "nothing to run"}` +
|
|
489
518
|
`, ${s.unchanged} unchanged since their last passing run.${s.ok ? "" : " FAILED"}` +
|
|
490
519
|
(s.deselected ? `\n${s.deselected} law(s) not selected by --tag or --exclude-tag.` : "") +
|
|
491
520
|
(s.unrun ? `\nNo tests ran for ${s.unrun.join(", ")}; the runner matched none of their tests.` : "") +
|
|
@@ -493,6 +522,7 @@ async function runTests(compiler, input, selected, roots, config) {
|
|
|
493
522
|
(s.unmetCover ? `\nCover not met: ${s.unmetCover.join("; ")}` : "") +
|
|
494
523
|
(s.parallelism ? "\n" + s.parallelism.map((p) => `parallel ${p.unit}: ${p.mode}, ${p.workers} worker(s)`).join("\n") : "") +
|
|
495
524
|
(s.benchmarks ? "\n" + s.benchmarks.map((b) => `benchmark ${b.benchmark}: mean ${(b.mean_ns / 1000).toFixed(2)} us over ${b.iterations} iteration(s)`).join("\n") : "") +
|
|
525
|
+
(s.coverageError ? `\nCoverage failed: ${s.coverageError}` : "") +
|
|
496
526
|
(s.coverage ? `\nCoverage written to ${s.coverage}.` : "")).join("\n"));
|
|
497
527
|
if (summaries.some((s) => !s.ok)) process.exitCode = 1;
|
|
498
528
|
}
|
|
@@ -515,10 +545,14 @@ async function junitOf(run, executed, root, since) {
|
|
|
515
545
|
}
|
|
516
546
|
return xml.join("\n");
|
|
517
547
|
}
|
|
518
|
-
|
|
548
|
+
// BEAM reports only executed cases. Intentional harness skips remain
|
|
549
|
+
// reportable obligations; they are never evidence that a test passed.
|
|
550
|
+
const skipped = run.report?.kind === "beam-events" ? run.laws.filter((entry) => entry.skip)
|
|
551
|
+
.map((entry) => ({name: entry.label, classname: entry.unit, status: "skipped", reason: entry.skip})) : [];
|
|
552
|
+
return junitFromTests([...executed, ...skipped]);
|
|
519
553
|
}
|
|
520
554
|
// The tool --coverage needs, if it is missing.
|
|
521
|
-
async function coverageMissing(target, root) {
|
|
555
|
+
async function coverageMissing(target, root, report) {
|
|
522
556
|
const tool = coverageTools[target.language];
|
|
523
557
|
if (!tool) return { tool: "a coverage tool", install: "no coverage tool is known for this target" };
|
|
524
558
|
const probe = (command, args) => new Promise((resolve) => {
|
|
@@ -526,6 +560,12 @@ async function coverageMissing(target, root) {
|
|
|
526
560
|
child.on("error", () => resolve(false));
|
|
527
561
|
child.on("close", (code) => resolve(code === 0));
|
|
528
562
|
});
|
|
563
|
+
if (["erlang", "elixir", "gleam"].includes(target.language)) {
|
|
564
|
+
const requirement = beamCoverageRequirement(target, report);
|
|
565
|
+
if (requirement) return requirement;
|
|
566
|
+
return (await probe("erl", ["-noshell", "-eval",
|
|
567
|
+
'case code:ensure_loaded(cover) of {module,cover} -> halt(0); _ -> halt(1) end.'])) ? null : tool;
|
|
568
|
+
}
|
|
529
569
|
if (target.language === "python") return (await probe(target.python || "python3", tool.check)) ? null : tool;
|
|
530
570
|
if (["javascript", "typescript"].includes(target.language)) return (await probe("npx", ["--no-install", "c8", "--version"])) ? null : tool;
|
|
531
571
|
if (target.language === "rust") return (await probe("cargo", ["llvm-cov", "--version"])) ? null : tool;
|