lawspec 0.20.1 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +361 -2
- package/README.md +3 -3
- package/beam-coverage.mjs +160 -0
- package/beam-doctor.mjs +306 -0
- package/bin/lawspec.mjs +306 -38
- package/build.json +492 -106
- package/compatibility.json +17 -0
- package/core.wasm +0 -0
- package/core_jsffi.js +2 -2
- package/doctor.mjs +7 -1
- package/examples/native-payments/elixir/lawspec.json +149 -0
- package/examples/native-payments/elixir/lib/payments_domain.ex +54 -0
- package/examples/native-payments/elixir/test/support/payment_generators.ex +9 -0
- package/examples/native-payments/erlang/include/payments_domain.hrl +2 -0
- package/examples/native-payments/erlang/lawspec.json +148 -0
- package/examples/native-payments/erlang/src/payments_domain.erl +21 -0
- package/examples/native-payments/erlang/test/payment_generators.erl +10 -0
- package/examples/native-payments/gleam/lawspec.json +149 -0
- package/examples/native-payments/gleam/prepare.mjs +48 -0
- package/examples/native-payments/gleam/src/payments_domain.gleam +32 -0
- package/examples/native-payments/gleam/test/payment_generators.gleam +13 -0
- package/examples/specs/abilities.lawspec +103 -0
- package/examples/specs/ability_imports.lawspec +107 -0
- package/examples/specs/builtins.lawspec +121 -0
- package/examples/specs/crypto.lawspec +51 -0
- package/examples/specs/distribution.lawspec +29 -0
- package/examples/specs/failures.lawspec +96 -0
- package/examples/specs/handler_bindings.lawspec +65 -0
- package/examples/specs/harness.lawspec +75 -0
- package/examples/specs/harness_testing.lawspec +45 -0
- package/examples/specs/matchers.lawspec +112 -0
- package/examples/specs/resources.lawspec +116 -0
- package/examples/specs/scheduling_order.lawspec +18 -0
- package/examples/specs/scheduling_overlap.lawspec +14 -0
- package/examples/specs/tables.lawspec +64 -0
- package/failure-database.mjs +46 -0
- package/index.d.ts +12 -1
- package/native-examples.mjs +7 -2
- package/package.json +1 -1
- package/templates.mjs +105 -16
- package/test-command.mjs +316 -35
package/bin/lawspec.mjs
CHANGED
|
@@ -8,8 +8,11 @@ import { targets, templates, commands, setup } from "../templates.mjs";
|
|
|
8
8
|
import { generateExamples } from "../examples-command.mjs";
|
|
9
9
|
import { showScalar } from "../scalars.mjs";
|
|
10
10
|
import { doctor } from "../doctor.mjs";
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
11
|
+
import { beamCoverageRequirement, collectBeamCoverage } from "../beam-coverage.mjs";
|
|
12
|
+
import { failureDatabase, projectKey } from "../failure-database.mjs";
|
|
13
|
+
import { spawn, spawnSync } from "node:child_process";
|
|
14
|
+
import { benchmarkInvocations, preparationInvocations, environmentDigest, executedTests, invocations, lawKeys, projectDigest, recordedDigest, selectByTags, mergeJunit,
|
|
15
|
+
harnessStatistics, coverageTools, junitFromTests, testsToRun } from "../test-command.mjs";
|
|
13
16
|
import {
|
|
14
17
|
readOptional,
|
|
15
18
|
planWrites,
|
|
@@ -17,18 +20,23 @@ import {
|
|
|
17
20
|
atomicWrite,
|
|
18
21
|
safePath,
|
|
19
22
|
} from "../files.mjs";
|
|
20
|
-
const VERSION = "0.
|
|
23
|
+
const VERSION = "0.22.0";
|
|
21
24
|
const args = process.argv.slice(2);
|
|
22
25
|
const verb = args.shift();
|
|
23
26
|
const options = {};
|
|
24
27
|
const positional = [];
|
|
25
28
|
for (let i = 0; i < args.length; i++) {
|
|
26
29
|
const arg = args[i];
|
|
27
|
-
if (["--target", "--project", "--config", "--output", "--machine-bits", "--example", "--seed"].includes(arg)) {
|
|
30
|
+
if (["--target", "--project", "--config", "--output", "--machine-bits", "--example", "--seed", "--report"].includes(arg)) {
|
|
28
31
|
if (!args[i + 1] || args[i + 1].startsWith("--"))
|
|
29
32
|
throw new Error(`Missing value for ${arg}`);
|
|
30
33
|
options[arg.slice(2)] = args[++i];
|
|
31
|
-
} else if (["--
|
|
34
|
+
} else if (["--tag", "--exclude-tag"].includes(arg)) {
|
|
35
|
+
// Repeatable, and each may list several: --tag a,b --tag c.
|
|
36
|
+
if (!args[i + 1] || args[i + 1].startsWith("--"))
|
|
37
|
+
throw new Error(`Missing value for ${arg}`);
|
|
38
|
+
options[arg.slice(2)] = [...(options[arg.slice(2)] ?? []), ...args[++i].split(",").filter(Boolean)];
|
|
39
|
+
} else if (["--dry-run", "--check", "--json", "--minify", "--no-cache", "--fresh", "--coverage", "--update-recorded", "--benchmarks"].includes(arg))
|
|
32
40
|
options[arg.slice(2)] = true;
|
|
33
41
|
else if (arg.startsWith("--")) throw new Error(`Unknown option: ${arg}`);
|
|
34
42
|
else positional.push(arg);
|
|
@@ -191,6 +199,9 @@ async function init() {
|
|
|
191
199
|
"build.gradle",
|
|
192
200
|
"build.gradle.kts",
|
|
193
201
|
"settings.gradle.kts",
|
|
202
|
+
"rebar.config",
|
|
203
|
+
"mix.exs",
|
|
204
|
+
"gleam.toml",
|
|
194
205
|
];
|
|
195
206
|
const hasBuild = (await readdir(root)).some(
|
|
196
207
|
(f) => buildFiles.includes(f) || f.endsWith(".cabal"),
|
|
@@ -236,6 +247,24 @@ function showAssertion(assertion) {
|
|
|
236
247
|
if (assertion.kind === "implies") return `${assertion.guard.text} implies ${showAssertion(assertion.body)}`;
|
|
237
248
|
return assertion.items.map(showAssertion).join(" and ");
|
|
238
249
|
}
|
|
250
|
+
// An ability key, as the spec names it: example.shop::ability::Store(Int32)
|
|
251
|
+
// is Store Int32, lawspec::ability::Fail(example.shop::type::E) is Fail E.
|
|
252
|
+
function abilityName(key) {
|
|
253
|
+
const short = (text) => text.replace(/[A-Za-z0-9_.]+::(ability|type)::/g, "");
|
|
254
|
+
const match = /^(.*?)(\((.*)\))?$/.exec(short(key));
|
|
255
|
+
return match[3] ? `${match[1]} ${match[3].replace(/[()]/g, " ").replace(/;/g, " ").trim()}` : match[1];
|
|
256
|
+
}
|
|
257
|
+
// The handlers a law runs under, and the ability row of each function its
|
|
258
|
+
// expansion calls.
|
|
259
|
+
function explainAbilities(law, expansion, units) {
|
|
260
|
+
const unit = (units || []).find((u) => u.id === law.owner);
|
|
261
|
+
const called = (unit ? unit.declarations : []).filter(
|
|
262
|
+
(d) => d.uses && new RegExp(`\\b${d.name}\\b`).test(expansion),
|
|
263
|
+
);
|
|
264
|
+
const handlers = (law.handlers || []).map((h) => `\n ${abilityName(h.ability)}: ${h.handler}`).join("");
|
|
265
|
+
const rows = called.map((d) => `\n ${d.name} uses ${d.uses.map(abilityName).join(", ")}`).join("");
|
|
266
|
+
return (handlers ? `\nhandlers${handlers}` : "") + (rows ? `\nrows${rows}` : "");
|
|
267
|
+
}
|
|
239
268
|
function explainExamples(law) {
|
|
240
269
|
return law.examples.map(ex =>
|
|
241
270
|
`\nexample ${JSON.stringify(ex.name)}\n` +
|
|
@@ -243,6 +272,50 @@ function explainExamples(law) {
|
|
|
243
272
|
ex.expectations.map(e => ` expect ${showAssertion(e)}`).join("\n")
|
|
244
273
|
).join("\n");
|
|
245
274
|
}
|
|
275
|
+
// What lawspec test last recorded for each law, across targets.
|
|
276
|
+
async function lastRuns() {
|
|
277
|
+
const folder = path.join(configRoot, ".lawspec", "results");
|
|
278
|
+
const runs = new Map();
|
|
279
|
+
for (const name of (await readOptionalDirectory(folder)) ?? []) {
|
|
280
|
+
if (!name.endsWith(".json")) continue;
|
|
281
|
+
const recorded = JSON.parse((await readOptional(path.join(folder, name))) ?? "{}");
|
|
282
|
+
for (const [law, run] of Object.entries(recorded.laws ?? {})) {
|
|
283
|
+
const known = runs.get(law) ?? {};
|
|
284
|
+
runs.set(law, { flaky: known.flaky || run.flaky, adequacy: run.adequacy ?? known.adequacy });
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
return runs;
|
|
288
|
+
}
|
|
289
|
+
// One law's harness, as lawspec evidence shows it.
|
|
290
|
+
function harnessText(item) {
|
|
291
|
+
const h = item.harness ?? {};
|
|
292
|
+
const parts = [];
|
|
293
|
+
if (h.unit) parts.push(`harness ${h.unit}`);
|
|
294
|
+
if (h.tags?.length) parts.push(`tags ${h.tags.join(", ")}`);
|
|
295
|
+
if (h.strategies?.length) parts.push(h.strategies.map((s) => `${s.input} drawn by ${s.strategy}`).join(", "));
|
|
296
|
+
for (const c of h.cover ?? []) parts.push(`cover ${c.percent}% "${c.label}" when ${c.when.text}`);
|
|
297
|
+
for (const c of h.classify ?? []) parts.push(`classify ${c.when.text} as "${c.label}"`);
|
|
298
|
+
for (const l of h.labels ?? []) parts.push(`label ${l.text}`);
|
|
299
|
+
if (h.target) parts.push(`target maximize ${h.target.text}`);
|
|
300
|
+
if (h.timeoutMilliseconds) parts.push(`timeout ${h.timeoutMilliseconds} ms`);
|
|
301
|
+
if (h.repeat > 1) parts.push(`repeat ${h.repeat}`);
|
|
302
|
+
if (h.retries) parts.push(`retry flaky ${h.retries}`);
|
|
303
|
+
if (h.skip) parts.push(`skip "${h.skip}"`);
|
|
304
|
+
if (h.knownFailing) parts.push(`known failing "${h.knownFailing}"`);
|
|
305
|
+
if (h.orderRandom) parts.push("order random");
|
|
306
|
+
// Python runs a parallel unit's tests at the same time only with
|
|
307
|
+
// pytest-xdist; without it, parallel changes nothing there.
|
|
308
|
+
if (h.parallel) parts.push("parallel (on Python only with pytest-xdist; otherwise one after another)");
|
|
309
|
+
const lines = [` ${item.declaration.replace("::law::", "::")}: ${parts.join("; ") || "no harness settings"}`];
|
|
310
|
+
for (const run of item.adequacy ?? []) {
|
|
311
|
+
lines.push(` last run: ${run.cases} generated case(s)` +
|
|
312
|
+
(run.cover ?? []).map((c) => `; cover "${c.label}" ${c.observed}% (needs ${c.required}%)${c.met ? "" : " NOT MET"}`).join(""));
|
|
313
|
+
const counts = { ...(run.classes ?? {}), ...Object.fromEntries(Object.entries(run.labels ?? {}).map(([k, v]) => [`label ${k}`, v])) };
|
|
314
|
+
if (Object.keys(counts).length)
|
|
315
|
+
lines.push(` ${Object.entries(counts).map(([k, v]) => `${k}: ${run.cases ? (100 * v / run.cases).toFixed(1) : 0}%`).join(", ")}`);
|
|
316
|
+
}
|
|
317
|
+
return lines.join("\n");
|
|
318
|
+
}
|
|
246
319
|
// The compiler keeps work between runs in .lawspec/cache, in one folder per
|
|
247
320
|
// compiler build, so a different build never reads another's entries. The
|
|
248
321
|
// WebAssembly compiler sees only the working directory. A command that fails
|
|
@@ -272,12 +345,19 @@ async function buildDigest() {
|
|
|
272
345
|
return buildDigestValue;
|
|
273
346
|
}
|
|
274
347
|
// lawspec test: run the tests of the laws whose results may have changed,
|
|
275
|
-
// and record each passing law's key and seed (see test-command.mjs).
|
|
348
|
+
// and record each passing law's key and seed (see test-command.mjs). The
|
|
349
|
+
// harness plane chooses which: --tag and --exclude-tag select laws by their
|
|
350
|
+
// harness tags, and a skipped law runs nothing. A law whose last run failed
|
|
351
|
+
// runs first, and its failing inputs are replayed first (.lawspec/failures).
|
|
276
352
|
async function runTests(compiler, input, selected, roots, config) {
|
|
353
|
+
// Recorded values live beside lawspec.json, under recorded/<unit>/<name>.
|
|
354
|
+
const recordedFolder = path.join(configRoot, "recorded");
|
|
277
355
|
const seed = options.seed ?? String(1 + Math.floor(Math.random() * 2147483646));
|
|
278
356
|
const offline = process.env.LAWSPEC_OFFLINE === "1";
|
|
279
357
|
const build = await buildDigest();
|
|
280
358
|
const summaries = [];
|
|
359
|
+
const junit = [];
|
|
360
|
+
const report = options.report === undefined ? null : parseReport(options.report);
|
|
281
361
|
for (const [i, target] of selected.entries()) {
|
|
282
362
|
const root = roots[i];
|
|
283
363
|
const planned = diagnostics(await compiler.planGeneration({
|
|
@@ -287,53 +367,214 @@ async function runTests(compiler, input, selected, roots, config) {
|
|
|
287
367
|
const pending = await planWrites(root, planned.files);
|
|
288
368
|
if (pending.changes.length)
|
|
289
369
|
throw new Error(`${target.language}: generated files are out of date; run lawspec generate${options.minify ? " --minify" : ""}`);
|
|
290
|
-
const
|
|
291
|
-
if (!
|
|
370
|
+
const report_ = await doctor(target, root, planned.files);
|
|
371
|
+
if (!report_.ok) throw new Error(`${target.language}: ${report_.message}\n${report_.instructions}`);
|
|
292
372
|
const generated = new Set(planned.files.filter((f) => f.ownership === "generated").map((f) => f.path));
|
|
293
373
|
const keys = lawKeys({
|
|
294
374
|
build, target, machineBits: input.machineBits, minify: options.minify === true,
|
|
295
375
|
tests: planned.tests, files: planned.files,
|
|
296
|
-
|
|
376
|
+
// Recorded values are spec data: a changed recording runs its laws again.
|
|
377
|
+
environment: await environmentDigest(root, report_) + (await recordedDigest(recordedFolder)),
|
|
297
378
|
project: await projectDigest(root, generated),
|
|
298
379
|
});
|
|
299
|
-
const
|
|
300
|
-
|
|
380
|
+
const identity = projectKey(root);
|
|
381
|
+
const resultsFile = path.join(configRoot, ".lawspec", "results", `${target.language}-${identity}.json`);
|
|
301
382
|
const previous = JSON.parse((await readOptional(resultsFile)) ?? '{"version":1,"laws":{}}');
|
|
302
|
-
const
|
|
383
|
+
const {directory: failures, file: databaseFile, database} = await failureDatabase(configRoot, target.language, root);
|
|
384
|
+
// Recording again runs every law, so each records what it sees now.
|
|
385
|
+
const chosen = selectByTags(planned.tests, options.tag ?? [], options["exclude-tag"] ?? []);
|
|
386
|
+
const stale = testsToRun(chosen, keys, previous.laws, database,
|
|
387
|
+
{fresh: options.fresh, coverage: options.coverage, updateRecorded: options["update-recorded"]});
|
|
388
|
+
// The failure database: each law whose last run failed, with the seed
|
|
389
|
+
// that exposed it. Those laws run first, with that seed, so the failing
|
|
390
|
+
// inputs are generated again; the runtimes keep their own counterexample
|
|
391
|
+
// databases beside it (Hypothesis, proptest).
|
|
392
|
+
const replayed = options.seed === undefined ? stale.filter((entry) => database[entry.law]) : [];
|
|
393
|
+
const batches = [
|
|
394
|
+
...[...new Set(replayed.map((entry) => database[entry.law].seed))].map((s) =>
|
|
395
|
+
({ seed: String(s), entries: replayed.filter((entry) => database[entry.law].seed === s) })),
|
|
396
|
+
{ seed, entries: stale.filter((entry) => !replayed.includes(entry)) },
|
|
397
|
+
].filter((batch) => batch.entries.length);
|
|
303
398
|
const passed = [];
|
|
304
399
|
const unrun = [];
|
|
305
400
|
let failed = false;
|
|
306
|
-
const scratch = path.join(configRoot, ".lawspec", "reports", target.language);
|
|
401
|
+
const scratch = path.join(configRoot, ".lawspec", "reports", target.language, identity);
|
|
402
|
+
const stats = path.join(scratch, "statistics");
|
|
403
|
+
const beam = ["erlang", "elixir", "gleam"].includes(target.language);
|
|
404
|
+
const coverage = options.coverage ? path.join(configRoot, ".lawspec", "coverage", target.language, identity) : null;
|
|
405
|
+
const coverageRuns = [];
|
|
406
|
+
let coverageWritten = false, coverageError;
|
|
307
407
|
await rm(scratch, { recursive: true, force: true });
|
|
308
|
-
await mkdir(
|
|
309
|
-
await
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
408
|
+
await mkdir(stats, { recursive: true });
|
|
409
|
+
await mkdir(failures, { recursive: true });
|
|
410
|
+
await writeFile(path.join(configRoot, ".lawspec", "reports", ".gitignore"), "*\n");
|
|
411
|
+
const missingCoverage = coverage ? await coverageMissing(target, root, report_) : null;
|
|
412
|
+
const useCoverage = coverage && !missingCoverage;
|
|
413
|
+
if (missingCoverage)
|
|
414
|
+
console.error(`${target.language}: --coverage needs ${missingCoverage.tool}, which is not available; ${missingCoverage.install}. Running without coverage.`);
|
|
415
|
+
if (useCoverage) {
|
|
416
|
+
if (beam) await rm(coverage, {recursive: true, force: true});
|
|
417
|
+
await mkdir(beam ? path.join(coverage, "runs") : coverage, { recursive: true });
|
|
418
|
+
}
|
|
419
|
+
// parallel on Python needs pytest-xdist; without it the tests run one
|
|
420
|
+
// after another, and lawspec test says so.
|
|
421
|
+
let xdist = false;
|
|
422
|
+
if (target.language === "python" && stale.some((entry) => entry.parallel)) {
|
|
423
|
+
xdist = spawnSync(target.python || "python3", ["-c", "import xdist"], { cwd: root, stdio: "ignore" }).status === 0;
|
|
424
|
+
if (!xdist) console.error("python: `parallel` runs tests at the same time only with pytest-xdist (pip install pytest-xdist); without it they run one after another.");
|
|
425
|
+
}
|
|
426
|
+
// A skipped law runs nothing. BEAM known-failing checks finish as native
|
|
427
|
+
// passing tests, so their completion must appear in the native report.
|
|
428
|
+
const expected = (entry) => !entry.skip && (!entry.knownFailing || ["erlang", "elixir", "gleam"].includes(target.language));
|
|
429
|
+
const benchmarking = options.benchmarks ? (planned.benchmarks ?? []) : [];
|
|
430
|
+
if (batches.length || benchmarking.length) {
|
|
431
|
+
for (const preparation of preparationInvocations(target, planned.files)) {
|
|
432
|
+
const {ok} = await spawned(preparation.command, preparation.args, root, process.env);
|
|
433
|
+
if (!ok) throw new Error(`${target.language}: native test preparation failed`);
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
const seedOf = new Map();
|
|
437
|
+
batches: for (const batch of batches) {
|
|
438
|
+
for (const run of invocations(target, batch.entries, { offline, scratch, coverage: useCoverage ? coverage : null,
|
|
439
|
+
xdist, root, coverageConfig: report_.coverage })) {
|
|
440
|
+
const since = Date.now() - 1000;
|
|
441
|
+
const { ok, output } = await spawned(run.command, run.args, root,
|
|
442
|
+
{ ...process.env, ...run.env, LAWSPEC_SEED: batch.seed, HSPEC_SEED: batch.seed,
|
|
443
|
+
LAWSPEC_STATS: stats, LAWSPEC_FAILURES: failures, LAWSPEC_RECORDED: recordedFolder,
|
|
444
|
+
...(options["update-recorded"] ? { LAWSPEC_UPDATE_RECORDED: "1" } : {}) }, run.report?.kind === "go-json");
|
|
445
|
+
if (run.coverage) coverageRuns.push(run.coverage);
|
|
446
|
+
const executed = await executedTests(run.report, output, root, since);
|
|
447
|
+
for (const law of run.laws) seedOf.set(law.law, batch.seed);
|
|
448
|
+
if (report) junit.push({ target: `${target.language} (${target.root})`, xml: await junitOf(run, executed, root, since) });
|
|
449
|
+
if (!ok || executed.some((test) => test.status === "failed")) { failed = true; break batches; }
|
|
450
|
+
// A runner whose filter matched nothing reports success, so a law
|
|
451
|
+
// counts as passed only if the runner's report shows its tests ran.
|
|
452
|
+
const ran = new Set(executed.flatMap(run.ran ?? (() => [])));
|
|
453
|
+
passed.push(...run.laws.filter((law) => ran.has(law) || !expected(law)));
|
|
454
|
+
unrun.push(...run.laws.filter((law) => !ran.has(law) && expected(law)));
|
|
455
|
+
}
|
|
320
456
|
}
|
|
321
457
|
if (unrun.length) failed = true;
|
|
458
|
+
if (useCoverage && beam) {
|
|
459
|
+
try {
|
|
460
|
+
coverageWritten = !!(await collectBeamCoverage({root, directory: coverage, runs: coverageRuns}));
|
|
461
|
+
} catch (error) {
|
|
462
|
+
failed = true;
|
|
463
|
+
coverageError = error.message;
|
|
464
|
+
}
|
|
465
|
+
}
|
|
466
|
+
// --benchmarks: the harness's benchmarks run after the laws, every time;
|
|
467
|
+
// they are measured, never asserted, and never cached.
|
|
468
|
+
if (!failed) for (const run of benchmarkInvocations(target, benchmarking, { offline })) {
|
|
469
|
+
const { ok } = await spawned(run.command, run.args, root,
|
|
470
|
+
{ ...process.env, ...run.env, LAWSPEC_SEED: seed, HSPEC_SEED: seed, LAWSPEC_STATS: stats, LAWSPEC_RECORDED: recordedFolder }, false);
|
|
471
|
+
if (!ok) { failed = true; break; }
|
|
472
|
+
}
|
|
473
|
+
const statistics = await harnessStatistics(stats);
|
|
322
474
|
const laws = Object.fromEntries(planned.tests.filter((entry) => previous.laws[entry.law]).map((entry) => [entry.law, previous.laws[entry.law]]));
|
|
323
|
-
for (const entry of passed)
|
|
475
|
+
for (const entry of passed) {
|
|
476
|
+
const label = entry.label;
|
|
477
|
+
const observed = statistics.filter((s) => s.law === label);
|
|
478
|
+
laws[entry.law] = { key: keys.get(entry.law), seed: Number(seedOf.get(entry.law) ?? seed), passed: new Date().toISOString(),
|
|
479
|
+
...(observed.some((s) => s.outcome === "flaky") ? { flaky: true } : {}),
|
|
480
|
+
...(observed.some((s) => s.cover || s.labels || s.classes) ? { adequacy: observed.filter((s) => s.cover || s.labels || s.classes)
|
|
481
|
+
.map(({ test, cases, cover, classes, labels }) => ({ test, cases, cover, classes, labels })) } : {}) };
|
|
482
|
+
}
|
|
483
|
+
// A law that failed is recorded with its seed; one that passed leaves.
|
|
484
|
+
for (const entry of stale) {
|
|
485
|
+
if (passed.includes(entry)) delete database[entry.law];
|
|
486
|
+
else if (seedOf.has(entry.law)) {
|
|
487
|
+
delete laws[entry.law];
|
|
488
|
+
database[entry.law] = { seed: Number(seedOf.get(entry.law)), failed: new Date().toISOString() };
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
await writeFile(path.join(configRoot, ".lawspec", "failures", ".gitignore"), "*\n");
|
|
492
|
+
await writeFile(databaseFile, JSON.stringify(database, null, 2) + "\n");
|
|
324
493
|
await mkdir(path.dirname(resultsFile), { recursive: true });
|
|
325
494
|
await writeFile(path.join(path.dirname(resultsFile), ".gitignore"), "*\n");
|
|
326
495
|
await writeFile(resultsFile, JSON.stringify({ version: 1, laws }, null, 2) + "\n");
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
496
|
+
const flaky = statistics.filter((s) => s.outcome === "flaky").map((s) => s.law);
|
|
497
|
+
const unmet = statistics.flatMap((s) => (s.cover ?? []).filter((c) => !c.met).map((c) => `${s.law}: cover ${c.required}% "${c.label}" (${c.observed}%)`));
|
|
498
|
+
const benchmarks = statistics.filter((s) => s.benchmark);
|
|
499
|
+
// How each parallel unit's tests actually ran at the same time.
|
|
500
|
+
const parallelism = statistics.filter((s) => s.parallel).map(({ parallel, mode, workers }) => ({ unit: parallel, mode, workers }));
|
|
501
|
+
summaries.push({ target: target.language, root: target.root, seed: Number(seed), ran: stale.map((e) => e.law),
|
|
502
|
+
unchanged: chosen.length - stale.length, ok: !failed,
|
|
503
|
+
...(chosen.length !== planned.tests.length ? { deselected: planned.tests.length - chosen.length } : {}),
|
|
504
|
+
...(unrun.length ? { unrun: unrun.map((e) => e.law) } : {}),
|
|
505
|
+
...(flaky.length ? { flaky: [...new Set(flaky)] } : {}),
|
|
506
|
+
...(unmet.length ? { unmetCover: unmet } : {}),
|
|
507
|
+
...(benchmarks.length ? { benchmarks } : {}),
|
|
508
|
+
...(parallelism.length ? { parallelism } : {}),
|
|
509
|
+
...(coverageError ? {coverageError} : {}),
|
|
510
|
+
...(useCoverage && (!beam || coverageWritten) ? { coverage: path.relative(process.cwd(), coverage) || "." } : {}) });
|
|
511
|
+
}
|
|
512
|
+
if (report) {
|
|
513
|
+
await mkdir(path.dirname(path.resolve(report.path)), { recursive: true });
|
|
514
|
+
await writeFile(path.resolve(report.path), mergeJunit(junit));
|
|
330
515
|
}
|
|
331
516
|
output(options.json ? summaries : summaries.map((s) =>
|
|
332
|
-
`${s.target}: ${s.ran.length ? `ran ${s.ran.length} law(s) with seed ${s.seed}` : "nothing to run"}` +
|
|
517
|
+
`${s.target}${summaries.filter(other => other.target === s.target).length > 1 ? ` (${s.root})` : ""}: ${s.ran.length ? `ran ${s.ran.length} law(s) with seed ${s.seed}` : "nothing to run"}` +
|
|
333
518
|
`, ${s.unchanged} unchanged since their last passing run.${s.ok ? "" : " FAILED"}` +
|
|
334
|
-
(s.
|
|
519
|
+
(s.deselected ? `\n${s.deselected} law(s) not selected by --tag or --exclude-tag.` : "") +
|
|
520
|
+
(s.unrun ? `\nNo tests ran for ${s.unrun.join(", ")}; the runner matched none of their tests.` : "") +
|
|
521
|
+
(s.flaky ? `\nFlaky (failed, then passed on a retry): ${s.flaky.join(", ")}` : "") +
|
|
522
|
+
(s.unmetCover ? `\nCover not met: ${s.unmetCover.join("; ")}` : "") +
|
|
523
|
+
(s.parallelism ? "\n" + s.parallelism.map((p) => `parallel ${p.unit}: ${p.mode}, ${p.workers} worker(s)`).join("\n") : "") +
|
|
524
|
+
(s.benchmarks ? "\n" + s.benchmarks.map((b) => `benchmark ${b.benchmark}: mean ${(b.mean_ns / 1000).toFixed(2)} us over ${b.iterations} iteration(s)`).join("\n") : "") +
|
|
525
|
+
(s.coverageError ? `\nCoverage failed: ${s.coverageError}` : "") +
|
|
526
|
+
(s.coverage ? `\nCoverage written to ${s.coverage}.` : "")).join("\n"));
|
|
335
527
|
if (summaries.some((s) => !s.ok)) process.exitCode = 1;
|
|
336
528
|
}
|
|
529
|
+
// --report junit=path: one JUnit report merged across targets.
|
|
530
|
+
function parseReport(value) {
|
|
531
|
+
const match = value.match(/^junit=(.+)$/);
|
|
532
|
+
if (!match) throw new Error("--report takes junit=<path>");
|
|
533
|
+
return { kind: "junit", path: match[1] };
|
|
534
|
+
}
|
|
535
|
+
// A run's JUnit XML: the runner's own where it writes one, otherwise made
|
|
536
|
+
// from what it printed.
|
|
537
|
+
async function junitOf(run, executed, root, since) {
|
|
538
|
+
if (run.report?.kind === "junit") {
|
|
539
|
+
const files = run.report.files ?? (await readdir(path.join(root, run.report.directory)).catch(() => []))
|
|
540
|
+
.filter((n) => n.endsWith(".xml")).map((n) => path.join(root, run.report.directory, n));
|
|
541
|
+
const xml = [];
|
|
542
|
+
for (const file of files) {
|
|
543
|
+
const info = await stat(path.resolve(root, file)).catch(() => null);
|
|
544
|
+
if (info && info.mtimeMs >= since) xml.push(await readFile(path.resolve(root, file), "utf8"));
|
|
545
|
+
}
|
|
546
|
+
return xml.join("\n");
|
|
547
|
+
}
|
|
548
|
+
// BEAM reports only executed cases. Intentional harness skips remain
|
|
549
|
+
// reportable obligations; they are never evidence that a test passed.
|
|
550
|
+
const skipped = run.report?.kind === "beam-events" ? run.laws.filter((entry) => entry.skip)
|
|
551
|
+
.map((entry) => ({name: entry.label, classname: entry.unit, status: "skipped", reason: entry.skip})) : [];
|
|
552
|
+
return junitFromTests([...executed, ...skipped]);
|
|
553
|
+
}
|
|
554
|
+
// The tool --coverage needs, if it is missing.
|
|
555
|
+
async function coverageMissing(target, root, report) {
|
|
556
|
+
const tool = coverageTools[target.language];
|
|
557
|
+
if (!tool) return { tool: "a coverage tool", install: "no coverage tool is known for this target" };
|
|
558
|
+
const probe = (command, args) => new Promise((resolve) => {
|
|
559
|
+
const child = spawn(command, args, { cwd: root, stdio: "ignore" });
|
|
560
|
+
child.on("error", () => resolve(false));
|
|
561
|
+
child.on("close", (code) => resolve(code === 0));
|
|
562
|
+
});
|
|
563
|
+
if (["erlang", "elixir", "gleam"].includes(target.language)) {
|
|
564
|
+
const requirement = beamCoverageRequirement(target, report);
|
|
565
|
+
if (requirement) return requirement;
|
|
566
|
+
return (await probe("erl", ["-noshell", "-eval",
|
|
567
|
+
'case code:ensure_loaded(cover) of {module,cover} -> halt(0); _ -> halt(1) end.'])) ? null : tool;
|
|
568
|
+
}
|
|
569
|
+
if (target.language === "python") return (await probe(target.python || "python3", tool.check)) ? null : tool;
|
|
570
|
+
if (["javascript", "typescript"].includes(target.language)) return (await probe("npx", ["--no-install", "c8", "--version"])) ? null : tool;
|
|
571
|
+
if (target.language === "rust") return (await probe("cargo", ["llvm-cov", "--version"])) ? null : tool;
|
|
572
|
+
if (target.language === "kotlin") {
|
|
573
|
+
const build = await readFile(path.join(root, "build.gradle.kts"), "utf8").catch(() => "");
|
|
574
|
+
return build.includes("kover") ? null : tool;
|
|
575
|
+
}
|
|
576
|
+
return null;
|
|
577
|
+
}
|
|
337
578
|
// Run a native test command, showing its output (on stderr with --json) and
|
|
338
579
|
// keeping it. Go's JSON events are shown as the text they carry.
|
|
339
580
|
function spawned(command, args, cwd, env, goJson = false) {
|
|
@@ -374,7 +615,7 @@ async function main() {
|
|
|
374
615
|
}
|
|
375
616
|
if (!verb || ["help", "--help", "-h"].includes(verb)) {
|
|
376
617
|
output(
|
|
377
|
-
"LawSpec " + VERSION + "\nUsage: lawspec init --target <language> [--project <directory>] [--minify]\n lawspec check | doctor | explain <unit>::<law> | generate\n lawspec evidence [<unit> | <unit>::<declaration>]\n lawspec test [--target <language>] [--fresh] [--seed <n>]\n lawspec package [--project <package directory>]\n lawspec examples [--example payments] [--target <language>] [--output <directory>]\nOptions: --config <path>, --target <language>, --machine-bits <32|64>, --json, --no-cache\nGeneration: --dry-run, --check, --minify\nTargets: " +
|
|
618
|
+
"LawSpec " + VERSION + "\nUsage: lawspec init --target <language> [--project <directory>] [--minify]\n lawspec check | doctor | explain <unit>::<law> | generate\n lawspec evidence [<unit> | <unit>::<declaration>]\n lawspec test [--target <language>] [--fresh] [--seed <n>] [--update-recorded] [--tag <t>] [--exclude-tag <t>] [--report junit=<path>] [--coverage] [--benchmarks]\n lawspec package [--project <package directory>]\n lawspec examples [--example payments] [--target <language>] [--output <directory>]\nOptions: --config <path>, --target <language>, --machine-bits <32|64>, --json, --no-cache\nGeneration: --dry-run, --check, --minify\nTargets: " +
|
|
378
619
|
targets.join(", "),
|
|
379
620
|
);
|
|
380
621
|
return;
|
|
@@ -418,8 +659,9 @@ async function main() {
|
|
|
418
659
|
if (options.example) throw new Error("--example is only supported by examples");
|
|
419
660
|
if (options.minify && !["init", "generate", "examples", "test"].includes(verb))
|
|
420
661
|
throw new Error("--minify applies to init, generate, test and examples");
|
|
421
|
-
if ((options.fresh || options.seed !== undefined
|
|
422
|
-
|
|
662
|
+
if ((options.fresh || options.seed !== undefined || options.tag || options["exclude-tag"] ||
|
|
663
|
+
options.report !== undefined || options.coverage || options["update-recorded"] || options.benchmarks) && verb !== "test")
|
|
664
|
+
throw new Error("--fresh, --seed, --update-recorded, --tag, --exclude-tag, --report, --coverage and --benchmarks apply to test");
|
|
423
665
|
if (options.seed !== undefined && !/^[0-9]+$/.test(options.seed))
|
|
424
666
|
throw new Error("--seed must be a whole number");
|
|
425
667
|
if (verb === "init") return init();
|
|
@@ -476,13 +718,24 @@ async function main() {
|
|
|
476
718
|
["proved", "PROVED"],
|
|
477
719
|
["exhaustively-checked", "EXHAUSTIVELY CHECKED"],
|
|
478
720
|
["property-tested", "PROPERTY TESTED"],
|
|
721
|
+
["measured", "MEASURED"],
|
|
479
722
|
["runtime-checked", "RUNTIME CHECKED"],
|
|
723
|
+
["default-handler", "DEFAULT HANDLER"],
|
|
480
724
|
["assumed", "ASSUMED / EXTERNAL"],
|
|
725
|
+
// The harness plane: a law it marks as known to fail, a law whose last
|
|
726
|
+
// run was flaky, and a law whose tests it skips.
|
|
727
|
+
["known-failing", "KNOWN FAILING"],
|
|
728
|
+
["flaky", "FLAKY"],
|
|
729
|
+
["skipped", "SKIPPED"],
|
|
481
730
|
];
|
|
482
731
|
const evidenceSummary = (evidence) => {
|
|
483
732
|
if (!evidence.length) return "";
|
|
484
733
|
const count = (status) => evidence.filter((item) => item.status === status).length;
|
|
485
|
-
|
|
734
|
+
// The harness statuses (after assumed) are counted only when a harness
|
|
735
|
+
// gives some.
|
|
736
|
+
const harnessFrom = statuses.findIndex(([status]) => status === "known-failing");
|
|
737
|
+
const shown = statuses.filter(([status], i) => i < harnessFrom || count(status));
|
|
738
|
+
return ` Evidence: ${shown.map(([status, label]) => `${count(status)} ${label.toLowerCase()}`).join(", ")}.`;
|
|
486
739
|
};
|
|
487
740
|
const obligationName = (item) => item.declaration.replace("::law::", "::");
|
|
488
741
|
if (verb === "check") {
|
|
@@ -512,10 +765,21 @@ async function main() {
|
|
|
512
765
|
evidence = evidence.filter((item) => !positional[0] ||
|
|
513
766
|
obligationName(item) === positional[0] || item.owner === positional[0]);
|
|
514
767
|
if (positional[0] && !evidence.length) throw new Error("No matching obligation");
|
|
768
|
+
// The last run's outcome (lawspec test): a law that failed, then passed
|
|
769
|
+
// on a retry, is flaky; what its generated cases covered is its adequacy.
|
|
770
|
+
const runs = await lastRuns();
|
|
771
|
+
evidence = evidence.map((item) => {
|
|
772
|
+
const run = item.stage === "law" ? runs.get(item.declaration) : undefined;
|
|
773
|
+
if (!run) return item;
|
|
774
|
+
return { ...item, ...(run.adequacy ? { adequacy: run.adequacy } : {}),
|
|
775
|
+
...(run.flaky && ["property-tested", "exhaustively-checked"].includes(item.status)
|
|
776
|
+
? { status: "flaky", reason: `${item.reason}; its last run failed, then passed on a retry` } : {}) };
|
|
777
|
+
});
|
|
778
|
+
const harnessed = evidence.filter((item) => item.harness || item.adequacy);
|
|
515
779
|
output(
|
|
516
780
|
options.json
|
|
517
781
|
? evidence
|
|
518
|
-
: statuses
|
|
782
|
+
: [...statuses
|
|
519
783
|
.map(([status, label]) => {
|
|
520
784
|
const items = evidence.filter((item) => item.status === status);
|
|
521
785
|
if (!items.length) return null;
|
|
@@ -523,7 +787,10 @@ async function main() {
|
|
|
523
787
|
` ${item.stage} ${obligationName(item)}${item.target ? ` [${item.target}]` : ""}` +
|
|
524
788
|
(item.claim ? `: ${item.claim.text}` : "") + `\n ${item.reason}`).join("\n");
|
|
525
789
|
})
|
|
526
|
-
.filter((section) => section !== null)
|
|
790
|
+
.filter((section) => section !== null),
|
|
791
|
+
// How each law is discharged: the harness plane, apart from what
|
|
792
|
+
// the laws say.
|
|
793
|
+
...(harnessed.length ? [`HARNESS (${harnessed.length})\n` + harnessed.map(harnessText).join("\n")] : [])]
|
|
527
794
|
.join("\n\n"),
|
|
528
795
|
);
|
|
529
796
|
return;
|
|
@@ -542,7 +809,8 @@ async function main() {
|
|
|
542
809
|
: indices
|
|
543
810
|
.map(
|
|
544
811
|
({ e, i }) =>
|
|
545
|
-
`${e.owner}::${e.name}\n${e.trace.join("\n=> ")}\n=> ${result.expansions[i]}
|
|
812
|
+
`${e.owner}::${e.name}\n${e.trace.join("\n=> ")}\n=> ${result.expansions[i]}` +
|
|
813
|
+
explainAbilities(e, result.expansions[i], result.units) + explainExamples(e),
|
|
546
814
|
)
|
|
547
815
|
.join("\n\n"),
|
|
548
816
|
);
|