lawspec 0.21.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +322 -2
- package/README.md +3 -3
- package/beam-coverage.mjs +160 -0
- package/beam-doctor.mjs +306 -0
- package/bin/lawspec.mjs +150 -41
- package/build.json +359 -101
- package/compatibility.json +25 -0
- package/core.wasm +2 -2
- package/doctor.mjs +29 -2
- package/examples/native-payments/csharp/lawspec.json +51 -0
- package/examples/native-payments/csharp/src/Domain/Payments.cs +19 -0
- package/examples/native-payments/csharp/tests/Native/PaymentGenerators.cs +14 -0
- package/examples/native-payments/elixir/lawspec.json +149 -0
- package/examples/native-payments/elixir/lib/payments_domain.ex +54 -0
- package/examples/native-payments/elixir/test/support/payment_generators.ex +9 -0
- package/examples/native-payments/erlang/include/payments_domain.hrl +2 -0
- package/examples/native-payments/erlang/lawspec.json +148 -0
- package/examples/native-payments/erlang/src/payments_domain.erl +21 -0
- package/examples/native-payments/erlang/test/payment_generators.erl +10 -0
- package/examples/native-payments/fsharp/lawspec.json +167 -0
- package/examples/native-payments/fsharp/src/Native/Types/PaymentsDomain.fs +23 -0
- package/examples/native-payments/fsharp/tests/Native/PaymentGenerators.fs +11 -0
- package/examples/native-payments/gleam/lawspec.json +149 -0
- package/examples/native-payments/gleam/prepare.mjs +48 -0
- package/examples/native-payments/gleam/src/payments_domain.gleam +32 -0
- package/examples/native-payments/gleam/test/payment_generators.gleam +13 -0
- package/examples/specs/collections.lawspec +44 -0
- package/examples/specs/composition.lawspec +110 -0
- package/examples/specs/contracts.lawspec +124 -0
- package/examples/specs/generation.lawspec +11 -0
- package/examples/specs/records.lawspec +191 -0
- package/examples/specs/records_inner.lawspec +10 -0
- package/examples/specs/sequences.lawspec +86 -0
- package/failure-database.mjs +46 -0
- package/index.d.ts +45 -1
- package/native-examples.mjs +7 -2
- package/package.json +2 -2
- package/templates.mjs +134 -3
- package/test-command.mjs +188 -17
package/bin/lawspec.mjs
CHANGED
|
@@ -8,9 +8,11 @@ import { targets, templates, commands, setup } from "../templates.mjs";
|
|
|
8
8
|
import { generateExamples } from "../examples-command.mjs";
|
|
9
9
|
import { showScalar } from "../scalars.mjs";
|
|
10
10
|
import { doctor } from "../doctor.mjs";
|
|
11
|
+
import { beamCoverageRequirement, collectBeamCoverage } from "../beam-coverage.mjs";
|
|
12
|
+
import { failureDatabase, projectKey } from "../failure-database.mjs";
|
|
11
13
|
import { spawn, spawnSync } from "node:child_process";
|
|
12
|
-
import { benchmarkInvocations, environmentDigest, executedTests, invocations, lawKeys, projectDigest, recordedDigest, selectByTags, mergeJunit,
|
|
13
|
-
harnessStatistics, coverageTools, junitFromTests } from "../test-command.mjs";
|
|
14
|
+
import { benchmarkInvocations, preparationInvocations, environmentDigest, executedTests, invocations, lawKeys, projectDigest, recordedDigest, selectByTags, mergeJunit,
|
|
15
|
+
harnessStatistics, coverageTools, junitFromTests, testsToRun } from "../test-command.mjs";
|
|
14
16
|
import {
|
|
15
17
|
readOptional,
|
|
16
18
|
planWrites,
|
|
@@ -18,14 +20,14 @@ import {
|
|
|
18
20
|
atomicWrite,
|
|
19
21
|
safePath,
|
|
20
22
|
} from "../files.mjs";
|
|
21
|
-
const VERSION = "0.
|
|
23
|
+
const VERSION = "0.23.0";
|
|
22
24
|
const args = process.argv.slice(2);
|
|
23
25
|
const verb = args.shift();
|
|
24
26
|
const options = {};
|
|
25
27
|
const positional = [];
|
|
26
28
|
for (let i = 0; i < args.length; i++) {
|
|
27
29
|
const arg = args[i];
|
|
28
|
-
if (["--target", "--project", "--config", "--output", "--machine-bits", "--example", "--seed", "--report"].includes(arg)) {
|
|
30
|
+
if (["--target", "--project", "--config", "--output", "--machine-bits", "--example", "--seed", "--report", "--attestation"].includes(arg)) {
|
|
29
31
|
if (!args[i + 1] || args[i + 1].startsWith("--"))
|
|
30
32
|
throw new Error(`Missing value for ${arg}`);
|
|
31
33
|
options[arg.slice(2)] = args[++i];
|
|
@@ -197,6 +199,9 @@ async function init() {
|
|
|
197
199
|
"build.gradle",
|
|
198
200
|
"build.gradle.kts",
|
|
199
201
|
"settings.gradle.kts",
|
|
202
|
+
"rebar.config",
|
|
203
|
+
"mix.exs",
|
|
204
|
+
"gleam.toml",
|
|
200
205
|
];
|
|
201
206
|
const hasBuild = (await readdir(root)).some(
|
|
202
207
|
(f) => buildFiles.includes(f) || f.endsWith(".cabal"),
|
|
@@ -267,19 +272,49 @@ function explainExamples(law) {
|
|
|
267
272
|
ex.expectations.map(e => ` expect ${showAssertion(e)}`).join("\n")
|
|
268
273
|
).join("\n");
|
|
269
274
|
}
|
|
270
|
-
// What lawspec test last recorded for each law, across targets
|
|
275
|
+
// What lawspec test last recorded for each law, across targets, and the
|
|
276
|
+
// environment attestations its runs were given, by run id. A law passed in
|
|
277
|
+
// an attested run names that run; across targets, its latest passing run.
|
|
271
278
|
async function lastRuns() {
|
|
272
279
|
const folder = path.join(configRoot, ".lawspec", "results");
|
|
273
280
|
const runs = new Map();
|
|
281
|
+
const attestations = new Map();
|
|
274
282
|
for (const name of (await readOptionalDirectory(folder)) ?? []) {
|
|
275
283
|
if (!name.endsWith(".json")) continue;
|
|
276
284
|
const recorded = JSON.parse((await readOptional(path.join(folder, name))) ?? "{}");
|
|
285
|
+
for (const [id, attestation] of Object.entries(recorded.attestations ?? {})) attestations.set(id, attestation);
|
|
277
286
|
for (const [law, run] of Object.entries(recorded.laws ?? {})) {
|
|
278
287
|
const known = runs.get(law) ?? {};
|
|
279
|
-
|
|
288
|
+
const latest = (known.passed ?? "") > (run.passed ?? "") ? known : run;
|
|
289
|
+
runs.set(law, { flaky: known.flaky || run.flaky, adequacy: run.adequacy ?? known.adequacy,
|
|
290
|
+
passed: latest.passed, environment: latest.environment });
|
|
280
291
|
}
|
|
281
292
|
}
|
|
282
|
-
return runs;
|
|
293
|
+
return { runs, attestations };
|
|
294
|
+
}
|
|
295
|
+
// An environment attestation: a versioned JSON document any tool may write,
|
|
296
|
+
// recording for one run where each dependency was wired and whether it was
|
|
297
|
+
// the real dependency or a stand-in. --attestation names one, or lawspec.json
|
|
298
|
+
// does; the compiler checks it. It is reporting only and changes no evidence.
|
|
299
|
+
async function readAttestation(config) {
|
|
300
|
+
const file = options.attestation !== undefined ? path.resolve(options.attestation)
|
|
301
|
+
: config.attestation !== undefined ? path.resolve(configRoot, config.attestation) : null;
|
|
302
|
+
if (file === null) return null;
|
|
303
|
+
const text = await readOptional(file);
|
|
304
|
+
if (text === null) throw new Error(`No environment attestation at ${file}`);
|
|
305
|
+
try {
|
|
306
|
+
return JSON.parse(text);
|
|
307
|
+
} catch (error) {
|
|
308
|
+
throw new Error(`${file}: the environment attestation is not JSON: ${error.message}`);
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
// One attested dependency, as lawspec evidence shows it.
|
|
312
|
+
function dependencyText(d) {
|
|
313
|
+
const level = (c) => /^[0-9]+$/.test(c) ? `conformance level ${c}` : `conformance ${c}`;
|
|
314
|
+
return `${d.dependency}: ` + (d.standIn
|
|
315
|
+
? `stand-in ${d.standIn.identity} ${d.standIn.version} (${level(d.standIn.conformance)}` +
|
|
316
|
+
(d.standIn.historyDigest ? `, history ${d.standIn.historyDigest}` : "") + ")"
|
|
317
|
+
: "real") + ` at ${d.endpoint}` + (d.reached ? "" : " (not reached when recorded)");
|
|
283
318
|
}
|
|
284
319
|
// One law's harness, as lawspec evidence shows it.
|
|
285
320
|
function harnessText(item) {
|
|
@@ -353,12 +388,18 @@ async function runTests(compiler, input, selected, roots, config) {
|
|
|
353
388
|
const summaries = [];
|
|
354
389
|
const junit = [];
|
|
355
390
|
const report = options.report === undefined ? null : parseReport(options.report);
|
|
391
|
+
const attestation = await readAttestation(config);
|
|
356
392
|
for (const [i, target] of selected.entries()) {
|
|
357
393
|
const root = roots[i];
|
|
358
394
|
const planned = diagnostics(await compiler.planGeneration({
|
|
359
395
|
...input, target: target.language, sourceDir: target.sourceDir, testDir: target.testDir,
|
|
360
396
|
nativeBindings: target.nativeBindings, minify: options.minify === true,
|
|
397
|
+
...(attestation ? { attestation } : {}),
|
|
361
398
|
}));
|
|
399
|
+
// The laws the attestation touches, each with how its dependencies
|
|
400
|
+
// were wired (the run's id and times aside).
|
|
401
|
+
const attested = new Map((planned.evidence ?? []).filter((item) => item.against)
|
|
402
|
+
.map((item) => [item.declaration, JSON.stringify([item.environment, item.against])]));
|
|
362
403
|
const pending = await planWrites(root, planned.files);
|
|
363
404
|
if (pending.changes.length)
|
|
364
405
|
throw new Error(`${target.language}: generated files are out of date; run lawspec generate${options.minify ? " --minify" : ""}`);
|
|
@@ -371,20 +412,20 @@ async function runTests(compiler, input, selected, roots, config) {
|
|
|
371
412
|
// Recorded values are spec data: a changed recording runs its laws again.
|
|
372
413
|
environment: await environmentDigest(root, report_) + (await recordedDigest(recordedFolder)),
|
|
373
414
|
project: await projectDigest(root, generated),
|
|
415
|
+
attested,
|
|
374
416
|
});
|
|
375
|
-
const
|
|
376
|
-
|
|
417
|
+
const identity = projectKey(root);
|
|
418
|
+
const resultsFile = path.join(configRoot, ".lawspec", "results", `${target.language}-${identity}.json`);
|
|
377
419
|
const previous = JSON.parse((await readOptional(resultsFile)) ?? '{"version":1,"laws":{}}');
|
|
420
|
+
const {directory: failures, file: databaseFile, database} = await failureDatabase(configRoot, target.language, root);
|
|
378
421
|
// Recording again runs every law, so each records what it sees now.
|
|
379
422
|
const chosen = selectByTags(planned.tests, options.tag ?? [], options["exclude-tag"] ?? []);
|
|
380
|
-
const stale = chosen
|
|
423
|
+
const stale = testsToRun(chosen, keys, previous.laws, database,
|
|
424
|
+
{fresh: options.fresh, coverage: options.coverage, updateRecorded: options["update-recorded"]});
|
|
381
425
|
// The failure database: each law whose last run failed, with the seed
|
|
382
426
|
// that exposed it. Those laws run first, with that seed, so the failing
|
|
383
427
|
// inputs are generated again; the runtimes keep their own counterexample
|
|
384
428
|
// databases beside it (Hypothesis, proptest).
|
|
385
|
-
const failures = path.join(configRoot, ".lawspec", "failures", target.language);
|
|
386
|
-
const databaseFile = path.join(failures, "laws.json");
|
|
387
|
-
const database = JSON.parse((await readOptional(databaseFile)) ?? "{}");
|
|
388
429
|
const replayed = options.seed === undefined ? stale.filter((entry) => database[entry.law]) : [];
|
|
389
430
|
const batches = [
|
|
390
431
|
...[...new Set(replayed.map((entry) => database[entry.law].seed))].map((s) =>
|
|
@@ -394,19 +435,24 @@ async function runTests(compiler, input, selected, roots, config) {
|
|
|
394
435
|
const passed = [];
|
|
395
436
|
const unrun = [];
|
|
396
437
|
let failed = false;
|
|
397
|
-
const scratch = path.join(configRoot, ".lawspec", "reports", target.language);
|
|
438
|
+
const scratch = path.join(configRoot, ".lawspec", "reports", target.language, identity);
|
|
398
439
|
const stats = path.join(scratch, "statistics");
|
|
399
|
-
const
|
|
440
|
+
const beam = ["erlang", "elixir", "gleam"].includes(target.language);
|
|
441
|
+
const coverage = options.coverage ? path.join(configRoot, ".lawspec", "coverage", target.language, identity) : null;
|
|
442
|
+
const coverageRuns = [];
|
|
443
|
+
let coverageWritten = false, coverageError;
|
|
400
444
|
await rm(scratch, { recursive: true, force: true });
|
|
401
445
|
await mkdir(stats, { recursive: true });
|
|
402
446
|
await mkdir(failures, { recursive: true });
|
|
403
|
-
await writeFile(path.join(
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
447
|
+
await writeFile(path.join(configRoot, ".lawspec", "reports", ".gitignore"), "*\n");
|
|
448
|
+
const missingCoverage = coverage ? await coverageMissing(target, root, report_) : null;
|
|
449
|
+
const useCoverage = coverage && !missingCoverage;
|
|
450
|
+
if (missingCoverage)
|
|
451
|
+
console.error(`${target.language}: --coverage needs ${missingCoverage.tool}, which is not available; ${missingCoverage.install}. Running without coverage.`);
|
|
452
|
+
if (useCoverage) {
|
|
453
|
+
if (beam) await rm(coverage, {recursive: true, force: true});
|
|
454
|
+
await mkdir(beam ? path.join(coverage, "runs") : coverage, { recursive: true });
|
|
408
455
|
}
|
|
409
|
-
const useCoverage = coverage && !(await coverageMissing(target, root));
|
|
410
456
|
// parallel on Python needs pytest-xdist; without it the tests run one
|
|
411
457
|
// after another, and lawspec test says so.
|
|
412
458
|
let xdist = false;
|
|
@@ -414,21 +460,30 @@ async function runTests(compiler, input, selected, roots, config) {
|
|
|
414
460
|
xdist = spawnSync(target.python || "python3", ["-c", "import xdist"], { cwd: root, stdio: "ignore" }).status === 0;
|
|
415
461
|
if (!xdist) console.error("python: `parallel` runs tests at the same time only with pytest-xdist (pip install pytest-xdist); without it they run one after another.");
|
|
416
462
|
}
|
|
417
|
-
// A skipped law runs nothing
|
|
418
|
-
//
|
|
419
|
-
const expected = (entry) => !entry.skip && !entry.knownFailing;
|
|
463
|
+
// A skipped law runs nothing. BEAM known-failing checks finish as native
|
|
464
|
+
// passing tests, so their completion must appear in the native report.
|
|
465
|
+
const expected = (entry) => !entry.skip && (!entry.knownFailing || ["erlang", "elixir", "gleam"].includes(target.language));
|
|
466
|
+
const benchmarking = options.benchmarks ? (planned.benchmarks ?? []) : [];
|
|
467
|
+
if (batches.length || benchmarking.length) {
|
|
468
|
+
for (const preparation of preparationInvocations(target, planned.files)) {
|
|
469
|
+
const {ok} = await spawned(preparation.command, preparation.args, root, process.env);
|
|
470
|
+
if (!ok) throw new Error(`${target.language}: native test preparation failed`);
|
|
471
|
+
}
|
|
472
|
+
}
|
|
420
473
|
const seedOf = new Map();
|
|
421
474
|
batches: for (const batch of batches) {
|
|
422
|
-
for (const run of invocations(target, batch.entries, { offline, scratch, coverage: useCoverage ? coverage : null,
|
|
475
|
+
for (const run of invocations(target, batch.entries, { offline, scratch, coverage: useCoverage ? coverage : null,
|
|
476
|
+
xdist, root, coverageConfig: report_.coverage })) {
|
|
423
477
|
const since = Date.now() - 1000;
|
|
424
478
|
const { ok, output } = await spawned(run.command, run.args, root,
|
|
425
479
|
{ ...process.env, ...run.env, LAWSPEC_SEED: batch.seed, HSPEC_SEED: batch.seed,
|
|
426
480
|
LAWSPEC_STATS: stats, LAWSPEC_FAILURES: failures, LAWSPEC_RECORDED: recordedFolder,
|
|
427
481
|
...(options["update-recorded"] ? { LAWSPEC_UPDATE_RECORDED: "1" } : {}) }, run.report?.kind === "go-json");
|
|
482
|
+
if (run.coverage) coverageRuns.push(run.coverage);
|
|
428
483
|
const executed = await executedTests(run.report, output, root, since);
|
|
429
484
|
for (const law of run.laws) seedOf.set(law.law, batch.seed);
|
|
430
|
-
if (report) junit.push({ target: target.language
|
|
431
|
-
if (!ok) { failed = true; break batches; }
|
|
485
|
+
if (report) junit.push({ target: `${target.language} (${target.root})`, xml: await junitOf(run, executed, root, since) });
|
|
486
|
+
if (!ok || executed.some((test) => test.status === "failed")) { failed = true; break batches; }
|
|
432
487
|
// A runner whose filter matched nothing reports success, so a law
|
|
433
488
|
// counts as passed only if the runner's report shows its tests ran.
|
|
434
489
|
const ran = new Set(executed.flatMap(run.ran ?? (() => [])));
|
|
@@ -437,9 +492,16 @@ async function runTests(compiler, input, selected, roots, config) {
|
|
|
437
492
|
}
|
|
438
493
|
}
|
|
439
494
|
if (unrun.length) failed = true;
|
|
495
|
+
if (useCoverage && beam) {
|
|
496
|
+
try {
|
|
497
|
+
coverageWritten = !!(await collectBeamCoverage({root, directory: coverage, runs: coverageRuns}));
|
|
498
|
+
} catch (error) {
|
|
499
|
+
failed = true;
|
|
500
|
+
coverageError = error.message;
|
|
501
|
+
}
|
|
502
|
+
}
|
|
440
503
|
// --benchmarks: the harness's benchmarks run after the laws, every time;
|
|
441
504
|
// they are measured, never asserted, and never cached.
|
|
442
|
-
const benchmarking = options.benchmarks ? (planned.benchmarks ?? []) : [];
|
|
443
505
|
if (!failed) for (const run of benchmarkInvocations(target, benchmarking, { offline })) {
|
|
444
506
|
const { ok } = await spawned(run.command, run.args, root,
|
|
445
507
|
{ ...process.env, ...run.env, LAWSPEC_SEED: seed, HSPEC_SEED: seed, LAWSPEC_STATS: stats, LAWSPEC_RECORDED: recordedFolder }, false);
|
|
@@ -451,6 +513,8 @@ async function runTests(compiler, input, selected, roots, config) {
|
|
|
451
513
|
const label = entry.label;
|
|
452
514
|
const observed = statistics.filter((s) => s.law === label);
|
|
453
515
|
laws[entry.law] = { key: keys.get(entry.law), seed: Number(seedOf.get(entry.law) ?? seed), passed: new Date().toISOString(),
|
|
516
|
+
// A law the attestation touches names the run it passed in.
|
|
517
|
+
...(attestation && attested.has(entry.law) ? { environment: attestation.run.id } : {}),
|
|
454
518
|
...(observed.some((s) => s.outcome === "flaky") ? { flaky: true } : {}),
|
|
455
519
|
...(observed.some((s) => s.cover || s.labels || s.classes) ? { adequacy: observed.filter((s) => s.cover || s.labels || s.classes)
|
|
456
520
|
.map(({ test, cases, cover, classes, labels }) => ({ test, cases, cover, classes, labels })) } : {}) };
|
|
@@ -458,19 +522,28 @@ async function runTests(compiler, input, selected, roots, config) {
|
|
|
458
522
|
// A law that failed is recorded with its seed; one that passed leaves.
|
|
459
523
|
for (const entry of stale) {
|
|
460
524
|
if (passed.includes(entry)) delete database[entry.law];
|
|
461
|
-
else if (seedOf.has(entry.law))
|
|
525
|
+
else if (seedOf.has(entry.law)) {
|
|
526
|
+
delete laws[entry.law];
|
|
527
|
+
database[entry.law] = { seed: Number(seedOf.get(entry.law)), failed: new Date().toISOString() };
|
|
528
|
+
}
|
|
462
529
|
}
|
|
463
|
-
await writeFile(path.join(
|
|
530
|
+
await writeFile(path.join(configRoot, ".lawspec", "failures", ".gitignore"), "*\n");
|
|
464
531
|
await writeFile(databaseFile, JSON.stringify(database, null, 2) + "\n");
|
|
465
532
|
await mkdir(path.dirname(resultsFile), { recursive: true });
|
|
466
533
|
await writeFile(path.join(path.dirname(resultsFile), ".gitignore"), "*\n");
|
|
467
|
-
|
|
534
|
+
// Each attestation a recorded law ran under is kept with the results.
|
|
535
|
+
const attestations = Object.fromEntries(Object.entries({ ...(previous.attestations ?? {}),
|
|
536
|
+
...(attestation ? { [attestation.run.id]: attestation } : {}) })
|
|
537
|
+
.filter(([id]) => Object.values(laws).some((run) => run.environment === id)));
|
|
538
|
+
await writeFile(resultsFile, JSON.stringify({ version: 1, laws,
|
|
539
|
+
...(Object.keys(attestations).length ? { attestations } : {}) }, null, 2) + "\n");
|
|
468
540
|
const flaky = statistics.filter((s) => s.outcome === "flaky").map((s) => s.law);
|
|
469
541
|
const unmet = statistics.flatMap((s) => (s.cover ?? []).filter((c) => !c.met).map((c) => `${s.law}: cover ${c.required}% "${c.label}" (${c.observed}%)`));
|
|
470
542
|
const benchmarks = statistics.filter((s) => s.benchmark);
|
|
471
543
|
// How each parallel unit's tests actually ran at the same time.
|
|
472
544
|
const parallelism = statistics.filter((s) => s.parallel).map(({ parallel, mode, workers }) => ({ unit: parallel, mode, workers }));
|
|
473
|
-
summaries.push({ target: target.language, seed: Number(seed), ran: stale.map((e) => e.law),
|
|
545
|
+
summaries.push({ target: target.language, root: target.root, seed: Number(seed), ran: stale.map((e) => e.law),
|
|
546
|
+
...(attestation ? { environment: attestation.environment } : {}),
|
|
474
547
|
unchanged: chosen.length - stale.length, ok: !failed,
|
|
475
548
|
...(chosen.length !== planned.tests.length ? { deselected: planned.tests.length - chosen.length } : {}),
|
|
476
549
|
...(unrun.length ? { unrun: unrun.map((e) => e.law) } : {}),
|
|
@@ -478,14 +551,15 @@ async function runTests(compiler, input, selected, roots, config) {
|
|
|
478
551
|
...(unmet.length ? { unmetCover: unmet } : {}),
|
|
479
552
|
...(benchmarks.length ? { benchmarks } : {}),
|
|
480
553
|
...(parallelism.length ? { parallelism } : {}),
|
|
481
|
-
...(
|
|
554
|
+
...(coverageError ? {coverageError} : {}),
|
|
555
|
+
...(useCoverage && (!beam || coverageWritten) ? { coverage: path.relative(process.cwd(), coverage) || "." } : {}) });
|
|
482
556
|
}
|
|
483
557
|
if (report) {
|
|
484
558
|
await mkdir(path.dirname(path.resolve(report.path)), { recursive: true });
|
|
485
559
|
await writeFile(path.resolve(report.path), mergeJunit(junit));
|
|
486
560
|
}
|
|
487
561
|
output(options.json ? summaries : summaries.map((s) =>
|
|
488
|
-
`${s.target}: ${s.ran.length ? `ran ${s.ran.length} law(s) with seed ${s.seed}` : "nothing to run"}` +
|
|
562
|
+
`${s.target}${summaries.filter(other => other.target === s.target).length > 1 ? ` (${s.root})` : ""}: ${s.ran.length ? `ran ${s.ran.length} law(s) with seed ${s.seed}${s.environment ? ` in environment ${s.environment}` : ""}` : "nothing to run"}` +
|
|
489
563
|
`, ${s.unchanged} unchanged since their last passing run.${s.ok ? "" : " FAILED"}` +
|
|
490
564
|
(s.deselected ? `\n${s.deselected} law(s) not selected by --tag or --exclude-tag.` : "") +
|
|
491
565
|
(s.unrun ? `\nNo tests ran for ${s.unrun.join(", ")}; the runner matched none of their tests.` : "") +
|
|
@@ -493,6 +567,7 @@ async function runTests(compiler, input, selected, roots, config) {
|
|
|
493
567
|
(s.unmetCover ? `\nCover not met: ${s.unmetCover.join("; ")}` : "") +
|
|
494
568
|
(s.parallelism ? "\n" + s.parallelism.map((p) => `parallel ${p.unit}: ${p.mode}, ${p.workers} worker(s)`).join("\n") : "") +
|
|
495
569
|
(s.benchmarks ? "\n" + s.benchmarks.map((b) => `benchmark ${b.benchmark}: mean ${(b.mean_ns / 1000).toFixed(2)} us over ${b.iterations} iteration(s)`).join("\n") : "") +
|
|
570
|
+
(s.coverageError ? `\nCoverage failed: ${s.coverageError}` : "") +
|
|
496
571
|
(s.coverage ? `\nCoverage written to ${s.coverage}.` : "")).join("\n"));
|
|
497
572
|
if (summaries.some((s) => !s.ok)) process.exitCode = 1;
|
|
498
573
|
}
|
|
@@ -515,10 +590,14 @@ async function junitOf(run, executed, root, since) {
|
|
|
515
590
|
}
|
|
516
591
|
return xml.join("\n");
|
|
517
592
|
}
|
|
518
|
-
|
|
593
|
+
// BEAM reports only executed cases. Intentional harness skips remain
|
|
594
|
+
// reportable obligations; they are never evidence that a test passed.
|
|
595
|
+
const skipped = run.report?.kind === "beam-events" ? run.laws.filter((entry) => entry.skip)
|
|
596
|
+
.map((entry) => ({name: entry.label, classname: entry.unit, status: "skipped", reason: entry.skip})) : [];
|
|
597
|
+
return junitFromTests([...executed, ...skipped]);
|
|
519
598
|
}
|
|
520
599
|
// The tool --coverage needs, if it is missing.
|
|
521
|
-
async function coverageMissing(target, root) {
|
|
600
|
+
async function coverageMissing(target, root, report) {
|
|
522
601
|
const tool = coverageTools[target.language];
|
|
523
602
|
if (!tool) return { tool: "a coverage tool", install: "no coverage tool is known for this target" };
|
|
524
603
|
const probe = (command, args) => new Promise((resolve) => {
|
|
@@ -526,6 +605,12 @@ async function coverageMissing(target, root) {
|
|
|
526
605
|
child.on("error", () => resolve(false));
|
|
527
606
|
child.on("close", (code) => resolve(code === 0));
|
|
528
607
|
});
|
|
608
|
+
if (["erlang", "elixir", "gleam"].includes(target.language)) {
|
|
609
|
+
const requirement = beamCoverageRequirement(target, report);
|
|
610
|
+
if (requirement) return requirement;
|
|
611
|
+
return (await probe("erl", ["-noshell", "-eval",
|
|
612
|
+
'case code:ensure_loaded(cover) of {module,cover} -> halt(0); _ -> halt(1) end.'])) ? null : tool;
|
|
613
|
+
}
|
|
529
614
|
if (target.language === "python") return (await probe(target.python || "python3", tool.check)) ? null : tool;
|
|
530
615
|
if (["javascript", "typescript"].includes(target.language)) return (await probe("npx", ["--no-install", "c8", "--version"])) ? null : tool;
|
|
531
616
|
if (target.language === "rust") return (await probe("cargo", ["llvm-cov", "--version"])) ? null : tool;
|
|
@@ -575,7 +660,7 @@ async function main() {
|
|
|
575
660
|
}
|
|
576
661
|
if (!verb || ["help", "--help", "-h"].includes(verb)) {
|
|
577
662
|
output(
|
|
578
|
-
"LawSpec " + VERSION + "\nUsage: lawspec init --target <language> [--project <directory>] [--minify]\n lawspec check | doctor | explain <unit>::<law> | generate\n lawspec evidence [<unit> | <unit>::<declaration>]\n lawspec test [--target <language>] [--fresh] [--seed <n>] [--update-recorded] [--tag <t>] [--exclude-tag <t>] [--report junit=<path>] [--coverage] [--benchmarks]\n lawspec package [--project <package directory>]\n lawspec examples [--example payments] [--target <language>] [--output <directory>]\nOptions: --config <path>, --target <language>, --machine-bits <32|64>, --json, --no-cache\nGeneration: --dry-run, --check, --minify\nTargets: " +
|
|
663
|
+
"LawSpec " + VERSION + "\nUsage: lawspec init --target <language> [--project <directory>] [--minify]\n lawspec check | doctor | explain <unit>::<law> | generate\n lawspec evidence [<unit> | <unit>::<declaration>] [--attestation <file>]\n lawspec test [--target <language>] [--fresh] [--seed <n>] [--attestation <file>] [--update-recorded] [--tag <t>] [--exclude-tag <t>] [--report junit=<path>] [--coverage] [--benchmarks]\n lawspec package [--project <package directory>]\n lawspec examples [--example payments] [--target <language>] [--output <directory>]\nOptions: --config <path>, --target <language>, --machine-bits <32|64>, --json, --no-cache\nGeneration: --dry-run, --check, --minify\nTargets: " +
|
|
579
664
|
targets.join(", "),
|
|
580
665
|
);
|
|
581
666
|
return;
|
|
@@ -622,6 +707,8 @@ async function main() {
|
|
|
622
707
|
if ((options.fresh || options.seed !== undefined || options.tag || options["exclude-tag"] ||
|
|
623
708
|
options.report !== undefined || options.coverage || options["update-recorded"] || options.benchmarks) && verb !== "test")
|
|
624
709
|
throw new Error("--fresh, --seed, --update-recorded, --tag, --exclude-tag, --report, --coverage and --benchmarks apply to test");
|
|
710
|
+
if (options.attestation !== undefined && !["evidence", "test"].includes(verb))
|
|
711
|
+
throw new Error("--attestation applies to evidence and test");
|
|
625
712
|
if (options.seed !== undefined && !/^[0-9]+$/.test(options.seed))
|
|
626
713
|
throw new Error("--seed must be a whole number");
|
|
627
714
|
if (verb === "init") return init();
|
|
@@ -664,6 +751,8 @@ async function main() {
|
|
|
664
751
|
}
|
|
665
752
|
if (config.machineBits !== undefined && ![32, 64].includes(config.machineBits)) throw new Error("machineBits must be 32 or 64");
|
|
666
753
|
if (config.cache !== undefined && typeof config.cache !== "boolean") throw new Error("cache must be true or false");
|
|
754
|
+
if (config.attestation !== undefined && typeof config.attestation !== "string")
|
|
755
|
+
throw new Error("attestation must be the path of an environment attestation");
|
|
667
756
|
const cache = await cacheDirectory(config);
|
|
668
757
|
const input = {
|
|
669
758
|
...(cache === undefined ? {} : { cacheDirectory: cache }),
|
|
@@ -712,8 +801,23 @@ async function main() {
|
|
|
712
801
|
return;
|
|
713
802
|
}
|
|
714
803
|
if (verb === "evidence") {
|
|
715
|
-
|
|
804
|
+
// An attestation named here, or in lawspec.json, says where the tests
|
|
805
|
+
// ran; otherwise each law shows the attested run it last passed in.
|
|
806
|
+
const named = await readAttestation(config);
|
|
807
|
+
const result = diagnostics(await compiler.check({ ...input, ...(named ? { attestation: named } : {}) }));
|
|
716
808
|
let evidence = result.evidence;
|
|
809
|
+
const { runs, attestations } = await lastRuns();
|
|
810
|
+
const environments = named ? [result.attestation] : [];
|
|
811
|
+
if (!named) for (const [id, attestation] of attestations) {
|
|
812
|
+
const laws = new Set([...runs].filter(([, run]) => run.environment === id).map(([law]) => law));
|
|
813
|
+
if (!laws.size) continue;
|
|
814
|
+
const attested = diagnostics(await compiler.check({ ...input, attestation }));
|
|
815
|
+
const fields = new Map(attested.evidence.filter((item) => item.against && laws.has(item.declaration))
|
|
816
|
+
.map(({ declaration, environment, environmentRun, against }) => [declaration, { environment, environmentRun, against }]));
|
|
817
|
+
evidence = evidence.map((item) => item.stage === "law" && fields.has(item.declaration) ? { ...item, ...fields.get(item.declaration) } : item);
|
|
818
|
+
environments.push({ ...attested.attestation, dependencies: attested.attestation.dependencies
|
|
819
|
+
.map((d) => ({ ...d, obligations: d.obligations.filter((law) => laws.has(law)) })) });
|
|
820
|
+
}
|
|
717
821
|
// Native bindings are configured per target; their obligations are added.
|
|
718
822
|
for (const target of selected) {
|
|
719
823
|
if (target.nativeBindings === undefined) continue;
|
|
@@ -727,7 +831,6 @@ async function main() {
|
|
|
727
831
|
if (positional[0] && !evidence.length) throw new Error("No matching obligation");
|
|
728
832
|
// The last run's outcome (lawspec test): a law that failed, then passed
|
|
729
833
|
// on a retry, is flaky; what its generated cases covered is its adequacy.
|
|
730
|
-
const runs = await lastRuns();
|
|
731
834
|
evidence = evidence.map((item) => {
|
|
732
835
|
const run = item.stage === "law" ? runs.get(item.declaration) : undefined;
|
|
733
836
|
if (!run) return item;
|
|
@@ -745,12 +848,18 @@ async function main() {
|
|
|
745
848
|
if (!items.length) return null;
|
|
746
849
|
return `${label} (${items.length})\n` + items.map((item) =>
|
|
747
850
|
` ${item.stage} ${obligationName(item)}${item.target ? ` [${item.target}]` : ""}` +
|
|
748
|
-
(item.claim ? `: ${item.claim.text}` : "") + `\n ${item.reason}`
|
|
851
|
+
(item.claim ? `: ${item.claim.text}` : "") + `\n ${item.reason}` +
|
|
852
|
+
(item.against ? `\n ran in environment ${item.environment} (run ${item.environmentRun.id}) against ` +
|
|
853
|
+
item.against.map(dependencyText).join("; ") : "")).join("\n");
|
|
749
854
|
})
|
|
750
855
|
.filter((section) => section !== null),
|
|
751
856
|
// How each law is discharged: the harness plane, apart from what
|
|
752
857
|
// the laws say.
|
|
753
|
-
...(harnessed.length ? [`HARNESS (${harnessed.length})\n` + harnessed.map(harnessText).join("\n")] : [])
|
|
858
|
+
...(harnessed.length ? [`HARNESS (${harnessed.length})\n` + harnessed.map(harnessText).join("\n")] : []),
|
|
859
|
+
// Where the tests ran: reporting only, so no status above changes.
|
|
860
|
+
...environments.map((e) => `ENVIRONMENT ${e.environment} (run ${e.run.id}, recorded ${e.run.recordedAt})\n` +
|
|
861
|
+
e.dependencies.map((d) => ` ${dependencyText(d)}; ` + (d.obligations.length
|
|
862
|
+
? `${d.obligations.length} tested obligation(s)` : "no tested obligation relies on it")).join("\n"))]
|
|
754
863
|
.join("\n\n"),
|
|
755
864
|
);
|
|
756
865
|
return;
|