lawspec 0.21.0 → 0.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/CHANGELOG.md +322 -2
  2. package/README.md +3 -3
  3. package/beam-coverage.mjs +160 -0
  4. package/beam-doctor.mjs +306 -0
  5. package/bin/lawspec.mjs +150 -41
  6. package/build.json +359 -101
  7. package/compatibility.json +25 -0
  8. package/core.wasm +2 -2
  9. package/doctor.mjs +29 -2
  10. package/examples/native-payments/csharp/lawspec.json +51 -0
  11. package/examples/native-payments/csharp/src/Domain/Payments.cs +19 -0
  12. package/examples/native-payments/csharp/tests/Native/PaymentGenerators.cs +14 -0
  13. package/examples/native-payments/elixir/lawspec.json +149 -0
  14. package/examples/native-payments/elixir/lib/payments_domain.ex +54 -0
  15. package/examples/native-payments/elixir/test/support/payment_generators.ex +9 -0
  16. package/examples/native-payments/erlang/include/payments_domain.hrl +2 -0
  17. package/examples/native-payments/erlang/lawspec.json +148 -0
  18. package/examples/native-payments/erlang/src/payments_domain.erl +21 -0
  19. package/examples/native-payments/erlang/test/payment_generators.erl +10 -0
  20. package/examples/native-payments/fsharp/lawspec.json +167 -0
  21. package/examples/native-payments/fsharp/src/Native/Types/PaymentsDomain.fs +23 -0
  22. package/examples/native-payments/fsharp/tests/Native/PaymentGenerators.fs +11 -0
  23. package/examples/native-payments/gleam/lawspec.json +149 -0
  24. package/examples/native-payments/gleam/prepare.mjs +48 -0
  25. package/examples/native-payments/gleam/src/payments_domain.gleam +32 -0
  26. package/examples/native-payments/gleam/test/payment_generators.gleam +13 -0
  27. package/examples/specs/collections.lawspec +44 -0
  28. package/examples/specs/composition.lawspec +110 -0
  29. package/examples/specs/contracts.lawspec +124 -0
  30. package/examples/specs/generation.lawspec +11 -0
  31. package/examples/specs/records.lawspec +191 -0
  32. package/examples/specs/records_inner.lawspec +10 -0
  33. package/examples/specs/sequences.lawspec +86 -0
  34. package/failure-database.mjs +46 -0
  35. package/index.d.ts +45 -1
  36. package/native-examples.mjs +7 -2
  37. package/package.json +2 -2
  38. package/templates.mjs +134 -3
  39. package/test-command.mjs +188 -17
package/bin/lawspec.mjs CHANGED
@@ -8,9 +8,11 @@ import { targets, templates, commands, setup } from "../templates.mjs";
8
8
  import { generateExamples } from "../examples-command.mjs";
9
9
  import { showScalar } from "../scalars.mjs";
10
10
  import { doctor } from "../doctor.mjs";
11
+ import { beamCoverageRequirement, collectBeamCoverage } from "../beam-coverage.mjs";
12
+ import { failureDatabase, projectKey } from "../failure-database.mjs";
11
13
  import { spawn, spawnSync } from "node:child_process";
12
- import { benchmarkInvocations, environmentDigest, executedTests, invocations, lawKeys, projectDigest, recordedDigest, selectByTags, mergeJunit,
13
- harnessStatistics, coverageTools, junitFromTests } from "../test-command.mjs";
14
+ import { benchmarkInvocations, preparationInvocations, environmentDigest, executedTests, invocations, lawKeys, projectDigest, recordedDigest, selectByTags, mergeJunit,
15
+ harnessStatistics, coverageTools, junitFromTests, testsToRun } from "../test-command.mjs";
14
16
  import {
15
17
  readOptional,
16
18
  planWrites,
@@ -18,14 +20,14 @@ import {
18
20
  atomicWrite,
19
21
  safePath,
20
22
  } from "../files.mjs";
21
- const VERSION = "0.21.0";
23
+ const VERSION = "0.23.0";
22
24
  const args = process.argv.slice(2);
23
25
  const verb = args.shift();
24
26
  const options = {};
25
27
  const positional = [];
26
28
  for (let i = 0; i < args.length; i++) {
27
29
  const arg = args[i];
28
- if (["--target", "--project", "--config", "--output", "--machine-bits", "--example", "--seed", "--report"].includes(arg)) {
30
+ if (["--target", "--project", "--config", "--output", "--machine-bits", "--example", "--seed", "--report", "--attestation"].includes(arg)) {
29
31
  if (!args[i + 1] || args[i + 1].startsWith("--"))
30
32
  throw new Error(`Missing value for ${arg}`);
31
33
  options[arg.slice(2)] = args[++i];
@@ -197,6 +199,9 @@ async function init() {
197
199
  "build.gradle",
198
200
  "build.gradle.kts",
199
201
  "settings.gradle.kts",
202
+ "rebar.config",
203
+ "mix.exs",
204
+ "gleam.toml",
200
205
  ];
201
206
  const hasBuild = (await readdir(root)).some(
202
207
  (f) => buildFiles.includes(f) || f.endsWith(".cabal"),
@@ -267,19 +272,49 @@ function explainExamples(law) {
267
272
  ex.expectations.map(e => ` expect ${showAssertion(e)}`).join("\n")
268
273
  ).join("\n");
269
274
  }
270
- // What lawspec test last recorded for each law, across targets.
275
+ // What lawspec test last recorded for each law, across targets, and the
276
+ // environment attestations its runs were given, by run id. A law passed in
277
+ // an attested run names that run; across targets, its latest passing run.
271
278
  async function lastRuns() {
272
279
  const folder = path.join(configRoot, ".lawspec", "results");
273
280
  const runs = new Map();
281
+ const attestations = new Map();
274
282
  for (const name of (await readOptionalDirectory(folder)) ?? []) {
275
283
  if (!name.endsWith(".json")) continue;
276
284
  const recorded = JSON.parse((await readOptional(path.join(folder, name))) ?? "{}");
285
+ for (const [id, attestation] of Object.entries(recorded.attestations ?? {})) attestations.set(id, attestation);
277
286
  for (const [law, run] of Object.entries(recorded.laws ?? {})) {
278
287
  const known = runs.get(law) ?? {};
279
- runs.set(law, { flaky: known.flaky || run.flaky, adequacy: run.adequacy ?? known.adequacy });
288
+ const latest = (known.passed ?? "") > (run.passed ?? "") ? known : run;
289
+ runs.set(law, { flaky: known.flaky || run.flaky, adequacy: run.adequacy ?? known.adequacy,
290
+ passed: latest.passed, environment: latest.environment });
280
291
  }
281
292
  }
282
- return runs;
293
+ return { runs, attestations };
294
+ }
295
+ // An environment attestation: a versioned JSON document any tool may write,
296
+ // recording for one run where each dependency was wired and whether it was
297
+ // the real dependency or a stand-in. --attestation names one, or lawspec.json
298
+ // does; the compiler checks it. It is reporting only and changes no evidence.
299
+ async function readAttestation(config) {
300
+ const file = options.attestation !== undefined ? path.resolve(options.attestation)
301
+ : config.attestation !== undefined ? path.resolve(configRoot, config.attestation) : null;
302
+ if (file === null) return null;
303
+ const text = await readOptional(file);
304
+ if (text === null) throw new Error(`No environment attestation at ${file}`);
305
+ try {
306
+ return JSON.parse(text);
307
+ } catch (error) {
308
+ throw new Error(`${file}: the environment attestation is not JSON: ${error.message}`);
309
+ }
310
+ }
311
+ // One attested dependency, as lawspec evidence shows it.
312
+ function dependencyText(d) {
313
+ const level = (c) => /^[0-9]+$/.test(c) ? `conformance level ${c}` : `conformance ${c}`;
314
+ return `${d.dependency}: ` + (d.standIn
315
+ ? `stand-in ${d.standIn.identity} ${d.standIn.version} (${level(d.standIn.conformance)}` +
316
+ (d.standIn.historyDigest ? `, history ${d.standIn.historyDigest}` : "") + ")"
317
+ : "real") + ` at ${d.endpoint}` + (d.reached ? "" : " (not reached when recorded)");
283
318
  }
284
319
  // One law's harness, as lawspec evidence shows it.
285
320
  function harnessText(item) {
@@ -353,12 +388,18 @@ async function runTests(compiler, input, selected, roots, config) {
353
388
  const summaries = [];
354
389
  const junit = [];
355
390
  const report = options.report === undefined ? null : parseReport(options.report);
391
+ const attestation = await readAttestation(config);
356
392
  for (const [i, target] of selected.entries()) {
357
393
  const root = roots[i];
358
394
  const planned = diagnostics(await compiler.planGeneration({
359
395
  ...input, target: target.language, sourceDir: target.sourceDir, testDir: target.testDir,
360
396
  nativeBindings: target.nativeBindings, minify: options.minify === true,
397
+ ...(attestation ? { attestation } : {}),
361
398
  }));
399
+ // The laws the attestation touches, each with how its dependencies
400
+ // were wired (the run's id and times aside).
401
+ const attested = new Map((planned.evidence ?? []).filter((item) => item.against)
402
+ .map((item) => [item.declaration, JSON.stringify([item.environment, item.against])]));
362
403
  const pending = await planWrites(root, planned.files);
363
404
  if (pending.changes.length)
364
405
  throw new Error(`${target.language}: generated files are out of date; run lawspec generate${options.minify ? " --minify" : ""}`);
@@ -371,20 +412,20 @@ async function runTests(compiler, input, selected, roots, config) {
371
412
  // Recorded values are spec data: a changed recording runs its laws again.
372
413
  environment: await environmentDigest(root, report_) + (await recordedDigest(recordedFolder)),
373
414
  project: await projectDigest(root, generated),
415
+ attested,
374
416
  });
375
- const resultsFile = path.join(configRoot, ".lawspec", "results",
376
- `${target.language}-${createHash("sha256").update(root).digest("hex").slice(0, 12)}.json`);
417
+ const identity = projectKey(root);
418
+ const resultsFile = path.join(configRoot, ".lawspec", "results", `${target.language}-${identity}.json`);
377
419
  const previous = JSON.parse((await readOptional(resultsFile)) ?? '{"version":1,"laws":{}}');
420
+ const {directory: failures, file: databaseFile, database} = await failureDatabase(configRoot, target.language, root);
378
421
  // Recording again runs every law, so each records what it sees now.
379
422
  const chosen = selectByTags(planned.tests, options.tag ?? [], options["exclude-tag"] ?? []);
380
- const stale = chosen.filter((entry) => options.fresh || options.coverage || options["update-recorded"] || previous.laws[entry.law]?.key !== keys.get(entry.law));
423
+ const stale = testsToRun(chosen, keys, previous.laws, database,
424
+ {fresh: options.fresh, coverage: options.coverage, updateRecorded: options["update-recorded"]});
381
425
  // The failure database: each law whose last run failed, with the seed
382
426
  // that exposed it. Those laws run first, with that seed, so the failing
383
427
  // inputs are generated again; the runtimes keep their own counterexample
384
428
  // databases beside it (Hypothesis, proptest).
385
- const failures = path.join(configRoot, ".lawspec", "failures", target.language);
386
- const databaseFile = path.join(failures, "laws.json");
387
- const database = JSON.parse((await readOptional(databaseFile)) ?? "{}");
388
429
  const replayed = options.seed === undefined ? stale.filter((entry) => database[entry.law]) : [];
389
430
  const batches = [
390
431
  ...[...new Set(replayed.map((entry) => database[entry.law].seed))].map((s) =>
@@ -394,19 +435,24 @@ async function runTests(compiler, input, selected, roots, config) {
394
435
  const passed = [];
395
436
  const unrun = [];
396
437
  let failed = false;
397
- const scratch = path.join(configRoot, ".lawspec", "reports", target.language);
438
+ const scratch = path.join(configRoot, ".lawspec", "reports", target.language, identity);
398
439
  const stats = path.join(scratch, "statistics");
399
- const coverage = options.coverage ? path.join(configRoot, ".lawspec", "coverage", target.language) : null;
440
+ const beam = ["erlang", "elixir", "gleam"].includes(target.language);
441
+ const coverage = options.coverage ? path.join(configRoot, ".lawspec", "coverage", target.language, identity) : null;
442
+ const coverageRuns = [];
443
+ let coverageWritten = false, coverageError;
400
444
  await rm(scratch, { recursive: true, force: true });
401
445
  await mkdir(stats, { recursive: true });
402
446
  await mkdir(failures, { recursive: true });
403
- await writeFile(path.join(path.dirname(scratch), ".gitignore"), "*\n");
404
- if (coverage) {
405
- await mkdir(coverage, { recursive: true });
406
- const missing = await coverageMissing(target, root);
407
- if (missing) console.error(`${target.language}: --coverage needs ${missing.tool}, which is not available; ${missing.install}. Running without coverage.`);
447
+ await writeFile(path.join(configRoot, ".lawspec", "reports", ".gitignore"), "*\n");
448
+ const missingCoverage = coverage ? await coverageMissing(target, root, report_) : null;
449
+ const useCoverage = coverage && !missingCoverage;
450
+ if (missingCoverage)
451
+ console.error(`${target.language}: --coverage needs ${missingCoverage.tool}, which is not available; ${missingCoverage.install}. Running without coverage.`);
452
+ if (useCoverage) {
453
+ if (beam) await rm(coverage, {recursive: true, force: true});
454
+ await mkdir(beam ? path.join(coverage, "runs") : coverage, { recursive: true });
408
455
  }
409
- const useCoverage = coverage && !(await coverageMissing(target, root));
410
456
  // parallel on Python needs pytest-xdist; without it the tests run one
411
457
  // after another, and lawspec test says so.
412
458
  let xdist = false;
@@ -414,21 +460,30 @@ async function runTests(compiler, input, selected, roots, config) {
414
460
  xdist = spawnSync(target.python || "python3", ["-c", "import xdist"], { cwd: root, stdio: "ignore" }).status === 0;
415
461
  if (!xdist) console.error("python: `parallel` runs tests at the same time only with pytest-xdist (pip install pytest-xdist); without it they run one after another.");
416
462
  }
417
- // A skipped law runs nothing, and a known-failing law's one test is
418
- // expected to fail, so neither is required to show as run.
419
- const expected = (entry) => !entry.skip && !entry.knownFailing;
463
+ // A skipped law runs nothing. BEAM known-failing checks finish as native
464
+ // passing tests, so their completion must appear in the native report.
465
+ const expected = (entry) => !entry.skip && (!entry.knownFailing || ["erlang", "elixir", "gleam"].includes(target.language));
466
+ const benchmarking = options.benchmarks ? (planned.benchmarks ?? []) : [];
467
+ if (batches.length || benchmarking.length) {
468
+ for (const preparation of preparationInvocations(target, planned.files)) {
469
+ const {ok} = await spawned(preparation.command, preparation.args, root, process.env);
470
+ if (!ok) throw new Error(`${target.language}: native test preparation failed`);
471
+ }
472
+ }
420
473
  const seedOf = new Map();
421
474
  batches: for (const batch of batches) {
422
- for (const run of invocations(target, batch.entries, { offline, scratch, coverage: useCoverage ? coverage : null, xdist })) {
475
+ for (const run of invocations(target, batch.entries, { offline, scratch, coverage: useCoverage ? coverage : null,
476
+ xdist, root, coverageConfig: report_.coverage })) {
423
477
  const since = Date.now() - 1000;
424
478
  const { ok, output } = await spawned(run.command, run.args, root,
425
479
  { ...process.env, ...run.env, LAWSPEC_SEED: batch.seed, HSPEC_SEED: batch.seed,
426
480
  LAWSPEC_STATS: stats, LAWSPEC_FAILURES: failures, LAWSPEC_RECORDED: recordedFolder,
427
481
  ...(options["update-recorded"] ? { LAWSPEC_UPDATE_RECORDED: "1" } : {}) }, run.report?.kind === "go-json");
482
+ if (run.coverage) coverageRuns.push(run.coverage);
428
483
  const executed = await executedTests(run.report, output, root, since);
429
484
  for (const law of run.laws) seedOf.set(law.law, batch.seed);
430
- if (report) junit.push({ target: target.language, xml: await junitOf(run, executed, root, since) });
431
- if (!ok) { failed = true; break batches; }
485
+ if (report) junit.push({ target: `${target.language} (${target.root})`, xml: await junitOf(run, executed, root, since) });
486
+ if (!ok || executed.some((test) => test.status === "failed")) { failed = true; break batches; }
432
487
  // A runner whose filter matched nothing reports success, so a law
433
488
  // counts as passed only if the runner's report shows its tests ran.
434
489
  const ran = new Set(executed.flatMap(run.ran ?? (() => [])));
@@ -437,9 +492,16 @@ async function runTests(compiler, input, selected, roots, config) {
437
492
  }
438
493
  }
439
494
  if (unrun.length) failed = true;
495
+ if (useCoverage && beam) {
496
+ try {
497
+ coverageWritten = !!(await collectBeamCoverage({root, directory: coverage, runs: coverageRuns}));
498
+ } catch (error) {
499
+ failed = true;
500
+ coverageError = error.message;
501
+ }
502
+ }
440
503
  // --benchmarks: the harness's benchmarks run after the laws, every time;
441
504
  // they are measured, never asserted, and never cached.
442
- const benchmarking = options.benchmarks ? (planned.benchmarks ?? []) : [];
443
505
  if (!failed) for (const run of benchmarkInvocations(target, benchmarking, { offline })) {
444
506
  const { ok } = await spawned(run.command, run.args, root,
445
507
  { ...process.env, ...run.env, LAWSPEC_SEED: seed, HSPEC_SEED: seed, LAWSPEC_STATS: stats, LAWSPEC_RECORDED: recordedFolder }, false);
@@ -451,6 +513,8 @@ async function runTests(compiler, input, selected, roots, config) {
451
513
  const label = entry.label;
452
514
  const observed = statistics.filter((s) => s.law === label);
453
515
  laws[entry.law] = { key: keys.get(entry.law), seed: Number(seedOf.get(entry.law) ?? seed), passed: new Date().toISOString(),
516
+ // A law the attestation touches names the run it passed in.
517
+ ...(attestation && attested.has(entry.law) ? { environment: attestation.run.id } : {}),
454
518
  ...(observed.some((s) => s.outcome === "flaky") ? { flaky: true } : {}),
455
519
  ...(observed.some((s) => s.cover || s.labels || s.classes) ? { adequacy: observed.filter((s) => s.cover || s.labels || s.classes)
456
520
  .map(({ test, cases, cover, classes, labels }) => ({ test, cases, cover, classes, labels })) } : {}) };
@@ -458,19 +522,28 @@ async function runTests(compiler, input, selected, roots, config) {
458
522
  // A law that failed is recorded with its seed; one that passed leaves.
459
523
  for (const entry of stale) {
460
524
  if (passed.includes(entry)) delete database[entry.law];
461
- else if (seedOf.has(entry.law)) database[entry.law] = { seed: Number(seedOf.get(entry.law)), failed: new Date().toISOString() };
525
+ else if (seedOf.has(entry.law)) {
526
+ delete laws[entry.law];
527
+ database[entry.law] = { seed: Number(seedOf.get(entry.law)), failed: new Date().toISOString() };
528
+ }
462
529
  }
463
- await writeFile(path.join(path.dirname(failures), ".gitignore"), "*\n");
530
+ await writeFile(path.join(configRoot, ".lawspec", "failures", ".gitignore"), "*\n");
464
531
  await writeFile(databaseFile, JSON.stringify(database, null, 2) + "\n");
465
532
  await mkdir(path.dirname(resultsFile), { recursive: true });
466
533
  await writeFile(path.join(path.dirname(resultsFile), ".gitignore"), "*\n");
467
- await writeFile(resultsFile, JSON.stringify({ version: 1, laws }, null, 2) + "\n");
534
+ // Each attestation a recorded law ran under is kept with the results.
535
+ const attestations = Object.fromEntries(Object.entries({ ...(previous.attestations ?? {}),
536
+ ...(attestation ? { [attestation.run.id]: attestation } : {}) })
537
+ .filter(([id]) => Object.values(laws).some((run) => run.environment === id)));
538
+ await writeFile(resultsFile, JSON.stringify({ version: 1, laws,
539
+ ...(Object.keys(attestations).length ? { attestations } : {}) }, null, 2) + "\n");
468
540
  const flaky = statistics.filter((s) => s.outcome === "flaky").map((s) => s.law);
469
541
  const unmet = statistics.flatMap((s) => (s.cover ?? []).filter((c) => !c.met).map((c) => `${s.law}: cover ${c.required}% "${c.label}" (${c.observed}%)`));
470
542
  const benchmarks = statistics.filter((s) => s.benchmark);
471
543
  // How each parallel unit's tests actually ran at the same time.
472
544
  const parallelism = statistics.filter((s) => s.parallel).map(({ parallel, mode, workers }) => ({ unit: parallel, mode, workers }));
473
- summaries.push({ target: target.language, seed: Number(seed), ran: stale.map((e) => e.law),
545
+ summaries.push({ target: target.language, root: target.root, seed: Number(seed), ran: stale.map((e) => e.law),
546
+ ...(attestation ? { environment: attestation.environment } : {}),
474
547
  unchanged: chosen.length - stale.length, ok: !failed,
475
548
  ...(chosen.length !== planned.tests.length ? { deselected: planned.tests.length - chosen.length } : {}),
476
549
  ...(unrun.length ? { unrun: unrun.map((e) => e.law) } : {}),
@@ -478,14 +551,15 @@ async function runTests(compiler, input, selected, roots, config) {
478
551
  ...(unmet.length ? { unmetCover: unmet } : {}),
479
552
  ...(benchmarks.length ? { benchmarks } : {}),
480
553
  ...(parallelism.length ? { parallelism } : {}),
481
- ...(useCoverage ? { coverage: path.relative(process.cwd(), coverage) || "." } : {}) });
554
+ ...(coverageError ? {coverageError} : {}),
555
+ ...(useCoverage && (!beam || coverageWritten) ? { coverage: path.relative(process.cwd(), coverage) || "." } : {}) });
482
556
  }
483
557
  if (report) {
484
558
  await mkdir(path.dirname(path.resolve(report.path)), { recursive: true });
485
559
  await writeFile(path.resolve(report.path), mergeJunit(junit));
486
560
  }
487
561
  output(options.json ? summaries : summaries.map((s) =>
488
- `${s.target}: ${s.ran.length ? `ran ${s.ran.length} law(s) with seed ${s.seed}` : "nothing to run"}` +
562
+ `${s.target}${summaries.filter(other => other.target === s.target).length > 1 ? ` (${s.root})` : ""}: ${s.ran.length ? `ran ${s.ran.length} law(s) with seed ${s.seed}${s.environment ? ` in environment ${s.environment}` : ""}` : "nothing to run"}` +
489
563
  `, ${s.unchanged} unchanged since their last passing run.${s.ok ? "" : " FAILED"}` +
490
564
  (s.deselected ? `\n${s.deselected} law(s) not selected by --tag or --exclude-tag.` : "") +
491
565
  (s.unrun ? `\nNo tests ran for ${s.unrun.join(", ")}; the runner matched none of their tests.` : "") +
@@ -493,6 +567,7 @@ async function runTests(compiler, input, selected, roots, config) {
493
567
  (s.unmetCover ? `\nCover not met: ${s.unmetCover.join("; ")}` : "") +
494
568
  (s.parallelism ? "\n" + s.parallelism.map((p) => `parallel ${p.unit}: ${p.mode}, ${p.workers} worker(s)`).join("\n") : "") +
495
569
  (s.benchmarks ? "\n" + s.benchmarks.map((b) => `benchmark ${b.benchmark}: mean ${(b.mean_ns / 1000).toFixed(2)} us over ${b.iterations} iteration(s)`).join("\n") : "") +
570
+ (s.coverageError ? `\nCoverage failed: ${s.coverageError}` : "") +
496
571
  (s.coverage ? `\nCoverage written to ${s.coverage}.` : "")).join("\n"));
497
572
  if (summaries.some((s) => !s.ok)) process.exitCode = 1;
498
573
  }
@@ -515,10 +590,14 @@ async function junitOf(run, executed, root, since) {
515
590
  }
516
591
  return xml.join("\n");
517
592
  }
518
- return junitFromTests(executed);
593
+ // BEAM reports only executed cases. Intentional harness skips remain
594
+ // reportable obligations; they are never evidence that a test passed.
595
+ const skipped = run.report?.kind === "beam-events" ? run.laws.filter((entry) => entry.skip)
596
+ .map((entry) => ({name: entry.label, classname: entry.unit, status: "skipped", reason: entry.skip})) : [];
597
+ return junitFromTests([...executed, ...skipped]);
519
598
  }
520
599
  // The tool --coverage needs, if it is missing.
521
- async function coverageMissing(target, root) {
600
+ async function coverageMissing(target, root, report) {
522
601
  const tool = coverageTools[target.language];
523
602
  if (!tool) return { tool: "a coverage tool", install: "no coverage tool is known for this target" };
524
603
  const probe = (command, args) => new Promise((resolve) => {
@@ -526,6 +605,12 @@ async function coverageMissing(target, root) {
526
605
  child.on("error", () => resolve(false));
527
606
  child.on("close", (code) => resolve(code === 0));
528
607
  });
608
+ if (["erlang", "elixir", "gleam"].includes(target.language)) {
609
+ const requirement = beamCoverageRequirement(target, report);
610
+ if (requirement) return requirement;
611
+ return (await probe("erl", ["-noshell", "-eval",
612
+ 'case code:ensure_loaded(cover) of {module,cover} -> halt(0); _ -> halt(1) end.'])) ? null : tool;
613
+ }
529
614
  if (target.language === "python") return (await probe(target.python || "python3", tool.check)) ? null : tool;
530
615
  if (["javascript", "typescript"].includes(target.language)) return (await probe("npx", ["--no-install", "c8", "--version"])) ? null : tool;
531
616
  if (target.language === "rust") return (await probe("cargo", ["llvm-cov", "--version"])) ? null : tool;
@@ -575,7 +660,7 @@ async function main() {
575
660
  }
576
661
  if (!verb || ["help", "--help", "-h"].includes(verb)) {
577
662
  output(
578
- "LawSpec " + VERSION + "\nUsage: lawspec init --target <language> [--project <directory>] [--minify]\n lawspec check | doctor | explain <unit>::<law> | generate\n lawspec evidence [<unit> | <unit>::<declaration>]\n lawspec test [--target <language>] [--fresh] [--seed <n>] [--update-recorded] [--tag <t>] [--exclude-tag <t>] [--report junit=<path>] [--coverage] [--benchmarks]\n lawspec package [--project <package directory>]\n lawspec examples [--example payments] [--target <language>] [--output <directory>]\nOptions: --config <path>, --target <language>, --machine-bits <32|64>, --json, --no-cache\nGeneration: --dry-run, --check, --minify\nTargets: " +
663
+ "LawSpec " + VERSION + "\nUsage: lawspec init --target <language> [--project <directory>] [--minify]\n lawspec check | doctor | explain <unit>::<law> | generate\n lawspec evidence [<unit> | <unit>::<declaration>] [--attestation <file>]\n lawspec test [--target <language>] [--fresh] [--seed <n>] [--attestation <file>] [--update-recorded] [--tag <t>] [--exclude-tag <t>] [--report junit=<path>] [--coverage] [--benchmarks]\n lawspec package [--project <package directory>]\n lawspec examples [--example payments] [--target <language>] [--output <directory>]\nOptions: --config <path>, --target <language>, --machine-bits <32|64>, --json, --no-cache\nGeneration: --dry-run, --check, --minify\nTargets: " +
579
664
  targets.join(", "),
580
665
  );
581
666
  return;
@@ -622,6 +707,8 @@ async function main() {
622
707
  if ((options.fresh || options.seed !== undefined || options.tag || options["exclude-tag"] ||
623
708
  options.report !== undefined || options.coverage || options["update-recorded"] || options.benchmarks) && verb !== "test")
624
709
  throw new Error("--fresh, --seed, --update-recorded, --tag, --exclude-tag, --report, --coverage and --benchmarks apply to test");
710
+ if (options.attestation !== undefined && !["evidence", "test"].includes(verb))
711
+ throw new Error("--attestation applies to evidence and test");
625
712
  if (options.seed !== undefined && !/^[0-9]+$/.test(options.seed))
626
713
  throw new Error("--seed must be a whole number");
627
714
  if (verb === "init") return init();
@@ -664,6 +751,8 @@ async function main() {
664
751
  }
665
752
  if (config.machineBits !== undefined && ![32, 64].includes(config.machineBits)) throw new Error("machineBits must be 32 or 64");
666
753
  if (config.cache !== undefined && typeof config.cache !== "boolean") throw new Error("cache must be true or false");
754
+ if (config.attestation !== undefined && typeof config.attestation !== "string")
755
+ throw new Error("attestation must be the path of an environment attestation");
667
756
  const cache = await cacheDirectory(config);
668
757
  const input = {
669
758
  ...(cache === undefined ? {} : { cacheDirectory: cache }),
@@ -712,8 +801,23 @@ async function main() {
712
801
  return;
713
802
  }
714
803
  if (verb === "evidence") {
715
- const result = diagnostics(await compiler.check(input));
804
+ // An attestation named here, or in lawspec.json, says where the tests
805
+ // ran; otherwise each law shows the attested run it last passed in.
806
+ const named = await readAttestation(config);
807
+ const result = diagnostics(await compiler.check({ ...input, ...(named ? { attestation: named } : {}) }));
716
808
  let evidence = result.evidence;
809
+ const { runs, attestations } = await lastRuns();
810
+ const environments = named ? [result.attestation] : [];
811
+ if (!named) for (const [id, attestation] of attestations) {
812
+ const laws = new Set([...runs].filter(([, run]) => run.environment === id).map(([law]) => law));
813
+ if (!laws.size) continue;
814
+ const attested = diagnostics(await compiler.check({ ...input, attestation }));
815
+ const fields = new Map(attested.evidence.filter((item) => item.against && laws.has(item.declaration))
816
+ .map(({ declaration, environment, environmentRun, against }) => [declaration, { environment, environmentRun, against }]));
817
+ evidence = evidence.map((item) => item.stage === "law" && fields.has(item.declaration) ? { ...item, ...fields.get(item.declaration) } : item);
818
+ environments.push({ ...attested.attestation, dependencies: attested.attestation.dependencies
819
+ .map((d) => ({ ...d, obligations: d.obligations.filter((law) => laws.has(law)) })) });
820
+ }
717
821
  // Native bindings are configured per target; their obligations are added.
718
822
  for (const target of selected) {
719
823
  if (target.nativeBindings === undefined) continue;
@@ -727,7 +831,6 @@ async function main() {
727
831
  if (positional[0] && !evidence.length) throw new Error("No matching obligation");
728
832
  // The last run's outcome (lawspec test): a law that failed, then passed
729
833
  // on a retry, is flaky; what its generated cases covered is its adequacy.
730
- const runs = await lastRuns();
731
834
  evidence = evidence.map((item) => {
732
835
  const run = item.stage === "law" ? runs.get(item.declaration) : undefined;
733
836
  if (!run) return item;
@@ -745,12 +848,18 @@ async function main() {
745
848
  if (!items.length) return null;
746
849
  return `${label} (${items.length})\n` + items.map((item) =>
747
850
  ` ${item.stage} ${obligationName(item)}${item.target ? ` [${item.target}]` : ""}` +
748
- (item.claim ? `: ${item.claim.text}` : "") + `\n ${item.reason}`).join("\n");
851
+ (item.claim ? `: ${item.claim.text}` : "") + `\n ${item.reason}` +
852
+ (item.against ? `\n ran in environment ${item.environment} (run ${item.environmentRun.id}) against ` +
853
+ item.against.map(dependencyText).join("; ") : "")).join("\n");
749
854
  })
750
855
  .filter((section) => section !== null),
751
856
  // How each law is discharged: the harness plane, apart from what
752
857
  // the laws say.
753
- ...(harnessed.length ? [`HARNESS (${harnessed.length})\n` + harnessed.map(harnessText).join("\n")] : [])]
858
+ ...(harnessed.length ? [`HARNESS (${harnessed.length})\n` + harnessed.map(harnessText).join("\n")] : []),
859
+ // Where the tests ran: reporting only, so no status above changes.
860
+ ...environments.map((e) => `ENVIRONMENT ${e.environment} (run ${e.run.id}, recorded ${e.run.recordedAt})\n` +
861
+ e.dependencies.map((d) => ` ${dependencyText(d)}; ` + (d.obligations.length
862
+ ? `${d.obligations.length} tested obligation(s)` : "no tested obligation relies on it")).join("\n"))]
754
863
  .join("\n\n"),
755
864
  );
756
865
  return;