lawspec 0.22.0 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/CHANGELOG.md +358 -2
  2. package/README.md +3 -3
  3. package/bin/lawspec.mjs +95 -14
  4. package/build.json +314 -120
  5. package/compatibility.json +29 -0
  6. package/core.wasm +2 -2
  7. package/doctor.mjs +119 -1
  8. package/examples/native-payments/clojure/lawspec.json +51 -0
  9. package/examples/native-payments/clojure/src/domain/payments.clj +11 -0
  10. package/examples/native-payments/clojure/test/domain/payment_generators.clj +9 -0
  11. package/examples/native-payments/csharp/lawspec.json +51 -0
  12. package/examples/native-payments/csharp/src/Domain/Payments.cs +19 -0
  13. package/examples/native-payments/csharp/tests/Native/PaymentGenerators.cs +14 -0
  14. package/examples/native-payments/fsharp/lawspec.json +167 -0
  15. package/examples/native-payments/fsharp/src/Native/Types/PaymentsDomain.fs +23 -0
  16. package/examples/native-payments/fsharp/tests/Native/PaymentGenerators.fs +11 -0
  17. package/examples/native-payments/groovy/lawspec.json +165 -0
  18. package/examples/native-payments/groovy/src/main/groovy/domain/PaymentsDomain.groovy +49 -0
  19. package/examples/native-payments/groovy/src/test/groovy/domain/PaymentGenerators.groovy +15 -0
  20. package/examples/native-payments/scala/lawspec.json +165 -0
  21. package/examples/native-payments/scala/src/main/scala/domain/PaymentsDomain.scala +18 -0
  22. package/examples/native-payments/scala/src/test/scala/domain/PaymentGenerators.scala +8 -0
  23. package/examples/native-payments/swift/lawspec.json +154 -0
  24. package/examples/native-payments/swift/src/Native/PaymentsDomain.swift +35 -0
  25. package/examples/native-payments/swift/tests/Native/PaymentGenerators.swift +12 -0
  26. package/examples/specs/collections.lawspec +44 -0
  27. package/examples/specs/composition.lawspec +110 -0
  28. package/examples/specs/contracts.lawspec +124 -0
  29. package/examples/specs/generation.lawspec +11 -0
  30. package/examples/specs/records.lawspec +191 -0
  31. package/examples/specs/records_inner.lawspec +10 -0
  32. package/examples/specs/sequences.lawspec +86 -0
  33. package/index.d.ts +40 -1
  34. package/package.json +2 -2
  35. package/scala-coverage.mjs +56 -0
  36. package/templates.mjs +107 -3
  37. package/test-command.mjs +105 -4
package/bin/lawspec.mjs CHANGED
@@ -9,6 +9,7 @@ import { generateExamples } from "../examples-command.mjs";
9
9
  import { showScalar } from "../scalars.mjs";
10
10
  import { doctor } from "../doctor.mjs";
11
11
  import { beamCoverageRequirement, collectBeamCoverage } from "../beam-coverage.mjs";
12
+ import { collectScalaCoverage } from "../scala-coverage.mjs";
12
13
  import { failureDatabase, projectKey } from "../failure-database.mjs";
13
14
  import { spawn, spawnSync } from "node:child_process";
14
15
  import { benchmarkInvocations, preparationInvocations, environmentDigest, executedTests, invocations, lawKeys, projectDigest, recordedDigest, selectByTags, mergeJunit,
@@ -20,14 +21,14 @@ import {
20
21
  atomicWrite,
21
22
  safePath,
22
23
  } from "../files.mjs";
23
- const VERSION = "0.22.0";
24
+ const VERSION = "0.24.0";
24
25
  const args = process.argv.slice(2);
25
26
  const verb = args.shift();
26
27
  const options = {};
27
28
  const positional = [];
28
29
  for (let i = 0; i < args.length; i++) {
29
30
  const arg = args[i];
30
- if (["--target", "--project", "--config", "--output", "--machine-bits", "--example", "--seed", "--report"].includes(arg)) {
31
+ if (["--target", "--project", "--config", "--output", "--machine-bits", "--example", "--seed", "--report", "--attestation"].includes(arg)) {
31
32
  if (!args[i + 1] || args[i + 1].startsWith("--"))
32
33
  throw new Error(`Missing value for ${arg}`);
33
34
  options[arg.slice(2)] = args[++i];
@@ -58,6 +59,8 @@ function testCommand(target, root) {
58
59
  command = `${shellQuote(target.maven || "mvn")} test`;
59
60
  if (target.language === "kotlin")
60
61
  command = `${shellQuote(target.gradle || "gradle")} test`;
62
+ if (target.language === "scala")
63
+ command = `${shellQuote(target.scala || "scala")} test .`;
61
64
  if (target.language === "javascript")
62
65
  command = `node --test ${shellQuote(testDir)}/*.test.mjs`;
63
66
  if (target.language === "typescript")
@@ -272,19 +275,49 @@ function explainExamples(law) {
272
275
  ex.expectations.map(e => ` expect ${showAssertion(e)}`).join("\n")
273
276
  ).join("\n");
274
277
  }
275
- // What lawspec test last recorded for each law, across targets.
278
+ // What lawspec test last recorded for each law, across targets, and the
279
+ // environment attestations its runs were given, by run id. A law passed in
280
+ // an attested run names that run; across targets, its latest passing run.
276
281
  async function lastRuns() {
277
282
  const folder = path.join(configRoot, ".lawspec", "results");
278
283
  const runs = new Map();
284
+ const attestations = new Map();
279
285
  for (const name of (await readOptionalDirectory(folder)) ?? []) {
280
286
  if (!name.endsWith(".json")) continue;
281
287
  const recorded = JSON.parse((await readOptional(path.join(folder, name))) ?? "{}");
288
+ for (const [id, attestation] of Object.entries(recorded.attestations ?? {})) attestations.set(id, attestation);
282
289
  for (const [law, run] of Object.entries(recorded.laws ?? {})) {
283
290
  const known = runs.get(law) ?? {};
284
- runs.set(law, { flaky: known.flaky || run.flaky, adequacy: run.adequacy ?? known.adequacy });
291
+ const latest = (known.passed ?? "") > (run.passed ?? "") ? known : run;
292
+ runs.set(law, { flaky: known.flaky || run.flaky, adequacy: run.adequacy ?? known.adequacy,
293
+ passed: latest.passed, environment: latest.environment });
285
294
  }
286
295
  }
287
- return runs;
296
+ return { runs, attestations };
297
+ }
298
+ // An environment attestation: a versioned JSON document any tool may write,
299
+ // recording for one run where each dependency was wired and whether it was
300
+ // the real dependency or a stand-in. --attestation names one, or lawspec.json
301
+ // does; the compiler checks it. It is reporting only and changes no evidence.
302
+ async function readAttestation(config) {
303
+ const file = options.attestation !== undefined ? path.resolve(options.attestation)
304
+ : config.attestation !== undefined ? path.resolve(configRoot, config.attestation) : null;
305
+ if (file === null) return null;
306
+ const text = await readOptional(file);
307
+ if (text === null) throw new Error(`No environment attestation at ${file}`);
308
+ try {
309
+ return JSON.parse(text);
310
+ } catch (error) {
311
+ throw new Error(`${file}: the environment attestation is not JSON: ${error.message}`);
312
+ }
313
+ }
314
+ // One attested dependency, as lawspec evidence shows it.
315
+ function dependencyText(d) {
316
+ const level = (c) => /^[0-9]+$/.test(c) ? `conformance level ${c}` : `conformance ${c}`;
317
+ return `${d.dependency}: ` + (d.standIn
318
+ ? `stand-in ${d.standIn.identity} ${d.standIn.version} (${level(d.standIn.conformance)}` +
319
+ (d.standIn.historyDigest ? `, history ${d.standIn.historyDigest}` : "") + ")"
320
+ : "real") + ` at ${d.endpoint}` + (d.reached ? "" : " (not reached when recorded)");
288
321
  }
289
322
  // One law's harness, as lawspec evidence shows it.
290
323
  function harnessText(item) {
@@ -358,12 +391,18 @@ async function runTests(compiler, input, selected, roots, config) {
358
391
  const summaries = [];
359
392
  const junit = [];
360
393
  const report = options.report === undefined ? null : parseReport(options.report);
394
+ const attestation = await readAttestation(config);
361
395
  for (const [i, target] of selected.entries()) {
362
396
  const root = roots[i];
363
397
  const planned = diagnostics(await compiler.planGeneration({
364
398
  ...input, target: target.language, sourceDir: target.sourceDir, testDir: target.testDir,
365
399
  nativeBindings: target.nativeBindings, minify: options.minify === true,
400
+ ...(attestation ? { attestation } : {}),
366
401
  }));
402
+ // The laws the attestation touches, each with how its dependencies
403
+ // were wired (the run's id and times aside).
404
+ const attested = new Map((planned.evidence ?? []).filter((item) => item.against)
405
+ .map((item) => [item.declaration, JSON.stringify([item.environment, item.against])]));
367
406
  const pending = await planWrites(root, planned.files);
368
407
  if (pending.changes.length)
369
408
  throw new Error(`${target.language}: generated files are out of date; run lawspec generate${options.minify ? " --minify" : ""}`);
@@ -376,6 +415,7 @@ async function runTests(compiler, input, selected, roots, config) {
376
415
  // Recorded values are spec data: a changed recording runs its laws again.
377
416
  environment: await environmentDigest(root, report_) + (await recordedDigest(recordedFolder)),
378
417
  project: await projectDigest(root, generated),
418
+ attested,
379
419
  });
380
420
  const identity = projectKey(root);
381
421
  const resultsFile = path.join(configRoot, ".lawspec", "results", `${target.language}-${identity}.json`);
@@ -413,7 +453,7 @@ async function runTests(compiler, input, selected, roots, config) {
413
453
  if (missingCoverage)
414
454
  console.error(`${target.language}: --coverage needs ${missingCoverage.tool}, which is not available; ${missingCoverage.install}. Running without coverage.`);
415
455
  if (useCoverage) {
416
- if (beam) await rm(coverage, {recursive: true, force: true});
456
+ if (beam || target.language === "scala") await rm(coverage, {recursive: true, force: true});
417
457
  await mkdir(beam ? path.join(coverage, "runs") : coverage, { recursive: true });
418
458
  }
419
459
  // parallel on Python needs pytest-xdist; without it the tests run one
@@ -455,6 +495,14 @@ async function runTests(compiler, input, selected, roots, config) {
455
495
  }
456
496
  }
457
497
  if (unrun.length) failed = true;
498
+ if (useCoverage && target.language === "scala" && !failed) {
499
+ try {
500
+ coverageWritten = await collectScalaCoverage({root, directory: coverage, command: target.scala || "scala", offline});
501
+ } catch (error) {
502
+ failed = true;
503
+ coverageError = error.message;
504
+ }
505
+ }
458
506
  if (useCoverage && beam) {
459
507
  try {
460
508
  coverageWritten = !!(await collectBeamCoverage({root, directory: coverage, runs: coverageRuns}));
@@ -476,6 +524,8 @@ async function runTests(compiler, input, selected, roots, config) {
476
524
  const label = entry.label;
477
525
  const observed = statistics.filter((s) => s.law === label);
478
526
  laws[entry.law] = { key: keys.get(entry.law), seed: Number(seedOf.get(entry.law) ?? seed), passed: new Date().toISOString(),
527
+ // A law the attestation touches names the run it passed in.
528
+ ...(attestation && attested.has(entry.law) ? { environment: attestation.run.id } : {}),
479
529
  ...(observed.some((s) => s.outcome === "flaky") ? { flaky: true } : {}),
480
530
  ...(observed.some((s) => s.cover || s.labels || s.classes) ? { adequacy: observed.filter((s) => s.cover || s.labels || s.classes)
481
531
  .map(({ test, cases, cover, classes, labels }) => ({ test, cases, cover, classes, labels })) } : {}) };
@@ -492,13 +542,19 @@ async function runTests(compiler, input, selected, roots, config) {
492
542
  await writeFile(databaseFile, JSON.stringify(database, null, 2) + "\n");
493
543
  await mkdir(path.dirname(resultsFile), { recursive: true });
494
544
  await writeFile(path.join(path.dirname(resultsFile), ".gitignore"), "*\n");
495
- await writeFile(resultsFile, JSON.stringify({ version: 1, laws }, null, 2) + "\n");
545
+ // Each attestation a recorded law ran under is kept with the results.
546
+ const attestations = Object.fromEntries(Object.entries({ ...(previous.attestations ?? {}),
547
+ ...(attestation ? { [attestation.run.id]: attestation } : {}) })
548
+ .filter(([id]) => Object.values(laws).some((run) => run.environment === id)));
549
+ await writeFile(resultsFile, JSON.stringify({ version: 1, laws,
550
+ ...(Object.keys(attestations).length ? { attestations } : {}) }, null, 2) + "\n");
496
551
  const flaky = statistics.filter((s) => s.outcome === "flaky").map((s) => s.law);
497
552
  const unmet = statistics.flatMap((s) => (s.cover ?? []).filter((c) => !c.met).map((c) => `${s.law}: cover ${c.required}% "${c.label}" (${c.observed}%)`));
498
553
  const benchmarks = statistics.filter((s) => s.benchmark);
499
554
  // How each parallel unit's tests actually ran at the same time.
500
555
  const parallelism = statistics.filter((s) => s.parallel).map(({ parallel, mode, workers }) => ({ unit: parallel, mode, workers }));
501
556
  summaries.push({ target: target.language, root: target.root, seed: Number(seed), ran: stale.map((e) => e.law),
557
+ ...(attestation ? { environment: attestation.environment } : {}),
502
558
  unchanged: chosen.length - stale.length, ok: !failed,
503
559
  ...(chosen.length !== planned.tests.length ? { deselected: planned.tests.length - chosen.length } : {}),
504
560
  ...(unrun.length ? { unrun: unrun.map((e) => e.law) } : {}),
@@ -507,14 +563,14 @@ async function runTests(compiler, input, selected, roots, config) {
507
563
  ...(benchmarks.length ? { benchmarks } : {}),
508
564
  ...(parallelism.length ? { parallelism } : {}),
509
565
  ...(coverageError ? {coverageError} : {}),
510
- ...(useCoverage && (!beam || coverageWritten) ? { coverage: path.relative(process.cwd(), coverage) || "." } : {}) });
566
+ ...(useCoverage && ((!beam && target.language !== "scala") || coverageWritten) ? { coverage: path.relative(process.cwd(), coverage) || "." } : {}) });
511
567
  }
512
568
  if (report) {
513
569
  await mkdir(path.dirname(path.resolve(report.path)), { recursive: true });
514
570
  await writeFile(path.resolve(report.path), mergeJunit(junit));
515
571
  }
516
572
  output(options.json ? summaries : summaries.map((s) =>
517
- `${s.target}${summaries.filter(other => other.target === s.target).length > 1 ? ` (${s.root})` : ""}: ${s.ran.length ? `ran ${s.ran.length} law(s) with seed ${s.seed}` : "nothing to run"}` +
573
+ `${s.target}${summaries.filter(other => other.target === s.target).length > 1 ? ` (${s.root})` : ""}: ${s.ran.length ? `ran ${s.ran.length} law(s) with seed ${s.seed}${s.environment ? ` in environment ${s.environment}` : ""}` : "nothing to run"}` +
518
574
  `, ${s.unchanged} unchanged since their last passing run.${s.ok ? "" : " FAILED"}` +
519
575
  (s.deselected ? `\n${s.deselected} law(s) not selected by --tag or --exclude-tag.` : "") +
520
576
  (s.unrun ? `\nNo tests ran for ${s.unrun.join(", ")}; the runner matched none of their tests.` : "") +
@@ -573,6 +629,7 @@ async function coverageMissing(target, root, report) {
573
629
  const build = await readFile(path.join(root, "build.gradle.kts"), "utf8").catch(() => "");
574
630
  return build.includes("kover") ? null : tool;
575
631
  }
632
+ if (target.language === "scala") return null;
576
633
  return null;
577
634
  }
578
635
  // Run a native test command, showing its output (on stderr with --json) and
@@ -615,7 +672,7 @@ async function main() {
615
672
  }
616
673
  if (!verb || ["help", "--help", "-h"].includes(verb)) {
617
674
  output(
618
- "LawSpec " + VERSION + "\nUsage: lawspec init --target <language> [--project <directory>] [--minify]\n lawspec check | doctor | explain <unit>::<law> | generate\n lawspec evidence [<unit> | <unit>::<declaration>]\n lawspec test [--target <language>] [--fresh] [--seed <n>] [--update-recorded] [--tag <t>] [--exclude-tag <t>] [--report junit=<path>] [--coverage] [--benchmarks]\n lawspec package [--project <package directory>]\n lawspec examples [--example payments] [--target <language>] [--output <directory>]\nOptions: --config <path>, --target <language>, --machine-bits <32|64>, --json, --no-cache\nGeneration: --dry-run, --check, --minify\nTargets: " +
675
+ "LawSpec " + VERSION + "\nUsage: lawspec init --target <language> [--project <directory>] [--minify]\n lawspec check | doctor | explain <unit>::<law> | generate\n lawspec evidence [<unit> | <unit>::<declaration>] [--attestation <file>]\n lawspec test [--target <language>] [--fresh] [--seed <n>] [--attestation <file>] [--update-recorded] [--tag <t>] [--exclude-tag <t>] [--report junit=<path>] [--coverage] [--benchmarks]\n lawspec package [--project <package directory>]\n lawspec examples [--example payments] [--target <language>] [--output <directory>]\nOptions: --config <path>, --target <language>, --machine-bits <32|64>, --json, --no-cache\nGeneration: --dry-run, --check, --minify\nTargets: " +
619
676
  targets.join(", "),
620
677
  );
621
678
  return;
@@ -662,6 +719,8 @@ async function main() {
662
719
  if ((options.fresh || options.seed !== undefined || options.tag || options["exclude-tag"] ||
663
720
  options.report !== undefined || options.coverage || options["update-recorded"] || options.benchmarks) && verb !== "test")
664
721
  throw new Error("--fresh, --seed, --update-recorded, --tag, --exclude-tag, --report, --coverage and --benchmarks apply to test");
722
+ if (options.attestation !== undefined && !["evidence", "test"].includes(verb))
723
+ throw new Error("--attestation applies to evidence and test");
665
724
  if (options.seed !== undefined && !/^[0-9]+$/.test(options.seed))
666
725
  throw new Error("--seed must be a whole number");
667
726
  if (verb === "init") return init();
@@ -704,6 +763,8 @@ async function main() {
704
763
  }
705
764
  if (config.machineBits !== undefined && ![32, 64].includes(config.machineBits)) throw new Error("machineBits must be 32 or 64");
706
765
  if (config.cache !== undefined && typeof config.cache !== "boolean") throw new Error("cache must be true or false");
766
+ if (config.attestation !== undefined && typeof config.attestation !== "string")
767
+ throw new Error("attestation must be the path of an environment attestation");
707
768
  const cache = await cacheDirectory(config);
708
769
  const input = {
709
770
  ...(cache === undefined ? {} : { cacheDirectory: cache }),
@@ -752,8 +813,23 @@ async function main() {
752
813
  return;
753
814
  }
754
815
  if (verb === "evidence") {
755
- const result = diagnostics(await compiler.check(input));
816
+ // An attestation named here, or in lawspec.json, says where the tests
817
+ // ran; otherwise each law shows the attested run it last passed in.
818
+ const named = await readAttestation(config);
819
+ const result = diagnostics(await compiler.check({ ...input, ...(named ? { attestation: named } : {}) }));
756
820
  let evidence = result.evidence;
821
+ const { runs, attestations } = await lastRuns();
822
+ const environments = named ? [result.attestation] : [];
823
+ if (!named) for (const [id, attestation] of attestations) {
824
+ const laws = new Set([...runs].filter(([, run]) => run.environment === id).map(([law]) => law));
825
+ if (!laws.size) continue;
826
+ const attested = diagnostics(await compiler.check({ ...input, attestation }));
827
+ const fields = new Map(attested.evidence.filter((item) => item.against && laws.has(item.declaration))
828
+ .map(({ declaration, environment, environmentRun, against }) => [declaration, { environment, environmentRun, against }]));
829
+ evidence = evidence.map((item) => item.stage === "law" && fields.has(item.declaration) ? { ...item, ...fields.get(item.declaration) } : item);
830
+ environments.push({ ...attested.attestation, dependencies: attested.attestation.dependencies
831
+ .map((d) => ({ ...d, obligations: d.obligations.filter((law) => laws.has(law)) })) });
832
+ }
757
833
  // Native bindings are configured per target; their obligations are added.
758
834
  for (const target of selected) {
759
835
  if (target.nativeBindings === undefined) continue;
@@ -767,7 +843,6 @@ async function main() {
767
843
  if (positional[0] && !evidence.length) throw new Error("No matching obligation");
768
844
  // The last run's outcome (lawspec test): a law that failed, then passed
769
845
  // on a retry, is flaky; what its generated cases covered is its adequacy.
770
- const runs = await lastRuns();
771
846
  evidence = evidence.map((item) => {
772
847
  const run = item.stage === "law" ? runs.get(item.declaration) : undefined;
773
848
  if (!run) return item;
@@ -785,12 +860,18 @@ async function main() {
785
860
  if (!items.length) return null;
786
861
  return `${label} (${items.length})\n` + items.map((item) =>
787
862
  ` ${item.stage} ${obligationName(item)}${item.target ? ` [${item.target}]` : ""}` +
788
- (item.claim ? `: ${item.claim.text}` : "") + `\n ${item.reason}`).join("\n");
863
+ (item.claim ? `: ${item.claim.text}` : "") + `\n ${item.reason}` +
864
+ (item.against ? `\n ran in environment ${item.environment} (run ${item.environmentRun.id}) against ` +
865
+ item.against.map(dependencyText).join("; ") : "")).join("\n");
789
866
  })
790
867
  .filter((section) => section !== null),
791
868
  // How each law is discharged: the harness plane, apart from what
792
869
  // the laws say.
793
- ...(harnessed.length ? [`HARNESS (${harnessed.length})\n` + harnessed.map(harnessText).join("\n")] : [])]
870
+ ...(harnessed.length ? [`HARNESS (${harnessed.length})\n` + harnessed.map(harnessText).join("\n")] : []),
871
+ // Where the tests ran: reporting only, so no status above changes.
872
+ ...environments.map((e) => `ENVIRONMENT ${e.environment} (run ${e.run.id}, recorded ${e.run.recordedAt})\n` +
873
+ e.dependencies.map((d) => ` ${dependencyText(d)}; ` + (d.obligations.length
874
+ ? `${d.obligations.length} tested obligation(s)` : "no tested obligation relies on it")).join("\n"))]
794
875
  .join("\n\n"),
795
876
  );
796
877
  return;